@crawlee/utils 4.0.0-beta.9 → 4.0.0-beta.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/index.d.ts +4 -6
- package/index.js +3 -5
- package/internals/blocked.d.ts +0 -1
- package/internals/blocked.js +0 -1
- package/internals/cheerio.d.ts +3 -2
- package/internals/cheerio.js +4 -5
- package/internals/extract-urls.d.ts +5 -1
- package/internals/extract-urls.js +8 -5
- package/internals/general.d.ts +16 -19
- package/internals/general.js +44 -96
- package/internals/iterables.d.ts +126 -0
- package/internals/iterables.js +230 -0
- package/internals/open_graph_parser.d.ts +2 -3
- package/internals/open_graph_parser.js +8 -9
- package/internals/robots.d.ts +14 -5
- package/internals/robots.js +31 -35
- package/internals/sitemap.d.ts +65 -8
- package/internals/sitemap.js +174 -34
- package/internals/social.d.ts +1 -2
- package/internals/social.js +7 -5
- package/internals/typedefs.d.ts +0 -1
- package/internals/typedefs.js +0 -1
- package/internals/url.d.ts +1 -2
- package/internals/url.js +1 -2
- package/package.json +6 -6
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -31
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -29
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/internals/sitemap.js
CHANGED
|
@@ -2,9 +2,12 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
|
+
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
|
+
import { fileTypeStream } from 'file-type';
|
|
5
7
|
import sax from 'sax';
|
|
6
8
|
import MIMEType from 'whatwg-mimetype';
|
|
7
|
-
import
|
|
9
|
+
import { mergeAsyncIterables } from './iterables.js';
|
|
10
|
+
import { RobotsTxtFile } from './robots.js';
|
|
8
11
|
class SitemapTxtParser extends Transform {
|
|
9
12
|
decoder = new StringDecoder('utf8');
|
|
10
13
|
buffer = '';
|
|
@@ -91,8 +94,10 @@ class SitemapXmlParser extends Transform {
|
|
|
91
94
|
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
92
95
|
this.currentTag = undefined;
|
|
93
96
|
}
|
|
94
|
-
if (name === 'url'
|
|
95
|
-
|
|
97
|
+
if (name === 'url') {
|
|
98
|
+
if (this.url.loc !== undefined) {
|
|
99
|
+
this.push({ type: 'url', ...this.url, loc: this.url.loc });
|
|
100
|
+
}
|
|
96
101
|
this.url = {};
|
|
97
102
|
}
|
|
98
103
|
}
|
|
@@ -108,7 +113,10 @@ class SitemapXmlParser extends Transform {
|
|
|
108
113
|
}
|
|
109
114
|
text = text.trim();
|
|
110
115
|
if (this.currentTag === 'lastmod') {
|
|
111
|
-
|
|
116
|
+
const lastmod = new Date(text);
|
|
117
|
+
if (!Number.isNaN(lastmod.getTime())) {
|
|
118
|
+
this.url.lastmod = lastmod;
|
|
119
|
+
}
|
|
112
120
|
}
|
|
113
121
|
if (this.currentTag === 'priority') {
|
|
114
122
|
this.url.priority = Number(text);
|
|
@@ -121,9 +129,7 @@ class SitemapXmlParser extends Transform {
|
|
|
121
129
|
}
|
|
122
130
|
}
|
|
123
131
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
124
|
-
const {
|
|
125
|
-
const { fileTypeStream } = await import('file-type');
|
|
126
|
-
const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts } = options ?? {};
|
|
132
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
|
|
127
133
|
const sources = [...initialSources];
|
|
128
134
|
const visitedSitemapUrls = new Set();
|
|
129
135
|
const createParser = (contentType = '', url) => {
|
|
@@ -145,7 +151,6 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
145
151
|
while (sources.length > 0) {
|
|
146
152
|
const source = sources.shift();
|
|
147
153
|
if ((source?.depth ?? 0) > maxDepth) {
|
|
148
|
-
log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
149
154
|
continue;
|
|
150
155
|
}
|
|
151
156
|
let items = null;
|
|
@@ -155,23 +160,28 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
155
160
|
let retriesLeft = sitemapRetries + 1;
|
|
156
161
|
while (retriesLeft-- > 0) {
|
|
157
162
|
try {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
proxyUrl,
|
|
163
|
+
let sitemapResponse;
|
|
164
|
+
try {
|
|
165
|
+
sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
|
|
162
166
|
method: 'GET',
|
|
163
|
-
timeout: networkTimeouts,
|
|
164
167
|
headers: {
|
|
165
|
-
accept: '
|
|
168
|
+
accept: '*/*',
|
|
166
169
|
},
|
|
170
|
+
}), {
|
|
171
|
+
proxyUrl,
|
|
172
|
+
timeoutMillis: timeout,
|
|
167
173
|
});
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
sitemapResponse = null;
|
|
177
|
+
}
|
|
171
178
|
let error = null;
|
|
172
|
-
if (
|
|
173
|
-
let contentType =
|
|
174
|
-
|
|
179
|
+
if (sitemapResponse && sitemapResponse.status >= 200 && sitemapResponse.status < 300) {
|
|
180
|
+
let contentType = sitemapResponse.headers.get('content-type');
|
|
181
|
+
if (sitemapResponse.body === null) {
|
|
182
|
+
break;
|
|
183
|
+
}
|
|
184
|
+
const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
|
|
175
185
|
if (streamWithType.fileType !== undefined) {
|
|
176
186
|
contentType = streamWithType.fileType.mime;
|
|
177
187
|
}
|
|
@@ -184,29 +194,37 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
184
194
|
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
185
195
|
}
|
|
186
196
|
}
|
|
187
|
-
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType, sitemapUrl), (e) => {
|
|
188
|
-
if (e !== undefined) {
|
|
189
|
-
error = e;
|
|
197
|
+
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
198
|
+
if (e !== undefined && e !== null) {
|
|
199
|
+
error = { type: 'parser', error: e };
|
|
190
200
|
}
|
|
191
201
|
});
|
|
192
202
|
}
|
|
193
203
|
else {
|
|
194
|
-
error =
|
|
204
|
+
error = {
|
|
205
|
+
type: 'fetch',
|
|
206
|
+
error: new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapResponse?.status}`),
|
|
207
|
+
};
|
|
195
208
|
}
|
|
196
209
|
if (error !== null) {
|
|
197
|
-
|
|
210
|
+
const shouldIgnoreError = error.type === 'fetch' && !reportNetworkErrors;
|
|
211
|
+
if (!shouldIgnoreError) {
|
|
212
|
+
throw error.error;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
break;
|
|
198
217
|
}
|
|
199
|
-
break;
|
|
200
218
|
}
|
|
201
219
|
catch (e) {
|
|
202
|
-
|
|
220
|
+
logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
203
221
|
}
|
|
204
222
|
}
|
|
205
223
|
}
|
|
206
224
|
else if (source.type === 'raw') {
|
|
207
225
|
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
208
226
|
if (error !== undefined) {
|
|
209
|
-
|
|
227
|
+
logger?.warning(`Malformed sitemap content: ${error}`);
|
|
210
228
|
}
|
|
211
229
|
});
|
|
212
230
|
}
|
|
@@ -215,6 +233,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
215
233
|
}
|
|
216
234
|
for await (const item of items) {
|
|
217
235
|
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
236
|
+
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
237
|
+
logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
218
240
|
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
219
241
|
if (emitNestedSitemaps) {
|
|
220
242
|
yield { loc: item.url, originSitemapUrl: null };
|
|
@@ -254,7 +276,7 @@ export class Sitemap {
|
|
|
254
276
|
* @param url The domain URL to fetch the sitemap for.
|
|
255
277
|
* @param proxyUrl A proxy to be used for fetching the sitemap file.
|
|
256
278
|
*/
|
|
257
|
-
static async tryCommonNames(url, proxyUrl) {
|
|
279
|
+
static async tryCommonNames(url, proxyUrl, parseSitemapOptions) {
|
|
258
280
|
const sitemapUrls = [];
|
|
259
281
|
const sitemapUrl = new URL(url);
|
|
260
282
|
sitemapUrl.search = '';
|
|
@@ -262,7 +284,7 @@ export class Sitemap {
|
|
|
262
284
|
sitemapUrls.push(sitemapUrl.toString());
|
|
263
285
|
sitemapUrl.pathname = '/sitemap.txt';
|
|
264
286
|
sitemapUrls.push(sitemapUrl.toString());
|
|
265
|
-
return Sitemap.load(sitemapUrls, proxyUrl);
|
|
287
|
+
return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false, ...parseSitemapOptions });
|
|
266
288
|
}
|
|
267
289
|
/**
|
|
268
290
|
* Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
|
|
@@ -277,8 +299,8 @@ export class Sitemap {
|
|
|
277
299
|
* @param content XML sitemap content
|
|
278
300
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
279
301
|
*/
|
|
280
|
-
static async fromXmlString(content, proxyUrl) {
|
|
281
|
-
return await this.parse([{ type: 'raw', content }], proxyUrl);
|
|
302
|
+
static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
|
|
303
|
+
return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
|
|
282
304
|
}
|
|
283
305
|
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
284
306
|
const urls = [];
|
|
@@ -287,10 +309,128 @@ export class Sitemap {
|
|
|
287
309
|
urls.push(item.loc);
|
|
288
310
|
}
|
|
289
311
|
}
|
|
290
|
-
catch {
|
|
312
|
+
catch (e) {
|
|
313
|
+
parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
|
|
291
314
|
return new Sitemap([]);
|
|
292
315
|
}
|
|
293
316
|
return new Sitemap(urls);
|
|
294
317
|
}
|
|
295
318
|
}
|
|
296
|
-
|
|
319
|
+
/**
|
|
320
|
+
* Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
|
|
321
|
+
* the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
|
|
322
|
+
* @param `urls` The list of URLs to discover sitemaps for.
|
|
323
|
+
* @param `options` Options for sitemap discovery
|
|
324
|
+
* @returns An async iterable with the discovered sitemap URLs.
|
|
325
|
+
*/
|
|
326
|
+
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
327
|
+
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
|
|
328
|
+
const controller = new AbortController();
|
|
329
|
+
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
|
|
330
|
+
const onExternalAbort = () => controller.abort();
|
|
331
|
+
if (externalSignal) {
|
|
332
|
+
if (externalSignal.aborted) {
|
|
333
|
+
controller.abort();
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
const signal = controller.signal;
|
|
340
|
+
const sitemapUrls = new Set();
|
|
341
|
+
const addSitemapUrl = (url) => {
|
|
342
|
+
const sizeBefore = sitemapUrls.size;
|
|
343
|
+
sitemapUrls.add(url);
|
|
344
|
+
if (sitemapUrls.size > sizeBefore) {
|
|
345
|
+
return url;
|
|
346
|
+
}
|
|
347
|
+
return undefined;
|
|
348
|
+
};
|
|
349
|
+
const urlExists = async (url) => {
|
|
350
|
+
if (!httpClient) {
|
|
351
|
+
return false;
|
|
352
|
+
}
|
|
353
|
+
try {
|
|
354
|
+
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
355
|
+
proxyUrl,
|
|
356
|
+
timeoutMillis: requestTimeoutMillis,
|
|
357
|
+
signal,
|
|
358
|
+
});
|
|
359
|
+
return response.status >= 200 && response.status < 400;
|
|
360
|
+
}
|
|
361
|
+
catch {
|
|
362
|
+
return false;
|
|
363
|
+
}
|
|
364
|
+
};
|
|
365
|
+
const discoverSitemapsForDomainUrls = async function* (hostname, domainUrls) {
|
|
366
|
+
if (!hostname) {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
try {
|
|
370
|
+
const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
|
|
371
|
+
proxyUrl,
|
|
372
|
+
timeoutMillis: requestTimeoutMillis,
|
|
373
|
+
signal,
|
|
374
|
+
httpClient,
|
|
375
|
+
logger,
|
|
376
|
+
});
|
|
377
|
+
for (const sitemapUrl of robotsFile.getSitemaps()) {
|
|
378
|
+
if (addSitemapUrl(sitemapUrl)) {
|
|
379
|
+
yield sitemapUrl;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
catch (err) {
|
|
384
|
+
logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
385
|
+
}
|
|
386
|
+
const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
387
|
+
if (sitemapUrl !== undefined) {
|
|
388
|
+
if (addSitemapUrl(sitemapUrl)) {
|
|
389
|
+
yield sitemapUrl;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
else {
|
|
393
|
+
const firstUrl = new URL(domainUrls[0]);
|
|
394
|
+
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
395
|
+
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
396
|
+
firstUrl.pathname = pathname;
|
|
397
|
+
return firstUrl.toString();
|
|
398
|
+
});
|
|
399
|
+
const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
|
|
400
|
+
for (const [index, result] of candidateResults.entries()) {
|
|
401
|
+
const candidateSitemapUrl = candidateSitemapUrls[index];
|
|
402
|
+
if (result.status === 'fulfilled') {
|
|
403
|
+
if (result.value && addSitemapUrl(candidateSitemapUrl)) {
|
|
404
|
+
yield candidateSitemapUrl;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
else {
|
|
408
|
+
logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
|
|
409
|
+
error: result.reason,
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
};
|
|
415
|
+
const groupedUrls = urls.reduce((acc, url) => {
|
|
416
|
+
const hostname = new URL(url)?.hostname ?? '';
|
|
417
|
+
acc[hostname] ??= [];
|
|
418
|
+
acc[hostname].push(url);
|
|
419
|
+
return acc;
|
|
420
|
+
}, {});
|
|
421
|
+
const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
|
|
422
|
+
const discoveredUrls = new Set();
|
|
423
|
+
try {
|
|
424
|
+
for await (const url of mergeAsyncIterables(...iterables)) {
|
|
425
|
+
if (discoveredUrls.has(url)) {
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
discoveredUrls.add(url);
|
|
429
|
+
yield url;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
finally {
|
|
433
|
+
clearTimeout(timeoutHandle);
|
|
434
|
+
externalSignal?.removeEventListener('abort', onExternalAbort);
|
|
435
|
+
}
|
|
436
|
+
}
|
package/internals/social.d.ts
CHANGED
|
@@ -488,5 +488,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
|
|
|
488
488
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
489
489
|
* @return An object with the social handles.
|
|
490
490
|
*/
|
|
491
|
-
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles
|
|
492
|
-
//# sourceMappingURL=social.d.ts.map
|
|
491
|
+
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
|
package/internals/social.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { htmlToText } from './cheerio.js';
|
|
3
2
|
// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
|
|
4
|
-
|
|
3
|
+
// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
|
|
4
|
+
// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
|
|
5
|
+
// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
|
|
6
|
+
const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
|
|
5
7
|
/**
|
|
6
8
|
* Regular expression to exactly match a single email address.
|
|
7
9
|
* It has the following form: `/^...$/i`.
|
|
@@ -587,7 +589,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
|
|
|
587
589
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
588
590
|
* @return An object with the social handles.
|
|
589
591
|
*/
|
|
590
|
-
export function parseHandlesFromHtml(html, data = null) {
|
|
592
|
+
export async function parseHandlesFromHtml(html, data = null) {
|
|
593
|
+
const cheerio = await import('cheerio');
|
|
591
594
|
const result = {
|
|
592
595
|
emails: [],
|
|
593
596
|
phones: [],
|
|
@@ -606,7 +609,7 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
606
609
|
const $ = cheerio.load(html, { xml: { decodeEntities: true } });
|
|
607
610
|
if (data)
|
|
608
611
|
data.$ = $;
|
|
609
|
-
const text = htmlToText($);
|
|
612
|
+
const text = await htmlToText($);
|
|
610
613
|
if (data)
|
|
611
614
|
data.text = text;
|
|
612
615
|
// NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
|
|
@@ -644,4 +647,3 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
644
647
|
}
|
|
645
648
|
return result;
|
|
646
649
|
}
|
|
647
|
-
//# sourceMappingURL=social.js.map
|
package/internals/typedefs.d.ts
CHANGED
package/internals/typedefs.js
CHANGED
package/internals/url.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import type { SearchParams } from '@crawlee/types';
|
|
2
2
|
/**
|
|
3
3
|
* Appends search (query string) parameters to a URL, replacing the original value (if any).
|
|
4
4
|
*
|
|
@@ -7,4 +7,3 @@ export type SearchParams = string | URLSearchParams | Record<string, string | nu
|
|
|
7
7
|
* @internal
|
|
8
8
|
*/
|
|
9
9
|
export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
|
|
10
|
-
//# sourceMappingURL=url.d.ts.map
|
package/internals/url.js
CHANGED
|
@@ -19,7 +19,7 @@ export function applySearchParams(url, searchParams) {
|
|
|
19
19
|
}
|
|
20
20
|
else {
|
|
21
21
|
newSearchParams = new URLSearchParams();
|
|
22
|
-
for (const [key, value] of Object.entries(
|
|
22
|
+
for (const [key, value] of Object.entries(searchParams)) {
|
|
23
23
|
if (value === undefined) {
|
|
24
24
|
newSearchParams.delete(key);
|
|
25
25
|
}
|
|
@@ -33,4 +33,3 @@ export function applySearchParams(url, searchParams) {
|
|
|
33
33
|
}
|
|
34
34
|
url.search = newSearchParams.toString();
|
|
35
35
|
}
|
|
36
|
-
//# sourceMappingURL=url.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.90",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -35,19 +35,19 @@
|
|
|
35
35
|
},
|
|
36
36
|
"homepage": "https://crawlee.dev",
|
|
37
37
|
"scripts": {
|
|
38
|
-
"build": "
|
|
38
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
39
39
|
"clean": "rimraf ./dist",
|
|
40
40
|
"compile": "tsc -p tsconfig.build.json",
|
|
41
41
|
"copy": "tsx ../../scripts/copy.ts"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@apify/log": "^2.5.18",
|
|
45
44
|
"@apify/ps-tree": "^1.2.0",
|
|
46
|
-
"@crawlee/
|
|
45
|
+
"@crawlee/http-client": "4.0.0-beta.90",
|
|
46
|
+
"@crawlee/types": "4.0.0-beta.90",
|
|
47
47
|
"@types/sax": "^1.2.7",
|
|
48
48
|
"cheerio": "^1.0.0",
|
|
49
|
+
"domhandler": "^5.0.3",
|
|
49
50
|
"file-type": "^21.0.0",
|
|
50
|
-
"got-scraping": "^4.1.1",
|
|
51
51
|
"ow": "^2.0.0",
|
|
52
52
|
"robots-parser": "^3.0.1",
|
|
53
53
|
"sax": "^1.4.1",
|
|
@@ -61,5 +61,5 @@
|
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
},
|
|
64
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "7afdd44c5b6850b4775cfc08ffbbdd1791ae6402"
|
|
65
65
|
}
|
package/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAa,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAc,MAAM,wCAAwC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"blocked.d.ts","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,8BAA8B,UAA0E,CAAC;AAEtH;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAI/B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAM/B,CAAC"}
|
package/internals/blocked.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"blocked.js","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,qEAAqE,CAAC,CAAC;AAEtH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,GAAG,8BAA8B;IACjC,0DAA0D;IAC1D,oCAAoC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,YAAY;IACZ,cAAc;IACd,6BAA6B;IAC7B,8BAA8B;IAC9B,sBAAsB;CACzB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AAOrC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}
|
package/internals/cheerio.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
|
package/internals/chunk.d.ts
DELETED
package/internals/chunk.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AA2BA,wBAAgB,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAQtE"}
|
package/internals/chunk.js
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
The MIT License (MIT)
|
|
3
|
-
|
|
4
|
-
Copyright © `2020` `The Sapphire Community and its contributors`
|
|
5
|
-
|
|
6
|
-
Permission is hereby granted, free of charge, to any person
|
|
7
|
-
obtaining a copy of this software and associated documentation
|
|
8
|
-
files (the “Software”), to deal in the Software without
|
|
9
|
-
restriction, including without limitation the rights to use,
|
|
10
|
-
copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
-
copies of the Software, and to permit persons to whom the
|
|
12
|
-
Software is furnished to do so, subject to the following
|
|
13
|
-
conditions:
|
|
14
|
-
|
|
15
|
-
The above copyright notice and this permission notice shall be
|
|
16
|
-
included in all copies or substantial portions of the Software.
|
|
17
|
-
|
|
18
|
-
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
|
|
19
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|
20
|
-
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
21
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
22
|
-
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
23
|
-
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
24
|
-
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
25
|
-
OTHER DEALINGS IN THE SOFTWARE.
|
|
26
|
-
*/
|
|
27
|
-
export function chunk(array, chunkSize) {
|
|
28
|
-
if (!Array.isArray(array))
|
|
29
|
-
throw new TypeError('entries must be an array.');
|
|
30
|
-
if (!Number.isInteger(chunkSize))
|
|
31
|
-
throw new TypeError('chunkSize must be an integer.');
|
|
32
|
-
if (chunkSize < 1)
|
|
33
|
-
throw new RangeError('chunkSize must be 1 or greater.');
|
|
34
|
-
const clone = array.slice();
|
|
35
|
-
const chunks = [];
|
|
36
|
-
while (clone.length)
|
|
37
|
-
chunks.push(clone.splice(0, chunkSize));
|
|
38
|
-
return chunks;
|
|
39
|
-
}
|
|
40
|
-
//# sourceMappingURL=chunk.js.map
|
package/internals/chunk.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;AAEF,MAAM,UAAU,KAAK,CAAI,KAAmB,EAAE,SAAiB;IAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;IACvF,IAAI,SAAS,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;IACjC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/internals/debug.d.ts
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { IncomingMessage } from 'node:http';
|
|
2
|
-
import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
|
|
3
|
-
interface BrowserResponseLike {
|
|
4
|
-
status(): number;
|
|
5
|
-
}
|
|
6
|
-
interface Request<UserData extends Dictionary = Dictionary> {
|
|
7
|
-
id?: string;
|
|
8
|
-
url: string;
|
|
9
|
-
loadedUrl?: string;
|
|
10
|
-
uniqueKey: string;
|
|
11
|
-
method: AllowedHttpMethods;
|
|
12
|
-
payload?: string;
|
|
13
|
-
noRetry: boolean;
|
|
14
|
-
retryCount: number;
|
|
15
|
-
errorMessages: string[];
|
|
16
|
-
headers?: Record<string, string>;
|
|
17
|
-
userData: UserData;
|
|
18
|
-
handledAt?: string;
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
22
|
-
*
|
|
23
|
-
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
24
|
-
* @param [response]
|
|
25
|
-
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
26
|
-
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
27
|
-
* @param [additionalFields] Object containing additional fields to be added.
|
|
28
|
-
*/
|
|
29
|
-
export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
|
|
30
|
-
export {};
|
|
31
|
-
//# sourceMappingURL=debug.d.ts.map
|
package/internals/debug.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ"}
|
package/internals/debug.js
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import ow from 'ow';
|
|
2
|
-
/**
|
|
3
|
-
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
4
|
-
*
|
|
5
|
-
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
6
|
-
* @param [response]
|
|
7
|
-
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
8
|
-
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
9
|
-
* @param [additionalFields] Object containing additional fields to be added.
|
|
10
|
-
*/
|
|
11
|
-
export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
|
|
12
|
-
ow(request, ow.object);
|
|
13
|
-
ow(response, ow.object);
|
|
14
|
-
ow(additionalFields, ow.object);
|
|
15
|
-
return {
|
|
16
|
-
requestId: request.id,
|
|
17
|
-
url: request.url,
|
|
18
|
-
loadedUrl: request.loadedUrl,
|
|
19
|
-
method: request.method,
|
|
20
|
-
retryCount: request.retryCount,
|
|
21
|
-
errorMessages: request.errorMessages,
|
|
22
|
-
// Puppeteer response has .status() function and NodeJS response, statusCode property.
|
|
23
|
-
statusCode: 'status' in response && response.status instanceof Function
|
|
24
|
-
? response.status()
|
|
25
|
-
: response.statusCode,
|
|
26
|
-
...additionalFields,
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
//# sourceMappingURL=debug.js.map
|
package/internals/debug.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,IAAI,CAAC;AAqBpB;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACvB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyBnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CACL,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,mBAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAC+G,CAAC;AAEhJ;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAC8G,CAAC;AAoBjJ;;GAEG;AACH,wBAAsB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrE;AAID;;GAEG;AACH,wBAAsB,eAAe,qBA0BpC;AAED,wBAAgB,QAAQ,YAEvB;AAGD;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,UAAU,CAAC,EAAE,OAAO,+BAoB3D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAW7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAQjE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8B5D"}
|
package/internals/general.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,EAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,EAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED,IAAI,qBAA0C,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,gFAAgF;IAChF,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,wCAAwC;IACxC,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,qBAAqB,GAAG,KAAK,CAAC;QAC9B,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,MAAM,cAAc,GAAG,EAAE;SACpB,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,WAAW,GAAG,EAAE;SACjB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,eAAe,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAEzF,qBAAqB,GAAG,eAAe,IAAI,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACjG,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,QAAQ;IACpB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;AACzD,CAAC;AAED,IAAI,eAAmC,CAAC;AACxC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,UAAoB;IACxD,wDAAwD;IACxD,IAAI,eAAe,KAAK,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,eAAe,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC;QACD,+DAA+D;QAC/D,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC;QACD,8EAA8E;QAC9E,MAAM,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErE,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,EAAE,CAAC,SAAS,IAAI,OAAO,CAAC;YAC5B,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"open_graph_parser.d.ts","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CACjC;AAED,KAAK,eAAe,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;AA2X3E;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;AACrH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAU/B;;;;;;GAMG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAC9C,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE/E,MAAM,qBAAqB,GAAwB;IAC/C;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,YAAY;QACxB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD,kDAAkD;IAClD,iHAAiH;IACjH,mDAAmD;IACnD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,mBAAmB;wBACzB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,iBAAiB;wBACvB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,UAAU,EAAE,eAAe;gBAC3B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,cAAc;gBAC1B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,uBAAuB;gBAC7B,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,UAAU;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;CACJ,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,CAAC;AAE3E,MAAM,sBAAsB,GAAG,CAAC,QAA2B,EAAE,CAAa,EAAuC,EAAE;IAC/G,6GAA6G;IAC7G,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnG,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,CAAY,CAAC,CAAC,CAAE,MAAmB,CAAC;IAElF,sEAAsE;IACtE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAE9C,wGAAwG;IACxG,OAAO;QACH,qGAAqG;QACrG,+EAA+E;QAC/E,wGAAwG;QACxG,uDAAuD;QACvD,GAAG,cAAc,CAAC,GAAG,QAAQ,CAAC,UAAU,OAAO,EAAE,OAAO,CAAC;QACzD,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAErC,OAAO;gBACH,GAAG,GAAG;gBACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;aACtE,CAAC;QACN,CAAC,EACD,EAAqC,CACxC;KACJ,CAAC;AACN,CAAC,CAAC;AAYF,MAAM,UAAU,cAAc,CAAC,IAAyB,EAAE,oBAA0C;IAChG,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,MAAM,CACrE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACV,OAAO;YACH,GAAG,GAAG;YACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;SACtE,CAAC;IACN,CAAC,EACD,EAAiC,CACpC,CAAC;AACN,CAAC"}
|