@crawlee/utils 4.0.0-beta.8 → 4.0.0-beta.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +17 -13
  2. package/index.d.ts +4 -5
  3. package/index.js +3 -4
  4. package/internals/blocked.d.ts +0 -1
  5. package/internals/blocked.js +0 -1
  6. package/internals/cheerio.d.ts +3 -2
  7. package/internals/cheerio.js +4 -5
  8. package/internals/chunk.d.ts +0 -1
  9. package/internals/chunk.js +0 -1
  10. package/internals/debug.d.ts +11 -1
  11. package/internals/debug.js +34 -1
  12. package/internals/extract-urls.d.ts +5 -1
  13. package/internals/extract-urls.js +8 -5
  14. package/internals/general.d.ts +17 -15
  15. package/internals/general.js +44 -83
  16. package/internals/iterables.d.ts +126 -0
  17. package/internals/iterables.js +230 -0
  18. package/internals/open_graph_parser.d.ts +2 -3
  19. package/internals/open_graph_parser.js +8 -9
  20. package/internals/robots.d.ts +20 -4
  21. package/internals/robots.js +31 -33
  22. package/internals/sitemap.d.ts +64 -7
  23. package/internals/sitemap.js +174 -34
  24. package/internals/social.d.ts +1 -2
  25. package/internals/social.js +7 -5
  26. package/internals/typedefs.d.ts +0 -1
  27. package/internals/typedefs.js +0 -1
  28. package/internals/url.d.ts +1 -2
  29. package/internals/url.js +1 -2
  30. package/package.json +6 -6
  31. package/index.d.ts.map +0 -1
  32. package/index.js.map +0 -1
  33. package/internals/blocked.d.ts.map +0 -1
  34. package/internals/blocked.js.map +0 -1
  35. package/internals/cheerio.d.ts.map +0 -1
  36. package/internals/cheerio.js.map +0 -1
  37. package/internals/chunk.d.ts.map +0 -1
  38. package/internals/chunk.js.map +0 -1
  39. package/internals/debug.d.ts.map +0 -1
  40. package/internals/debug.js.map +0 -1
  41. package/internals/extract-urls.d.ts.map +0 -1
  42. package/internals/extract-urls.js.map +0 -1
  43. package/internals/general.d.ts.map +0 -1
  44. package/internals/general.js.map +0 -1
  45. package/internals/memory-info.d.ts +0 -26
  46. package/internals/memory-info.d.ts.map +0 -1
  47. package/internals/memory-info.js +0 -131
  48. package/internals/memory-info.js.map +0 -1
  49. package/internals/open_graph_parser.d.ts.map +0 -1
  50. package/internals/open_graph_parser.js.map +0 -1
  51. package/internals/robots.d.ts.map +0 -1
  52. package/internals/robots.js.map +0 -1
  53. package/internals/sitemap.d.ts.map +0 -1
  54. package/internals/sitemap.js.map +0 -1
  55. package/internals/social.d.ts.map +0 -1
  56. package/internals/social.js.map +0 -1
  57. package/internals/systemInfoV2/cpu-info.d.ts +0 -64
  58. package/internals/systemInfoV2/cpu-info.d.ts.map +0 -1
  59. package/internals/systemInfoV2/cpu-info.js +0 -211
  60. package/internals/systemInfoV2/cpu-info.js.map +0 -1
  61. package/internals/systemInfoV2/memory-info.d.ts +0 -28
  62. package/internals/systemInfoV2/memory-info.d.ts.map +0 -1
  63. package/internals/systemInfoV2/memory-info.js +0 -118
  64. package/internals/systemInfoV2/memory-info.js.map +0 -1
  65. package/internals/systemInfoV2/ps-tree.d.ts +0 -18
  66. package/internals/systemInfoV2/ps-tree.d.ts.map +0 -1
  67. package/internals/systemInfoV2/ps-tree.js +0 -145
  68. package/internals/systemInfoV2/ps-tree.js.map +0 -1
  69. package/internals/typedefs.d.ts.map +0 -1
  70. package/internals/typedefs.js.map +0 -1
  71. package/internals/url.d.ts.map +0 -1
  72. package/internals/url.js.map +0 -1
  73. package/tsconfig.build.tsbuildinfo +0 -1
@@ -2,9 +2,12 @@ import { createHash } from 'node:crypto';
2
2
  import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
3
  import { StringDecoder } from 'node:string_decoder';
4
4
  import { createGunzip } from 'node:zlib';
5
+ import { FetchHttpClient } from '@crawlee/http-client';
6
+ import { fileTypeStream } from 'file-type';
5
7
  import sax from 'sax';
6
8
  import MIMEType from 'whatwg-mimetype';
7
- import log from '@apify/log';
9
+ import { mergeAsyncIterables } from './iterables.js';
10
+ import { RobotsFile } from './robots.js';
8
11
  class SitemapTxtParser extends Transform {
9
12
  decoder = new StringDecoder('utf8');
10
13
  buffer = '';
@@ -91,8 +94,10 @@ class SitemapXmlParser extends Transform {
91
94
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
92
95
  this.currentTag = undefined;
93
96
  }
94
- if (name === 'url' && this.url.loc !== undefined) {
95
- this.push({ type: 'url', ...this.url, loc: this.url.loc });
97
+ if (name === 'url') {
98
+ if (this.url.loc !== undefined) {
99
+ this.push({ type: 'url', ...this.url, loc: this.url.loc });
100
+ }
96
101
  this.url = {};
97
102
  }
98
103
  }
@@ -108,7 +113,10 @@ class SitemapXmlParser extends Transform {
108
113
  }
109
114
  text = text.trim();
110
115
  if (this.currentTag === 'lastmod') {
111
- this.url.lastmod = new Date(text);
116
+ const lastmod = new Date(text);
117
+ if (!Number.isNaN(lastmod.getTime())) {
118
+ this.url.lastmod = lastmod;
119
+ }
112
120
  }
113
121
  if (this.currentTag === 'priority') {
114
122
  this.url.priority = Number(text);
@@ -121,9 +129,7 @@ class SitemapXmlParser extends Transform {
121
129
  }
122
130
  }
123
131
  export async function* parseSitemap(initialSources, proxyUrl, options) {
124
- const { gotScraping } = await import('got-scraping');
125
- const { fileTypeStream } = await import('file-type');
126
- const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts } = options ?? {};
132
+ const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
127
133
  const sources = [...initialSources];
128
134
  const visitedSitemapUrls = new Set();
129
135
  const createParser = (contentType = '', url) => {
@@ -145,7 +151,6 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
145
151
  while (sources.length > 0) {
146
152
  const source = sources.shift();
147
153
  if ((source?.depth ?? 0) > maxDepth) {
148
- log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
149
154
  continue;
150
155
  }
151
156
  let items = null;
@@ -155,23 +160,28 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
155
160
  let retriesLeft = sitemapRetries + 1;
156
161
  while (retriesLeft-- > 0) {
157
162
  try {
158
- const sitemapStream = await new Promise((resolve, reject) => {
159
- const request = gotScraping.stream({
160
- url: sitemapUrl,
161
- proxyUrl,
163
+ let sitemapResponse;
164
+ try {
165
+ sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
162
166
  method: 'GET',
163
- timeout: networkTimeouts,
164
167
  headers: {
165
- accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
168
+ accept: '*/*',
166
169
  },
170
+ }), {
171
+ proxyUrl,
172
+ timeoutMillis: timeout,
167
173
  });
168
- request.on('response', () => resolve(request));
169
- request.on('error', reject);
170
- });
174
+ }
175
+ catch (error) {
176
+ sitemapResponse = null;
177
+ }
171
178
  let error = null;
172
- if (sitemapStream.response.statusCode >= 200 && sitemapStream.response.statusCode < 300) {
173
- let contentType = sitemapStream.response.headers['content-type'];
174
- const streamWithType = await fileTypeStream(sitemapStream);
179
+ if (sitemapResponse && sitemapResponse.status >= 200 && sitemapResponse.status < 300) {
180
+ let contentType = sitemapResponse.headers.get('content-type');
181
+ if (sitemapResponse.body === null) {
182
+ break;
183
+ }
184
+ const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
175
185
  if (streamWithType.fileType !== undefined) {
176
186
  contentType = streamWithType.fileType.mime;
177
187
  }
@@ -184,29 +194,37 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
184
194
  sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
185
195
  }
186
196
  }
187
- items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType, sitemapUrl), (e) => {
188
- if (e !== undefined) {
189
- error = e;
197
+ items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
198
+ if (e !== undefined && e !== null) {
199
+ error = { type: 'parser', error: e };
190
200
  }
191
201
  });
192
202
  }
193
203
  else {
194
- error = new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapStream.response.statusCode}`);
204
+ error = {
205
+ type: 'fetch',
206
+ error: new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapResponse?.status}`),
207
+ };
195
208
  }
196
209
  if (error !== null) {
197
- throw error;
210
+ const shouldIgnoreError = error.type === 'fetch' && !reportNetworkErrors;
211
+ if (!shouldIgnoreError) {
212
+ throw error.error;
213
+ }
214
+ }
215
+ else {
216
+ break;
198
217
  }
199
- break;
200
218
  }
201
219
  catch (e) {
202
- log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
220
+ logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
203
221
  }
204
222
  }
205
223
  }
206
224
  else if (source.type === 'raw') {
207
225
  items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
208
226
  if (error !== undefined) {
209
- log.warning(`Malformed sitemap content: ${error}`);
227
+ logger?.warning(`Malformed sitemap content: ${error}`);
210
228
  }
211
229
  });
212
230
  }
@@ -215,6 +233,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
215
233
  }
216
234
  for await (const item of items) {
217
235
  if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
236
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
237
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
238
+ continue;
239
+ }
218
240
  sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
219
241
  if (emitNestedSitemaps) {
220
242
  yield { loc: item.url, originSitemapUrl: null };
@@ -254,7 +276,7 @@ export class Sitemap {
254
276
  * @param url The domain URL to fetch the sitemap for.
255
277
  * @param proxyUrl A proxy to be used for fetching the sitemap file.
256
278
  */
257
- static async tryCommonNames(url, proxyUrl) {
279
+ static async tryCommonNames(url, proxyUrl, parseSitemapOptions) {
258
280
  const sitemapUrls = [];
259
281
  const sitemapUrl = new URL(url);
260
282
  sitemapUrl.search = '';
@@ -262,7 +284,7 @@ export class Sitemap {
262
284
  sitemapUrls.push(sitemapUrl.toString());
263
285
  sitemapUrl.pathname = '/sitemap.txt';
264
286
  sitemapUrls.push(sitemapUrl.toString());
265
- return Sitemap.load(sitemapUrls, proxyUrl);
287
+ return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false, ...parseSitemapOptions });
266
288
  }
267
289
  /**
268
290
  * Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
@@ -277,8 +299,8 @@ export class Sitemap {
277
299
  * @param content XML sitemap content
278
300
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
279
301
  */
280
- static async fromXmlString(content, proxyUrl) {
281
- return await this.parse([{ type: 'raw', content }], proxyUrl);
302
+ static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
303
+ return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
282
304
  }
283
305
  static async parse(sources, proxyUrl, parseSitemapOptions) {
284
306
  const urls = [];
@@ -287,10 +309,128 @@ export class Sitemap {
287
309
  urls.push(item.loc);
288
310
  }
289
311
  }
290
- catch {
312
+ catch (e) {
313
+ parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
291
314
  return new Sitemap([]);
292
315
  }
293
316
  return new Sitemap(urls);
294
317
  }
295
318
  }
296
- //# sourceMappingURL=sitemap.js.map
319
+ /**
320
+ * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
321
+ * the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
322
+ * @param `urls` The list of URLs to discover sitemaps for.
323
+ * @param `options` Options for sitemap discovery
324
+ * @returns An async iterable with the discovered sitemap URLs.
325
+ */
326
+ export async function* discoverValidSitemaps(urls, options = {}) {
327
+ const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
328
+ const controller = new AbortController();
329
+ const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
330
+ const onExternalAbort = () => controller.abort();
331
+ if (externalSignal) {
332
+ if (externalSignal.aborted) {
333
+ controller.abort();
334
+ }
335
+ else {
336
+ externalSignal.addEventListener('abort', onExternalAbort, { once: true });
337
+ }
338
+ }
339
+ const signal = controller.signal;
340
+ const sitemapUrls = new Set();
341
+ const addSitemapUrl = (url) => {
342
+ const sizeBefore = sitemapUrls.size;
343
+ sitemapUrls.add(url);
344
+ if (sitemapUrls.size > sizeBefore) {
345
+ return url;
346
+ }
347
+ return undefined;
348
+ };
349
+ const urlExists = async (url) => {
350
+ if (!httpClient) {
351
+ return false;
352
+ }
353
+ try {
354
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
355
+ proxyUrl,
356
+ timeoutMillis: requestTimeoutMillis,
357
+ signal,
358
+ });
359
+ return response.status >= 200 && response.status < 400;
360
+ }
361
+ catch {
362
+ return false;
363
+ }
364
+ };
365
+ const discoverSitemapsForDomainUrls = async function* (hostname, domainUrls) {
366
+ if (!hostname) {
367
+ return;
368
+ }
369
+ try {
370
+ const robotsFile = await RobotsFile.find(domainUrls[0], {
371
+ proxyUrl,
372
+ timeoutMillis: requestTimeoutMillis,
373
+ signal,
374
+ httpClient,
375
+ logger,
376
+ });
377
+ for (const sitemapUrl of robotsFile.getSitemaps()) {
378
+ if (addSitemapUrl(sitemapUrl)) {
379
+ yield sitemapUrl;
380
+ }
381
+ }
382
+ }
383
+ catch (err) {
384
+ logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
385
+ }
386
+ const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
387
+ if (sitemapUrl !== undefined) {
388
+ if (addSitemapUrl(sitemapUrl)) {
389
+ yield sitemapUrl;
390
+ }
391
+ }
392
+ else {
393
+ const firstUrl = new URL(domainUrls[0]);
394
+ const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
395
+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
396
+ firstUrl.pathname = pathname;
397
+ return firstUrl.toString();
398
+ });
399
+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
400
+ for (const [index, result] of candidateResults.entries()) {
401
+ const candidateSitemapUrl = candidateSitemapUrls[index];
402
+ if (result.status === 'fulfilled') {
403
+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {
404
+ yield candidateSitemapUrl;
405
+ }
406
+ }
407
+ else {
408
+ logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
409
+ error: result.reason,
410
+ });
411
+ }
412
+ }
413
+ }
414
+ };
415
+ const groupedUrls = urls.reduce((acc, url) => {
416
+ const hostname = new URL(url)?.hostname ?? '';
417
+ acc[hostname] ??= [];
418
+ acc[hostname].push(url);
419
+ return acc;
420
+ }, {});
421
+ const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
422
+ const discoveredUrls = new Set();
423
+ try {
424
+ for await (const url of mergeAsyncIterables(...iterables)) {
425
+ if (discoveredUrls.has(url)) {
426
+ continue;
427
+ }
428
+ discoveredUrls.add(url);
429
+ yield url;
430
+ }
431
+ }
432
+ finally {
433
+ clearTimeout(timeoutHandle);
434
+ externalSignal?.removeEventListener('abort', onExternalAbort);
435
+ }
436
+ }
@@ -488,5 +488,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
488
488
  * so that the caller doesn't need to parse the HTML document again, if needed.
489
489
  * @return An object with the social handles.
490
490
  */
491
- export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
492
- //# sourceMappingURL=social.d.ts.map
491
+ export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
@@ -1,7 +1,9 @@
1
- import * as cheerio from 'cheerio';
2
1
  import { htmlToText } from './cheerio.js';
3
2
  // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
4
- const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
3
+ // The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
4
+ // instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
5
+ // dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
6
+ const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
5
7
  /**
6
8
  * Regular expression to exactly match a single email address.
7
9
  * It has the following form: `/^...$/i`.
@@ -587,7 +589,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
587
589
  * so that the caller doesn't need to parse the HTML document again, if needed.
588
590
  * @return An object with the social handles.
589
591
  */
590
- export function parseHandlesFromHtml(html, data = null) {
592
+ export async function parseHandlesFromHtml(html, data = null) {
593
+ const cheerio = await import('cheerio');
591
594
  const result = {
592
595
  emails: [],
593
596
  phones: [],
@@ -606,7 +609,7 @@ export function parseHandlesFromHtml(html, data = null) {
606
609
  const $ = cheerio.load(html, { xml: { decodeEntities: true } });
607
610
  if (data)
608
611
  data.$ = $;
609
- const text = htmlToText($);
612
+ const text = await htmlToText($);
610
613
  if (data)
611
614
  data.text = text;
612
615
  // NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
@@ -644,4 +647,3 @@ export function parseHandlesFromHtml(html, data = null) {
644
647
  }
645
648
  return result;
646
649
  }
647
- //# sourceMappingURL=social.js.map
@@ -2,4 +2,3 @@
2
2
  export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
3
3
  /** @ignore */
4
4
  export declare function keys<T extends {}>(obj: T): (keyof T)[];
5
- //# sourceMappingURL=typedefs.d.ts.map
@@ -6,4 +6,3 @@ export function entries(obj) {
6
6
  export function keys(obj) {
7
7
  return Object.keys(obj);
8
8
  }
9
- //# sourceMappingURL=typedefs.js.map
@@ -1,4 +1,4 @@
1
- export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
1
+ import type { SearchParams } from '@crawlee/types';
2
2
  /**
3
3
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
4
4
  *
@@ -7,4 +7,3 @@ export type SearchParams = string | URLSearchParams | Record<string, string | nu
7
7
  * @internal
8
8
  */
9
9
  export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
10
- //# sourceMappingURL=url.d.ts.map
package/internals/url.js CHANGED
@@ -19,7 +19,7 @@ export function applySearchParams(url, searchParams) {
19
19
  }
20
20
  else {
21
21
  newSearchParams = new URLSearchParams();
22
- for (const [key, value] of Object.entries(newSearchParams)) {
22
+ for (const [key, value] of Object.entries(searchParams)) {
23
23
  if (value === undefined) {
24
24
  newSearchParams.delete(key);
25
25
  }
@@ -33,4 +33,3 @@ export function applySearchParams(url, searchParams) {
33
33
  }
34
34
  url.search = newSearchParams.toString();
35
35
  }
36
- //# sourceMappingURL=url.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.8",
3
+ "version": "4.0.0-beta.80",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -35,19 +35,19 @@
35
35
  },
36
36
  "homepage": "https://crawlee.dev",
37
37
  "scripts": {
38
- "build": "yarn clean && yarn compile && yarn copy",
38
+ "build": "pnpm clean && pnpm compile && pnpm copy",
39
39
  "clean": "rimraf ./dist",
40
40
  "compile": "tsc -p tsconfig.build.json",
41
41
  "copy": "tsx ../../scripts/copy.ts"
42
42
  },
43
43
  "dependencies": {
44
- "@apify/log": "^2.5.18",
45
44
  "@apify/ps-tree": "^1.2.0",
46
- "@crawlee/types": "4.0.0-beta.8",
45
+ "@crawlee/http-client": "4.0.0-beta.80",
46
+ "@crawlee/types": "4.0.0-beta.80",
47
47
  "@types/sax": "^1.2.7",
48
48
  "cheerio": "^1.0.0",
49
+ "domhandler": "^5.0.3",
49
50
  "file-type": "^21.0.0",
50
- "got-scraping": "^4.1.1",
51
51
  "ow": "^2.0.0",
52
52
  "robots-parser": "^3.0.1",
53
53
  "sax": "^1.4.1",
@@ -61,5 +61,5 @@
61
61
  }
62
62
  }
63
63
  },
64
- "gitHead": "a0f673e03208ad1dd8956ce3b49a2d1c9e3b2ed5"
64
+ "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
65
65
  }
package/index.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAE1E,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
package/index.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"blocked.d.ts","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,8BAA8B,UAA0E,CAAC;AAEtH;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAI/B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAM/B,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"blocked.js","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,qEAAqE,CAAC,CAAC;AAEtH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,GAAG,8BAA8B;IACjC,0DAA0D;IAC1D,oCAAoC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,YAAY;IACZ,cAAc;IACd,6BAA6B;IAC7B,8BAA8B;IAC9B,sBAAsB;CACzB,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AAOrC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AA2BA,wBAAgB,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAQtE"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;AAEF,MAAM,UAAU,KAAK,CAAI,KAAmB,EAAE,SAAiB;IAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;IACvF,IAAI,SAAS,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;IACjC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,IAAI,CAAC;AAqBpB;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACvB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyBnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CACL,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,mBAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAC+G,CAAC;AAEhJ;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAC8G,CAAC;AAoBjJ;;GAEG;AACH,wBAAsB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrE;AAID;;GAEG;AACH,wBAAsB,eAAe,qBA0BpC;AAED,wBAAgB,QAAQ,YAEvB;AAGD;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,UAAU,CAAC,EAAE,OAAO,+BAoB3D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAW7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAQjE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8B5D"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,EAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,EAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED,IAAI,qBAA0C,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,gFAAgF;IAChF,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,wCAAwC;IACxC,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,qBAAqB,GAAG,KAAK,CAAC;QAC9B,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,MAAM,cAAc,GAAG,EAAE;SACpB,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,WAAW,GAAG,EAAE;SACjB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,eAAe,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAEzF,qBAAqB,GAAG,eAAe,IAAI,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACjG,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,QAAQ;IACpB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;AACzD,CAAC;AAED,IAAI,eAAmC,CAAC;AACxC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,UAAoB;IACxD,wDAAwD;IACxD,IAAI,eAAe,KAAK,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,eAAe,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC;QACD,+DAA+D;QAC/D,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC;QACD,8EAA8E;QAC9E,MAAM,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErE,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,EAAE,CAAC,SAAS,IAAI,OAAO,CAAC;YAC5B,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
@@ -1,26 +0,0 @@
1
- /**
2
- * Describes memory usage of the process.
3
- */
4
- export interface MemoryInfo {
5
- /** Total memory available in the system or container */
6
- totalBytes: number;
7
- /** Amount of free memory in the system or container */
8
- freeBytes: number;
9
- /** Amount of memory used (= totalBytes - freeBytes) */
10
- usedBytes: number;
11
- /** Amount of memory used the current Node.js process */
12
- mainProcessBytes: number;
13
- /** Amount of memory used by child processes of the current Node.js process */
14
- childProcessesBytes: number;
15
- }
16
- /**
17
- * Returns memory statistics of the process and the system, see {@link MemoryInfo}.
18
- *
19
- * If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits,
20
- * otherwise it gets system memory limits.
21
- *
22
- * Beware that the function is quite inefficient because it spawns a new process.
23
- * Therefore you shouldn't call it too often, like more than once per second.
24
- */
25
- export declare function getMemoryInfo(): Promise<MemoryInfo>;
26
- //# sourceMappingURL=memory-info.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"memory-info.d.ts","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":"AAwBA;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IAEnB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IAEzB,8EAA8E;IAC9E,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CA+GzD"}