@crawlee/utils 4.0.0-beta.29 → 4.0.0-beta.290

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +10 -17
  3. package/index.js +8 -16
  4. package/internal.d.ts +12 -0
  5. package/internal.js +11 -0
  6. package/internals/blocked.d.ts +7 -1
  7. package/internals/blocked.js +11 -1
  8. package/internals/cheerio.d.ts +1 -3
  9. package/internals/cheerio.js +33 -23
  10. package/internals/extract-microdata.d.ts +24 -0
  11. package/internals/extract-microdata.js +136 -0
  12. package/internals/extract-urls.d.ts +1 -2
  13. package/internals/extract-urls.js +18 -23
  14. package/internals/general.d.ts +0 -25
  15. package/internals/general.js +2 -110
  16. package/internals/http-request.d.ts +5 -0
  17. package/internals/http-request.js +36 -0
  18. package/internals/iterables.d.ts +0 -80
  19. package/internals/iterables.js +0 -126
  20. package/internals/mutation-journal.d.ts +29 -0
  21. package/internals/mutation-journal.js +173 -0
  22. package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
  23. package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
  24. package/internals/robots.d.ts +35 -15
  25. package/internals/robots.js +66 -30
  26. package/internals/schemas.d.ts +114 -0
  27. package/internals/schemas.js +115 -0
  28. package/internals/sitemap.d.ts +44 -3
  29. package/internals/sitemap.js +243 -116
  30. package/internals/social.d.ts +4 -3
  31. package/internals/social.js +36 -11
  32. package/internals/url.d.ts +67 -1
  33. package/internals/url.js +118 -2
  34. package/internals/validation.d.ts +9 -0
  35. package/internals/validation.js +16 -0
  36. package/package.json +16 -15
  37. package/index.d.ts.map +0 -1
  38. package/index.js.map +0 -1
  39. package/internals/blocked.d.ts.map +0 -1
  40. package/internals/blocked.js.map +0 -1
  41. package/internals/cheerio.d.ts.map +0 -1
  42. package/internals/cheerio.js.map +0 -1
  43. package/internals/chunk.d.ts +0 -2
  44. package/internals/chunk.d.ts.map +0 -1
  45. package/internals/chunk.js +0 -40
  46. package/internals/chunk.js.map +0 -1
  47. package/internals/debug.d.ts +0 -32
  48. package/internals/debug.d.ts.map +0 -1
  49. package/internals/debug.js +0 -41
  50. package/internals/debug.js.map +0 -1
  51. package/internals/extract-urls.d.ts.map +0 -1
  52. package/internals/extract-urls.js.map +0 -1
  53. package/internals/general.d.ts.map +0 -1
  54. package/internals/general.js.map +0 -1
  55. package/internals/iterables.d.ts.map +0 -1
  56. package/internals/iterables.js.map +0 -1
  57. package/internals/open_graph_parser.d.ts.map +0 -1
  58. package/internals/open_graph_parser.js.map +0 -1
  59. package/internals/robots.d.ts.map +0 -1
  60. package/internals/robots.js.map +0 -1
  61. package/internals/sitemap.d.ts.map +0 -1
  62. package/internals/sitemap.js.map +0 -1
  63. package/internals/social.d.ts.map +0 -1
  64. package/internals/social.js.map +0 -1
  65. package/internals/system-info/cpu-info.d.ts +0 -64
  66. package/internals/system-info/cpu-info.d.ts.map +0 -1
  67. package/internals/system-info/cpu-info.js +0 -211
  68. package/internals/system-info/cpu-info.js.map +0 -1
  69. package/internals/system-info/memory-info.d.ts +0 -28
  70. package/internals/system-info/memory-info.d.ts.map +0 -1
  71. package/internals/system-info/memory-info.js +0 -118
  72. package/internals/system-info/memory-info.js.map +0 -1
  73. package/internals/system-info/ps-tree.d.ts +0 -18
  74. package/internals/system-info/ps-tree.d.ts.map +0 -1
  75. package/internals/system-info/ps-tree.js +0 -145
  76. package/internals/system-info/ps-tree.js.map +0 -1
  77. package/internals/typedefs.d.ts +0 -5
  78. package/internals/typedefs.d.ts.map +0 -1
  79. package/internals/typedefs.js +0 -9
  80. package/internals/typedefs.js.map +0 -1
  81. package/internals/url.d.ts.map +0 -1
  82. package/internals/url.js.map +0 -1
@@ -1,5 +1,15 @@
1
- import type { BaseHttpClient } from '@crawlee/types';
1
+ import type { BaseHttpClient } from '@crawlee/http-client';
2
+ import type { CrawleeLogger, ISession } from '@crawlee/types';
2
3
  import { Sitemap } from './sitemap.js';
4
+ import { type EnqueueStrategy } from './url.js';
5
+ export interface RobotsTxtFileSitemapsOptions {
6
+ /**
7
+ * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
8
+ * are always dropped. Pass `'all'` to disable host filtering.
9
+ * @default 'same-hostname'
10
+ */
11
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
12
+ }
3
13
  /**
4
14
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
5
15
  *
@@ -19,17 +29,24 @@ import { Sitemap } from './sitemap.js';
19
29
  * ```
20
30
  */
21
31
  export declare class RobotsTxtFile {
22
- private robots;
23
- private proxyUrl?;
32
+ #private;
24
33
  private constructor();
25
34
  /**
26
35
  * Determine the location of a robots.txt file for a URL and fetch it.
27
36
  * @param url the URL to fetch robots.txt for
28
- * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
37
+ * @param [options] additional options
38
+ * @param [options.signal] an AbortSignal to cancel the request
39
+ * @param [options.timeoutMillis] timeout in milliseconds for the request
40
+ * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
41
+ * @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
29
42
  */
30
43
  static find(url: string, options?: {
44
+ signal?: AbortSignal;
45
+ timeoutMillis?: number;
31
46
  proxyUrl?: string;
32
47
  httpClient?: BaseHttpClient;
48
+ session?: ISession;
49
+ logger?: CrawleeLogger;
33
50
  }): Promise<RobotsTxtFile>;
34
51
  /**
35
52
  * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -38,10 +55,11 @@ export declare class RobotsTxtFile {
38
55
  * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
39
56
  */
40
57
  static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
41
- protected static load(url: string, options?: {
42
- proxyUrl?: string;
43
- httpClient?: BaseHttpClient;
44
- }): Promise<RobotsTxtFile>;
58
+ /**
59
+ * Get crawl delay for a given user agent.
60
+ * @param [userAgent] relevant user agent, default to `*`
61
+ */
62
+ getCrawlDelay(userAgent?: string): number | undefined;
45
63
  /**
46
64
  * Check if a URL should be crawled by robots.
47
65
  * @param url the URL to check against the rules in robots.txt
@@ -49,17 +67,19 @@ export declare class RobotsTxtFile {
49
67
  */
50
68
  isAllowed(url: string, userAgent?: string): boolean;
51
69
  /**
52
- * Get URLs of sitemaps referenced in the robots file.
70
+ * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
71
+ * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
72
+ * always dropped.
53
73
  */
54
- getSitemaps(): string[];
74
+ getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
55
75
  /**
56
- * Parse all the sitemaps referenced in the robots file.
76
+ * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
77
+ * and the sitemap parser.
57
78
  */
58
- parseSitemaps(): Promise<Sitemap>;
79
+ parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
59
80
  /**
60
81
  * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
82
+ * `options` are forwarded to `parseSitemaps`.
61
83
  */
62
- parseUrlsFromSitemaps(): Promise<string[]>;
84
+ parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
63
85
  }
64
- export { RobotsTxtFile as RobotsFile };
65
- //# sourceMappingURL=robots.d.ts.map
@@ -1,6 +1,7 @@
1
1
  import { FetchHttpClient } from '@crawlee/http-client';
2
2
  import robotsParser from 'robots-parser';
3
3
  import { Sitemap } from './sitemap.js';
4
+ import { filterUrl } from './url.js';
4
5
  /**
5
6
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
6
7
  *
@@ -20,22 +21,30 @@ import { Sitemap } from './sitemap.js';
20
21
  * ```
21
22
  */
22
23
  export class RobotsTxtFile {
23
- robots;
24
- proxyUrl;
25
- constructor(robots, proxyUrl) {
26
- this.robots = robots;
27
- this.proxyUrl = proxyUrl;
24
+ #url;
25
+ #robots;
26
+ #proxyUrl;
27
+ #logger;
28
+ constructor(url, robots, proxyUrl, logger) {
29
+ this.#url = url;
30
+ this.#robots = robots;
31
+ this.#proxyUrl = proxyUrl;
32
+ this.#logger = logger;
28
33
  }
29
34
  /**
30
35
  * Determine the location of a robots.txt file for a URL and fetch it.
31
36
  * @param url the URL to fetch robots.txt for
32
- * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
37
+ * @param [options] additional options
38
+ * @param [options.signal] an AbortSignal to cancel the request
39
+ * @param [options.timeoutMillis] timeout in milliseconds for the request
40
+ * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
41
+ * @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
33
42
  */
34
43
  static async find(url, options) {
35
44
  const robotsTxtFileUrl = new URL(url);
36
45
  robotsTxtFileUrl.pathname = '/robots.txt';
37
46
  robotsTxtFileUrl.search = '';
38
- return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
47
+ return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
39
48
  }
40
49
  /**
41
50
  * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -45,28 +54,43 @@ export class RobotsTxtFile {
45
54
  */
46
55
  static from(url, content, proxyUrl) {
47
56
  // @ts-ignore
48
- return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
57
+ return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
49
58
  }
50
- static async load(url, options) {
51
- const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};
59
+ static async #load(url, options) {
60
+ const { proxyUrl, logger, session, httpClient = new FetchHttpClient() } = options || {};
52
61
  const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
62
+ session,
53
63
  proxyUrl,
64
+ timeoutMillis: options?.timeoutMillis,
65
+ signal: options?.signal,
54
66
  });
55
- if (response.status < 200 || response.status >= 300) {
56
- throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
57
- }
58
- if (response.status === 404) {
59
- return new RobotsTxtFile({
67
+ // RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
68
+ if (response.status >= 400 && response.status < 600) {
69
+ const allowed = response.status < 500;
70
+ return new RobotsTxtFile(url, {
60
71
  isAllowed() {
61
- return true;
72
+ return allowed;
62
73
  },
63
74
  getSitemaps() {
64
75
  return [];
65
76
  },
66
- }, proxyUrl);
77
+ getCrawlDelay() {
78
+ return undefined;
79
+ },
80
+ }, proxyUrl, logger);
81
+ }
82
+ if (response.status < 200 || response.status >= 300) {
83
+ throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
67
84
  }
68
85
  // @ts-ignore
69
- return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl);
86
+ return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
87
+ }
88
+ /**
89
+ * Get crawl delay for a given user agent.
90
+ * @param [userAgent] relevant user agent, default to `*`
91
+ */
92
+ getCrawlDelay(userAgent = '*') {
93
+ return this.#robots.getCrawlDelay(userAgent);
70
94
  }
71
95
  /**
72
96
  * Check if a URL should be crawled by robots.
@@ -74,27 +98,39 @@ export class RobotsTxtFile {
74
98
  * @param [userAgent] relevant user agent, default to `*`
75
99
  */
76
100
  isAllowed(url, userAgent = '*') {
77
- return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
101
+ return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
78
102
  }
79
103
  /**
80
- * Get URLs of sitemaps referenced in the robots file.
104
+ * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
105
+ * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
106
+ * always dropped.
81
107
  */
82
- getSitemaps() {
83
- return this.robots.getSitemaps();
108
+ getSitemaps(options = {}) {
109
+ const { enqueueStrategy = 'same-hostname' } = options;
110
+ const sitemaps = [];
111
+ for (const sitemapUrl of this.#robots.getSitemaps()) {
112
+ // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
113
+ const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);
114
+ if (!allowed) {
115
+ this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);
116
+ continue;
117
+ }
118
+ sitemaps.push(sitemapUrl);
119
+ }
120
+ return sitemaps;
84
121
  }
85
122
  /**
86
- * Parse all the sitemaps referenced in the robots file.
123
+ * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
124
+ * and the sitemap parser.
87
125
  */
88
- async parseSitemaps() {
89
- return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
126
+ async parseSitemaps(options = {}) {
127
+ return Sitemap.load(this.getSitemaps(options), this.#proxyUrl, { ...options, logger: this.#logger });
90
128
  }
91
129
  /**
92
130
  * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
131
+ * `options` are forwarded to `parseSitemaps`.
93
132
  */
94
- async parseUrlsFromSitemaps() {
95
- return (await this.parseSitemaps()).urls;
133
+ async parseUrlsFromSitemaps(options = {}) {
134
+ return (await this.parseSitemaps(options)).urls;
96
135
  }
97
136
  }
98
- // to stay backwards compatible
99
- export { RobotsTxtFile as RobotsFile };
100
- //# sourceMappingURL=robots.js.map
@@ -0,0 +1,114 @@
1
+ import { BaseHttpClient } from '@crawlee/http-client';
2
+ import type { Dictionary } from '@crawlee/types';
3
+ import { z } from 'zod';
4
+ /**
5
+ * Accepts any object (including arrays and functions).
6
+ * @internal
7
+ */
8
+ export declare const anyObject: z.ZodCustom<Dictionary, Dictionary>;
9
+ /**
10
+ * Accepts any array without validating its items (cheap for huge arrays).
11
+ * @internal
12
+ */
13
+ export declare const anyArray: z.ZodCustom<unknown[], unknown[]>;
14
+ /**
15
+ * Accepts any function.
16
+ * @internal
17
+ */
18
+ export declare const anyFunction: z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>;
19
+ /**
20
+ * Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
21
+ * @internal
22
+ */
23
+ export declare const anyNumber: z.ZodCustom<number, number>;
24
+ /**
25
+ * Accepts any object (including functions) that has all the given keys, own or inherited.
26
+ * @internal
27
+ */
28
+ export declare function objectWithKeys(keys: string[], message?: string): z.ZodType<Dictionary>;
29
+ /**
30
+ * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
31
+ * @internal
32
+ */
33
+ export declare const httpClient: z.ZodInstanceOf<BaseHttpClient>;
34
+ /**
35
+ * Accepts any object implementing the CrawleeLogger interface.
36
+ * @internal
37
+ */
38
+ export declare const logger: z.ZodType<Dictionary, unknown, z.core.$ZodTypeInternals<Dictionary, unknown>>;
39
+ /**
40
+ * Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
41
+ * @internal
42
+ */
43
+ export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>, NodeJS.TypedArray<ArrayBufferLike>>;
44
+ /**
45
+ * Accepts any non-null, non-array object.
46
+ * @internal
47
+ */
48
+ export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
49
+ /**
50
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
51
+ * @internal
52
+ */
53
+ export declare const storageRequest: z.ZodObject<{
54
+ id: z.ZodString;
55
+ url: z.ZodURL;
56
+ uniqueKey: z.ZodString;
57
+ method: z.ZodOptional<z.ZodString>;
58
+ retryCount: z.ZodOptional<z.ZodNumber>;
59
+ handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
60
+ }, z.core.$loose>;
61
+ /**
62
+ * {@link storageRequest} before an id is assigned.
63
+ * @internal
64
+ */
65
+ export declare const storageRequestWithoutId: z.ZodObject<{
66
+ url: z.ZodURL;
67
+ uniqueKey: z.ZodString;
68
+ method: z.ZodOptional<z.ZodString>;
69
+ retryCount: z.ZodOptional<z.ZodNumber>;
70
+ handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
71
+ }, z.core.$loose>;
72
+ /**
73
+ * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
74
+ * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
75
+ * is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
76
+ * @internal
77
+ */
78
+ export declare function arrayOf<TItem extends z.ZodType>(item: TItem, elements: string): z.ZodArray<TItem>;
79
+ /**
80
+ * Batch of {@link storageRequestWithoutId}.
81
+ * @internal
82
+ */
83
+ export declare const storageRequestBatch: z.ZodArray<z.ZodObject<{
84
+ url: z.ZodURL;
85
+ uniqueKey: z.ZodString;
86
+ method: z.ZodOptional<z.ZodString>;
87
+ retryCount: z.ZodOptional<z.ZodNumber>;
88
+ handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
89
+ }, z.core.$loose>>;
90
+ /**
91
+ * Options of request queue add/update operations.
92
+ * @internal
93
+ */
94
+ export declare const requestQueueOperationOptions: z.ZodObject<{
95
+ forefront: z.ZodOptional<z.ZodBoolean>;
96
+ }, z.core.$strip>;
97
+ /**
98
+ * Options of key-value store `listKeys`.
99
+ * @internal
100
+ */
101
+ export declare const keyValueStoreListKeysOptions: z.ZodObject<{
102
+ prefix: z.ZodOptional<z.ZodString>;
103
+ exclusiveStartKey: z.ZodOptional<z.ZodString>;
104
+ limit: z.ZodOptional<z.ZodNumber>;
105
+ }, z.core.$strip>;
106
+ /**
107
+ * Options of dataset item listing.
108
+ * @internal
109
+ */
110
+ export declare const datasetListItemsOptions: z.ZodObject<{
111
+ desc: z.ZodOptional<z.ZodBoolean>;
112
+ limit: z.ZodOptional<z.ZodNumber>;
113
+ offset: z.ZodOptional<z.ZodNumber>;
114
+ }, z.core.$strip>;
@@ -0,0 +1,115 @@
1
+ import { BaseHttpClient } from '@crawlee/http-client';
2
+ import { z } from 'zod';
3
+ /**
4
+ * Accepts any object (including arrays and functions).
5
+ * @internal
6
+ */
7
+ export const anyObject = z.custom((value) => (typeof value === 'object' && value !== null) || typeof value === 'function', { message: 'Invalid input: expected object' });
8
+ /**
9
+ * Accepts any array without validating its items (cheap for huge arrays).
10
+ * @internal
11
+ */
12
+ export const anyArray = z.custom(Array.isArray, { message: 'Invalid input: expected array' });
13
+ /**
14
+ * Accepts any function.
15
+ * @internal
16
+ */
17
+ export const anyFunction = z.custom((value) => typeof value === 'function', {
18
+ message: 'Invalid input: expected function',
19
+ });
20
+ /**
21
+ * Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
22
+ * @internal
23
+ */
24
+ export const anyNumber = z.custom((value) => typeof value === 'number' && !Number.isNaN(value), {
25
+ message: 'Invalid input: expected number',
26
+ });
27
+ /**
28
+ * Accepts any object (including functions) that has all the given keys, own or inherited.
29
+ * @internal
30
+ */
31
+ export function objectWithKeys(keys, message) {
32
+ return z.custom((value) => ((typeof value === 'object' && value !== null) || typeof value === 'function') &&
33
+ keys.every((key) => key in value), {
34
+ message: message ?? `Invalid input: expected an object with keys ${keys.map((key) => `'${key}'`).join(', ')}`,
35
+ });
36
+ }
37
+ /**
38
+ * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
39
+ * @internal
40
+ */
41
+ export const httpClient = z.instanceof(BaseHttpClient);
42
+ /**
43
+ * Accepts any object implementing the CrawleeLogger interface.
44
+ * @internal
45
+ */
46
+ export const logger = objectWithKeys(['child', 'info', 'error', 'warning'], "Expected an object implementing the CrawleeLogger interface (missing one of 'child', 'info', 'error', 'warning'), got something else.");
47
+ /**
48
+ * Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
49
+ * @internal
50
+ */
51
+ export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(value instanceof DataView), { message: 'Invalid input: expected a typed array' });
52
+ /**
53
+ * Accepts any non-null, non-array object.
54
+ * @internal
55
+ */
56
+ export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
57
+ const storageRequestShape = z.looseObject({
58
+ id: z.string(),
59
+ url: z.url({ protocol: /^https?$/ }),
60
+ uniqueKey: z.string(),
61
+ method: z.string().optional(),
62
+ retryCount: z.number().int().optional(),
63
+ handledAt: z.union([z.string(), z.date()]).optional(),
64
+ });
65
+ /**
66
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
67
+ * @internal
68
+ */
69
+ export const storageRequest = z.compile(storageRequestShape);
70
+ /**
71
+ * {@link storageRequest} before an id is assigned.
72
+ * @internal
73
+ */
74
+ export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
75
+ /**
76
+ * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
77
+ * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
78
+ * is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
79
+ * @internal
80
+ */
81
+ export function arrayOf(item, elements) {
82
+ return z.array(item, {
83
+ error: (issue) => issue.code === 'invalid_type' ? `Invalid input: expected an array of ${elements}` : undefined,
84
+ });
85
+ }
86
+ /**
87
+ * Batch of {@link storageRequestWithoutId}.
88
+ * @internal
89
+ */
90
+ export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
91
+ /**
92
+ * Options of request queue add/update operations.
93
+ * @internal
94
+ */
95
+ export const requestQueueOperationOptions = z.object({
96
+ forefront: z.boolean().optional(),
97
+ });
98
+ /**
99
+ * Options of key-value store `listKeys`.
100
+ * @internal
101
+ */
102
+ export const keyValueStoreListKeysOptions = z.object({
103
+ prefix: z.string().optional(),
104
+ exclusiveStartKey: z.string().optional(),
105
+ limit: z.number().int().gt(0).optional(),
106
+ });
107
+ /**
108
+ * Options of dataset item listing.
109
+ * @internal
110
+ */
111
+ export const datasetListItemsOptions = z.object({
112
+ desc: z.boolean().optional(),
113
+ limit: z.number().int().optional(),
114
+ offset: z.number().int().optional(),
115
+ });
@@ -1,4 +1,6 @@
1
- import type { BaseHttpClient } from '@crawlee/types';
1
+ import type { BaseHttpClient } from '@crawlee/http-client';
2
+ import type { CrawleeLogger } from '@crawlee/types';
3
+ import { type EnqueueStrategy } from './url.js';
2
4
  interface SitemapUrlData {
3
5
  loc: string;
4
6
  lastmod?: Date;
@@ -47,6 +49,24 @@ export interface ParseSitemapOptions {
47
49
  * Custom HTTP client to be used for fetching sitemaps.
48
50
  */
49
51
  httpClient?: BaseHttpClient;
52
+ /**
53
+ * Optional filter for nested sitemap URLs discovered in sitemap index files.
54
+ * Called with the URL of each child sitemap before it is fetched.
55
+ * Return `true` to include the sitemap, `false` to skip it.
56
+ * If not provided, all nested sitemaps are followed.
57
+ */
58
+ nestedSitemapFilter?: (sitemapUrl: string) => boolean;
59
+ /**
60
+ * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
61
+ * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
62
+ * sources (no parent URL). Pass `'all'` to disable host filtering.
63
+ * @default 'same-hostname'
64
+ */
65
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
66
+ /**
67
+ * Optional logger for reporting warnings during sitemap parsing.
68
+ */
69
+ logger?: CrawleeLogger;
50
70
  }
51
71
  export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
52
72
  /**
@@ -62,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
62
82
  * ```
63
83
  */
64
84
  export declare class Sitemap {
85
+ #private;
65
86
  readonly urls: string[];
66
87
  constructor(urls: string[]);
67
88
  /**
@@ -83,7 +104,6 @@ export declare class Sitemap {
83
104
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
84
105
  */
85
106
  static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
86
- protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
87
107
  }
88
108
  /**
89
109
  * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
@@ -97,10 +117,31 @@ export declare function discoverValidSitemaps(urls: string[], options?: {
97
117
  * Proxy URL to be used for network requests.
98
118
  */
99
119
  proxyUrl?: string;
120
+ /**
121
+ * Timeout in milliseconds for the entire `discoverValidSitemaps` call.
122
+ * An `AbortController` is created internally and its signal is passed to every HTTP request,
123
+ * so the whole discovery operation is cancelled once the timeout elapses.
124
+ * Defaults to `60_000` ms (60 seconds) to prevent indefinite hangs.
125
+ */
126
+ timeoutMillis?: number;
127
+ /**
128
+ * An external `AbortSignal` to cancel the entire discovery operation.
129
+ * If both `signal` and `timeout` are provided, the operation is cancelled
130
+ * when either the signal is aborted or the timeout elapses (whichever comes first).
131
+ */
132
+ signal?: AbortSignal;
133
+ /**
134
+ * Timeout in milliseconds for each individual HTTP request during discovery.
135
+ * Defaults to `20000` ms (20 seconds).
136
+ */
137
+ requestTimeoutMillis?: number;
100
138
  /**
101
139
  * HTTP client to be used for network requests.
102
140
  */
103
141
  httpClient?: BaseHttpClient;
142
+ /**
143
+ * Optional logger for reporting warnings during sitemap discovery.
144
+ */
145
+ logger?: CrawleeLogger;
104
146
  }): AsyncIterable<string>;
105
147
  export {};
106
- //# sourceMappingURL=sitemap.d.ts.map