@crawlee/basic 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/index.d.ts +6 -6
  2. package/index.js +6 -6
  3. package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +6 -6
  4. package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +3 -1
  5. package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
  6. package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
  7. package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
  8. package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
  9. package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
  10. package/internals/autoscaling/index.d.ts +8 -8
  11. package/internals/autoscaling/index.js +8 -8
  12. package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
  13. package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
  14. package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
  15. package/internals/autoscaling/snapshotter.d.ts +5 -5
  16. package/internals/autoscaling/snapshotter.js +4 -4
  17. package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
  18. package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
  19. package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
  20. package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
  21. package/internals/basic-crawler.d.ts +16 -11
  22. package/internals/basic-crawler.js +61 -66
  23. package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
  24. package/internals/crawler-run.d.ts +2 -2
  25. package/internals/crawler-run.js +3 -1
  26. package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
  27. package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
  28. package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
  29. package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
  30. package/internals/crawlers/index.d.ts +3 -3
  31. package/internals/crawlers/index.js +2 -2
  32. package/internals/crawlers/statistics.d.ts +11 -1
  33. package/internals/crawlers/statistics.js +39 -5
  34. package/internals/{crawling_request.js → crawling-request.js} +1 -1
  35. package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +1 -6
  36. package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -41
  37. package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
  38. package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
  39. package/internals/{enqueue_links → enqueue-links}/shared.d.ts +5 -4
  40. package/internals/{enqueue_links → enqueue-links}/shared.js +15 -45
  41. package/internals/errors.d.ts +6 -1
  42. package/internals/errors.js +25 -0
  43. package/internals/router.d.ts +2 -2
  44. package/internals/send-request.js +17 -8
  45. package/internals/{session_pool → session-pool}/index.d.ts +1 -1
  46. package/internals/{session_pool → session-pool}/index.js +1 -1
  47. package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +8 -1
  48. package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +116 -56
  49. package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +7 -7
  50. package/internals/{throttling_request_manager.js → throttling-request-manager.js} +61 -44
  51. package/package.json +15 -15
  52. /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
  53. /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
  54. /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
  55. /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
  56. /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
  57. /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
  58. /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
  59. /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
  60. /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
  61. /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
  62. /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
  63. /package/internals/{session_pool → session-pool}/consts.js +0 -0
  64. /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
  65. /package/internals/{session_pool → session-pool}/errors.js +0 -0
  66. /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
  67. /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
  68. /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
  69. /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
  70. /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
  71. /package/internals/{session_pool → session-pool}/session.js +0 -0
@@ -1,6 +1,6 @@
1
1
  import { serviceLocator } from '@crawlee/core';
2
2
  import { Cookie, CookieJar } from 'tough-cookie';
3
- import { CookieParseError } from './session_pool/errors.js';
3
+ import { CookieParseError } from './session-pool/errors.js';
4
4
  /**
5
5
  * @internal
6
6
  */
@@ -1,6 +1,6 @@
1
1
  import type { CrawleeLogger } from '@crawlee/core';
2
- import type { AutoscaledPoolOptions } from './autoscaling/autoscaled_pool.js';
3
- import type { IConcurrencySystem } from './autoscaling/concurrency_system.js';
2
+ import type { AutoscaledPoolOptions } from './autoscaling/autoscaled-pool.js';
3
+ import type { IConcurrencySystem } from './autoscaling/concurrency-system.js';
4
4
  /** Everything a run needs to exist. Whatever a crawler resolves per run is resolved before the run is built. */
5
5
  export interface CrawlerRunSetup {
6
6
  log: CrawleeLogger;
@@ -1,4 +1,4 @@
1
- import { AutoscaledPool } from './autoscaling/autoscaled_pool.js';
1
+ import { AutoscaledPool } from './autoscaling/autoscaled-pool.js';
2
2
  /**
3
3
  * Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
4
4
  * requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
@@ -38,6 +38,8 @@ export class CrawlerRun {
38
38
  * running by now — the task loop refuses to book anything against one that is not.
39
39
  */
40
40
  async dispatchRequests() {
41
+ if (this.#ended)
42
+ return;
41
43
  this.#dispatching = true;
42
44
  try {
43
45
  await this.#autoscaledPool.run();
@@ -1,8 +1,8 @@
1
1
  import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
2
2
  import type { ReadonlyDeep } from 'type-fest';
3
3
  import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
4
- import type { EnqueueUrlsOptions } from '../enqueue_links/index.js';
5
- import type { CrawlingRequest } from '../crawling_request.js';
4
+ import type { EnqueueUrlsOptions } from '../enqueue-links/index.js';
5
+ import type { CrawlingRequest } from '../crawling-request.js';
6
6
  /** @internal */
7
7
  export type IsAny<T> = 0 extends 1 & T ? true : false;
8
8
  /**
@@ -1,5 +1,5 @@
1
- import type { CrawlingContext } from '../crawlers/crawler_commons.js';
2
- import type { ErrnoException } from './error_tracker.js';
1
+ import type { CrawlingContext } from './crawler-commons.js';
2
+ import type { ErrnoException } from './error-tracker.js';
3
3
  import type { SnapshottableProperties } from './internals/types.js';
4
4
  interface ErrorSnapshot {
5
5
  screenshotFileName?: string;
@@ -1,4 +1,4 @@
1
- import type { CrawlingContext } from '../crawlers/crawler_commons.js';
1
+ import type { CrawlingContext } from './crawler-commons.js';
2
2
  /**
3
3
  * Node.js Error interface
4
4
  */
@@ -1,5 +1,5 @@
1
1
  import { inspect } from 'node:util';
2
- import { ErrorSnapshotter } from './error_snapshotter.js';
2
+ import { ErrorSnapshotter } from './error-snapshotter.js';
3
3
  const extractPathFromStackTraceLine = (line) => {
4
4
  const lastStartingRoundBracketIndex = line.lastIndexOf('(');
5
5
  if (lastStartingRoundBracketIndex !== -1) {
@@ -1,4 +1,4 @@
1
- export * from './context_pipeline.js';
2
- export type * from './crawler_commons.js';
1
+ export * from './context-pipeline.js';
2
+ export type * from './crawler-commons.js';
3
3
  export * from './statistics.js';
4
- export * from './error_tracker.js';
4
+ export * from './error-tracker.js';
@@ -1,3 +1,3 @@
1
- export * from './context_pipeline.js';
1
+ export * from './context-pipeline.js';
2
2
  export * from './statistics.js';
3
- export * from './error_tracker.js';
3
+ export * from './error-tracker.js';
@@ -1,6 +1,6 @@
1
1
  import type { CrawleeLogger, SyncStateConversion } from '@crawlee/core';
2
2
  import { KeyValueStore } from '@crawlee/core';
3
- import { ErrorTracker } from './error_tracker.js';
3
+ import { ErrorTracker } from './error-tracker.js';
4
4
  /**
5
5
  * Persistence-related options to control how and when crawler's data gets persisted.
6
6
  */
@@ -50,6 +50,11 @@ export interface IStatistics<StateExtension extends object = {}> {
50
50
  startCapturing(): Promise<void>;
51
51
  /** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
52
52
  stopCapturing(): Promise<void>;
53
+ /**
54
+ * Registers a retry attempt in real-time when a request temporarily fails.
55
+ * Optional - crawlers call this when a request is reclaimed back to the queue or list.
56
+ */
57
+ registerRetry?(retryCount: number): void;
53
58
  /**
54
59
  * Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
55
60
  * with no persistence of its own can omit it.
@@ -155,6 +160,11 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
155
160
  * Stops logging and remove event listeners, then persist
156
161
  */
157
162
  stopCapturing(): Promise<void>;
163
+ /**
164
+ * Registers a retry attempt in real-time when a request temporarily fails.
165
+ * Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
166
+ */
167
+ registerRetry(retryCount: number): void;
158
168
  /**
159
169
  * Persist internal state to the key value store.
160
170
  *
@@ -1,7 +1,7 @@
1
1
  import { convertStateSync, KeyValueStore, RecoverableState, serviceLocator, validators } from '@crawlee/core';
2
2
  import { parseArgument, schemas } from '@crawlee/utils/internal';
3
3
  import { z } from 'zod';
4
- import { ErrorTracker } from './error_tracker.js';
4
+ import { ErrorTracker } from './error-tracker.js';
5
5
  /**
6
6
  * @ignore
7
7
  */
@@ -38,7 +38,7 @@ const errorTrackerConfig = {
38
38
  * order here *is* the record's field order (guarded by a test).
39
39
  *
40
40
  * JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as
41
- * `null`. Both {@link Statistics.serializeState} and {@link Statistics.deserializeState} run through this,
41
+ * `null`. Both the private serialize and deserialize steps of `Statistics` run through this,
42
42
  * which is what keeps them describing the same record.
43
43
  *
44
44
  * Nothing is optional on purpose: the record has always carried every field, so one missing a field is not one of
@@ -410,11 +410,45 @@ export class Statistics {
410
410
  this.state.crawlerFinishedAt = new Date();
411
411
  await this.#recoverableState.teardown();
412
412
  }
413
- #saveRetryCountForRequest(retryCount) {
414
- if (retryCount > 0)
413
+ /**
414
+ * Registers a retry attempt in real-time when a request temporarily fails.
415
+ * Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
416
+ */
417
+ registerRetry(retryCount) {
418
+ if (!Number.isInteger(retryCount) || retryCount < 0)
419
+ return;
420
+ if (retryCount === 1) {
415
421
  this.state.requestsRetries++;
416
- this.requestRetryHistogram[retryCount] ??= 0;
422
+ }
423
+ // Fill intermediate indices with 0 to prevent sparse arrays and JSON serialization null values
424
+ while (this.requestRetryHistogram.length <= retryCount) {
425
+ this.requestRetryHistogram.push(0);
426
+ }
417
427
  this.requestRetryHistogram[retryCount]++;
428
+ const previousRetryCount = retryCount - 1;
429
+ if (previousRetryCount > 0) {
430
+ while (this.requestRetryHistogram.length <= previousRetryCount) {
431
+ this.requestRetryHistogram.push(0);
432
+ }
433
+ // Decrement previous count only if it is strictly greater than 0
434
+ if (this.requestRetryHistogram[previousRetryCount] > 0) {
435
+ this.requestRetryHistogram[previousRetryCount]--;
436
+ }
437
+ }
438
+ }
439
+ #saveRetryCountForRequest(retryCount) {
440
+ if (!Number.isInteger(retryCount) || retryCount < 0)
441
+ return;
442
+ while (this.requestRetryHistogram.length <= retryCount) {
443
+ this.requestRetryHistogram.push(0);
444
+ }
445
+ if (retryCount === 0) {
446
+ this.requestRetryHistogram[0]++;
447
+ }
448
+ else if (this.requestRetryHistogram[retryCount] === 0) {
449
+ this.requestRetryHistogram[retryCount]++;
450
+ this.state.requestsRetries += retryCount;
451
+ }
418
452
  }
419
453
  /**
420
454
  * Persist internal state to the key value store.
@@ -99,7 +99,7 @@ export class CrawlingRequest extends Request {
99
99
  message = omitStack
100
100
  ? errorOrMessage.message
101
101
  : // .stack includes the message
102
- errorOrMessage.stack;
102
+ (errorOrMessage.stack ?? errorOrMessage.message);
103
103
  }
104
104
  else if (Reflect.has(Object(errorOrMessage), 'message')) {
105
105
  message = Reflect.get(Object(errorOrMessage), 'message');
@@ -1,6 +1,6 @@
1
1
  import type { Dictionary } from '@crawlee/types';
2
2
  import type { EnqueueStrategyOption, RequestQueueOperationOptions } from '@crawlee/core';
3
- import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
3
+ import type { RequestTransform, SkippedRequestCallback, UrlPatternInput } from './shared.js';
4
4
  /**
5
5
  * Options shared by the `extractLinks()` context helper across crawler types.
6
6
  */
@@ -145,8 +145,3 @@ export interface ResolveBaseUrl {
145
145
  originalRequestUrl: string;
146
146
  finalRequestUrl?: string;
147
147
  }
148
- /**
149
- * @internal
150
- * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
151
- */
152
- export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];
@@ -33,44 +33,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
33
33
  // before actually finding the urls
34
34
  return originalUrlOrigin;
35
35
  }
36
- /**
37
- * @internal
38
- * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
39
- */
40
- export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
41
- const url = new URL(baseUrl);
42
- switch (strategy) {
43
- case EnqueueStrategy.SameHostname:
44
- // We need to get the origin of the passed in domain in the event someone sets baseUrl
45
- // to an url like https://example.com/deep/default/path and one of the found urls is an
46
- // absolute relative path (/path/to/page)
47
- return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
48
- case EnqueueStrategy.SameDomain: {
49
- // Get the actual hostname from the base url
50
- const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
51
- if (baseUrlHostname) {
52
- // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
53
- url.hostname = baseUrlHostname;
54
- return [
55
- { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
56
- { glob: ignoreHttpSchema(`${url.origin}/**`) },
57
- ];
58
- }
59
- // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
60
- // as SameDomainAndSubdomain
61
- return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
62
- }
63
- case EnqueueStrategy.SameOrigin:
64
- // The same behavior as SameHostname, but respecting the protocol of the URL
65
- return [{ glob: `${url.origin}/**` }];
66
- case EnqueueStrategy.All:
67
- default:
68
- return [{ glob: `http{s,}://**` }];
69
- }
70
- }
71
- /**
72
- * Internal function that changes the enqueue glob patterns to match both http and https
73
- */
74
- function ignoreHttpSchema(pattern) {
75
- return pattern.replace(/^(https?):\/\//, 'http{s,}://');
76
- }
@@ -1,3 +1,3 @@
1
- export * from './enqueue_links.js';
1
+ export * from './enqueue-links.js';
2
2
  export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
3
3
  export type { GlobInput, GlobObject, RegExpInput, RegExpObject, RequestTransform, SkippedRequestCallback, UrlPatternInput, } from './shared.js';
@@ -1,3 +1,3 @@
1
- export * from './enqueue_links.js';
1
+ export * from './enqueue-links.js';
2
2
  // Not `export *`: keeps the internal `UrlPatternObject` off the public surface.
3
3
  export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
@@ -2,7 +2,7 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
2
2
  import { z } from 'zod';
3
3
  import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
4
4
  import { Request } from '@crawlee/core';
5
- import type { SkippedRequestReason } from '../crawling_request.js';
5
+ import type { SkippedRequestReason } from '../crawling-request.js';
6
6
  export interface GlobObject {
7
7
  glob: string;
8
8
  }
@@ -34,13 +34,13 @@ export declare function createSkippedRequestArgs(source: string | Source, reason
34
34
  /**
35
35
  * @ignore
36
36
  */
37
- export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput, pattern: RegExpObject | GlobObject): void;
37
+ export declare function updateEnqueueLinksPatternCache(item: UrlPatternInput, pattern: UrlPatternObject): void;
38
38
  /**
39
39
  * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
40
40
  * to construct Glob objects from Glob pattern strings.
41
41
  * @ignore
42
42
  */
43
- export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
43
+ export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): (GlobObject & UrlPatternObject)[];
44
44
  /**
45
45
  * @internal
46
46
  */
@@ -50,7 +50,7 @@ export declare function validateGlobPattern(glob: string): string;
50
50
  * to check RegExps input and return valid RegExps.
51
51
  * @ignore
52
52
  */
53
- export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
53
+ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): (RegExpObject & UrlPatternObject)[];
54
54
  /**
55
55
  * The compiled form of a {@link UrlPatternInput}, produced by {@link constructUrlPatternObjects}. Deliberately
56
56
  * not re-exported from the package entry point — it is an implementation detail of the pattern matching helpers.
@@ -59,6 +59,7 @@ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegE
59
59
  export interface UrlPatternObject {
60
60
  glob?: string;
61
61
  regexp?: RegExp;
62
+ match: (url: string) => boolean;
62
63
  }
63
64
  /**
64
65
  * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
@@ -45,8 +45,7 @@ export function createSkippedRequestArgs(source, reason) {
45
45
  export function updateEnqueueLinksPatternCache(item, pattern) {
46
46
  enqueueLinksPatternCache.set(item, pattern);
47
47
  if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
48
- const key = enqueueLinksPatternCache.keys().next().value;
49
- enqueueLinksPatternCache.delete(key);
48
+ enqueueLinksPatternCache.delete(enqueueLinksPatternCache.keys().next().value);
50
49
  }
51
50
  }
52
51
  /**
@@ -70,16 +69,12 @@ export function constructGlobObjectsFromGlobs(globs) {
70
69
  return false;
71
70
  })
72
71
  .map((item) => {
73
- // Get glob object from cache.
74
- let globObject = enqueueLinksPatternCache.get(item);
75
- if (globObject)
76
- return globObject;
77
- if (typeof item === 'string') {
78
- globObject = { glob: validateGlobPattern(item) };
79
- }
80
- else {
81
- globObject = { glob: validateGlobPattern(item.glob) };
82
- }
72
+ const cached = enqueueLinksPatternCache.get(item);
73
+ if (cached)
74
+ return cached;
75
+ const glob = validateGlobPattern(typeof item === 'string' ? item : item.glob);
76
+ const minimatch = new Minimatch(glob, { nocase: true });
77
+ const globObject = { glob, match: (url) => minimatch.match(url) };
83
78
  updateEnqueueLinksPatternCache(item, globObject);
84
79
  return globObject;
85
80
  });
@@ -100,16 +95,11 @@ export function validateGlobPattern(glob) {
100
95
  */
101
96
  export function constructRegExpObjectsFromRegExps(regexps) {
102
97
  return regexps.map((item) => {
103
- // Get regexp object from cache.
104
- let regexpObject = enqueueLinksPatternCache.get(item);
105
- if (regexpObject)
106
- return regexpObject;
107
- if (item instanceof RegExp) {
108
- regexpObject = { regexp: item };
109
- }
110
- else {
111
- regexpObject = { regexp: item.regexp };
112
- }
98
+ const cached = enqueueLinksPatternCache.get(item);
99
+ if (cached)
100
+ return cached;
101
+ const regexp = item instanceof RegExp ? item : item.regexp;
102
+ const regexpObject = { regexp, match: (url) => regexp.test(url) };
113
103
  updateEnqueueLinksPatternCache(item, regexpObject);
114
104
  return regexpObject;
115
105
  });
@@ -138,21 +128,19 @@ export function constructUrlPatternObjects(patterns) {
138
128
  * @ignore
139
129
  */
140
130
  export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
141
- const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
142
- const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
143
131
  return requestOptions
144
132
  .filter((opts) => {
145
- const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
133
+ const matchesExclude = excludePatterns.some(({ match }) => match(opts.url));
146
134
  if (matchesExclude) {
147
135
  onSkippedRequestOptions?.(opts);
148
136
  }
149
137
  return !matchesExclude;
150
138
  })
151
139
  .map((opts) => {
152
- if (!includeMatchers) {
140
+ if (!includePatterns?.length) {
153
141
  return { ...opts, enqueueStrategy: strategy };
154
142
  }
155
- for (const { match } of includeMatchers) {
143
+ for (const { match } of includePatterns) {
156
144
  if (match(opts.url)) {
157
145
  return { ...opts, enqueueStrategy: strategy };
158
146
  }
@@ -212,24 +200,6 @@ export function createRequestOptions(sources, options = {}) {
212
200
  return requestOptions;
213
201
  });
214
202
  }
215
- /**
216
- * @ignore
217
- */
218
- function createPatternObjectMatcher(urlPatternObject) {
219
- const { regexp, glob } = urlPatternObject;
220
- let match;
221
- if (regexp) {
222
- match = (url) => regexp.test(url);
223
- }
224
- else if (glob) {
225
- const m = new Minimatch(glob, { nocase: true });
226
- match = (url) => m.match(url);
227
- }
228
- else {
229
- match = () => false;
230
- }
231
- return { match };
232
- }
233
203
  /**
234
204
  * Applies a {@link RequestTransform} function to a list of request options.
235
205
  * Options for which the transform returns a falsy value are removed from the list.
@@ -1,5 +1,5 @@
1
1
  import { CriticalError, NonRetryableError } from '@crawlee/core';
2
- import type { SkippedRequestReason } from './crawling_request.js';
2
+ import type { SkippedRequestReason } from './crawling-request.js';
3
3
  /**
4
4
  * @ignore
5
5
  */
@@ -66,3 +66,8 @@ export declare class RequestHandlerError extends Error {
66
66
  */
67
67
  export declare class NavigationSkippedError extends NonRetryableError {
68
68
  }
69
+ /**
70
+ * Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
71
+ * @internal
72
+ */
73
+ export declare function skippedNavigationRequest<R extends object>(request: R): R;
@@ -81,3 +81,28 @@ export class RequestHandlerError extends Error {
81
81
  */
82
82
  export class NavigationSkippedError extends NonRetryableError {
83
83
  }
84
+ /**
85
+ * Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
86
+ * @internal
87
+ */
88
+ export function skippedNavigationRequest(request) {
89
+ return new Proxy(request, {
90
+ get(target, propertyName) {
91
+ if (propertyName === 'loadedUrl') {
92
+ throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
93
+ }
94
+ // `target` as receiver and bound methods, or `#` members throw on the proxy
95
+ const value = Reflect.get(target, propertyName, target);
96
+ return typeof value === 'function' ? value.bind(target) : value;
97
+ },
98
+ // Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.
99
+ getOwnPropertyDescriptor(target, propertyName) {
100
+ const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);
101
+ if (propertyName === 'loadedUrl' && descriptor) {
102
+ return { ...descriptor, enumerable: false };
103
+ }
104
+ return descriptor;
105
+ },
106
+ set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
107
+ });
108
+ }
@@ -1,7 +1,7 @@
1
- import type { CrawlingRequest } from './crawling_request.js';
1
+ import type { CrawlingRequest } from './crawling-request.js';
2
2
  import type { Awaitable, Dictionary } from '@crawlee/types';
3
3
  import type { StandardSchemaV1 } from '@standard-schema/spec';
4
- import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler_commons.js';
4
+ import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler-commons.js';
5
5
  /**
6
6
  * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
7
7
  * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
@@ -1,3 +1,5 @@
1
+ import { Readable } from 'node:stream';
2
+ import { processHttpRequestOptions } from '@crawlee/utils/internal';
1
3
  /**
2
4
  * Prepares a function to be used as the `sendRequest` context helper.
3
5
  *
@@ -9,24 +11,31 @@
9
11
  export function createSendRequest(httpClient, originRequest, session) {
10
12
  return async (overrideRequest = {}, overrideOptions = {}) => {
11
13
  const baseRequest = originRequest.intoFetchAPIRequest();
12
- const mergedUrl = overrideRequest.url ?? baseRequest.url;
13
- const mergedMethod = overrideRequest.method ?? baseRequest.method;
14
14
  const mergedHeaders = new Headers(baseRequest.headers);
15
15
  if (overrideRequest.headers) {
16
16
  overrideRequest.headers.forEach((value, key) => {
17
17
  mergedHeaders.set(key, value);
18
18
  });
19
19
  }
20
- const request = new Request(mergedUrl, {
21
- method: mergedMethod,
20
+ const opts = processHttpRequestOptions({
21
+ ...overrideRequest,
22
+ url: overrideRequest.url ?? baseRequest.url,
23
+ method: overrideRequest.method ?? originRequest.method,
22
24
  headers: mergedHeaders,
23
- body: overrideRequest.body ?? baseRequest.body,
25
+ });
26
+ const request = new Request(opts.url, {
27
+ method: opts.method,
28
+ headers: opts.headers,
29
+ body: opts.body ? Readable.toWeb(opts.body) : baseRequest.body,
30
+ // Node-specific option to make the request body work with streams
31
+ duplex: 'half',
24
32
  });
25
33
  return httpClient.sendRequest(request, {
26
34
  session,
27
- cookieJar: overrideOptions?.cookieJar ?? session.cookieJar,
28
- timeoutMillis: overrideOptions.timeoutMillis,
29
- signal: overrideOptions.signal,
35
+ cookieJar: overrideOptions.cookieJar ?? opts.cookieJar ?? session.cookieJar,
36
+ proxyUrl: overrideOptions.proxyUrl ?? opts.proxyUrl,
37
+ timeoutMillis: overrideOptions.timeoutMillis ?? opts.timeout,
38
+ signal: overrideOptions.signal ?? opts.signal,
30
39
  ignoreTlsErrors: overrideOptions.ignoreTlsErrors,
31
40
  });
32
41
  };
@@ -1,4 +1,4 @@
1
1
  export * from './errors.js';
2
2
  export * from './session.js';
3
- export * from './session_pool.js';
3
+ export * from './session-pool.js';
4
4
  export * from './consts.js';
@@ -1,4 +1,4 @@
1
1
  export * from './errors.js';
2
2
  export * from './session.js';
3
- export * from './session_pool.js';
3
+ export * from './session-pool.js';
4
4
  export * from './consts.js';
@@ -3,7 +3,7 @@ import type { ParseSitemapOptions } from '@crawlee/utils';
3
3
  import { EnqueueStrategy } from '@crawlee/utils';
4
4
  import type { IRequestLoader, IRequestManager, RequestLoaderStatus } from '@crawlee/core';
5
5
  import { Request } from '@crawlee/core';
6
- import type { UrlPatternInput } from './enqueue_links/index.js';
6
+ import type { UrlPatternInput } from './enqueue-links/index.js';
7
7
  interface UrlConstraints {
8
8
  /**
9
9
  * An array of URL patterns that URLs must match to be included.
@@ -80,6 +80,13 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
80
80
  * Custom HTTP client to be used for sitemap loading.
81
81
  */
82
82
  httpClient?: BaseHttpClient;
83
+ /**
84
+ * Maximum number of sitemaps to load in parallel.
85
+ * This includes both the initial sitemaps and any nested sitemaps discovered during parsing.
86
+ *
87
+ * @default 1
88
+ */
89
+ concurrency?: number;
83
90
  }
84
91
  /**
85
92
  * A list of URLs to crawl parsed from a sitemap.