@crawlee/basic 4.0.0-beta.254 → 4.0.0-beta.256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -654,8 +654,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
654
654
  protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
655
655
  /**
656
656
  * Handles errors thrown by user provided requestHandler()
657
- *
658
- * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
659
657
  */
660
658
  private requestFunctionErrorHandler;
661
659
  private handleFailedRequestHandler;
@@ -471,14 +471,14 @@ export class BasicCrawler {
471
471
  // Navigation, the navigation hooks and the request handler are timed individually, but the
472
472
  // phases between them are not, so a request could still get stuck indefinitely. This is the
473
473
  // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
474
- await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
474
+ await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
475
475
  }
476
476
  catch (error) {
477
477
  // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
478
478
  // take the normal error flow. Anything else is fatal.
479
479
  if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
480
480
  const unwrappedError = this.#unwrapError(error);
481
- await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
481
+ await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
482
482
  if (!this.#errorAbsolvesSession(unwrappedError)) {
483
483
  crawlingContext.session?.markBad();
484
484
  }
@@ -1202,6 +1202,7 @@ export class BasicCrawler {
1202
1202
  });
1203
1203
  // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
1204
1204
  // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
1205
+ // Background failures are already logged by `drainRequestBatches`.
1205
1206
  void waitForAllRequestsToBeAdded.catch(() => { });
1206
1207
  return { ...result, waitForAllRequestsToBeAdded };
1207
1208
  }
@@ -1500,13 +1501,14 @@ export class BasicCrawler {
1500
1501
  }
1501
1502
  /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
1502
1503
  // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1503
- async handleRequest(crawlingContext, requestSource, request) {
1504
+ async handleRequest(crawlingContext, requestSource) {
1504
1505
  // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
1505
1506
  // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
1506
1507
  // execute (and re-report) on top of a request the crawler has already moved past.
1507
1508
  if (crawlingContext[timeoutExpiredKey]?.()) {
1508
1509
  return;
1509
1510
  }
1511
+ const { request } = crawlingContext;
1510
1512
  request.state = RequestState.REQUEST_HANDLER;
1511
1513
  await this.runRequestHandler(crawlingContext);
1512
1514
  // Commit *before* marking the request as handled, so a commit failure fails the request and
@@ -1524,7 +1526,8 @@ export class BasicCrawler {
1524
1526
  * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
1525
1527
  * error handler has seen the context.
1526
1528
  */
1527
- async #handleRequestFailure(rawError, crawlingContext, requestSource, request) {
1529
+ async #handleRequestFailure(rawError, crawlingContext, requestSource) {
1530
+ const { request } = crawlingContext;
1528
1531
  // Roll back *before* any error handler runs - error handlers write to real storage precisely
1529
1532
  // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
1530
1533
  currentStorageTransaction()?.rollback();
@@ -1536,18 +1539,19 @@ export class BasicCrawler {
1536
1539
  ? await this.#trySkipRequest(unwrappedError, request, requestSource)
1537
1540
  : unwrappedError;
1538
1541
  if (err) {
1539
- await this.#runErrorHandler(err, crawlingContext, requestSource, request);
1542
+ await this.#runErrorHandler(err, crawlingContext, requestSource);
1540
1543
  }
1541
1544
  }
1542
1545
  finally {
1543
1546
  resumeRequestTimeout?.();
1544
1547
  }
1545
1548
  }
1546
- async #runErrorHandler(err, context, requestSource, request) {
1549
+ async #runErrorHandler(err, context, requestSource) {
1550
+ const { request } = context;
1547
1551
  const retryCountBefore = request.retryCount;
1548
1552
  try {
1549
1553
  request.state = RequestState.ERROR_HANDLER;
1550
- await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1554
+ await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1551
1555
  // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
1552
1556
  if (request.state !== RequestState.SKIPPED) {
1553
1557
  request.state = RequestState.DONE;
@@ -1656,11 +1660,10 @@ export class BasicCrawler {
1656
1660
  }
1657
1661
  /**
1658
1662
  * Handles errors thrown by user provided requestHandler()
1659
- *
1660
- * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
1661
1663
  */
1662
1664
  // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1663
- async requestFunctionErrorHandler(error, crawlingContext, request, source) {
1665
+ async requestFunctionErrorHandler(error, crawlingContext, source) {
1666
+ const { request } = crawlingContext;
1664
1667
  if (error instanceof RequestThrottledError) {
1665
1668
  // The domain told us to come back later, so the request was never really attempted. Put it back where
1666
1669
  // it was, without recording a failure - it costs neither a retry nor session reputation.
@@ -1703,6 +1706,7 @@ export class BasicCrawler {
1703
1706
  }
1704
1707
  if (!request.noRetry) {
1705
1708
  request.retryCount++;
1709
+ this.statistics.registerRetry?.(request.retryCount);
1706
1710
  const { url, retryCount, id } = request;
1707
1711
  // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
1708
1712
  // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
@@ -50,6 +50,11 @@ export interface IStatistics<StateExtension extends object = {}> {
50
50
  startCapturing(): Promise<void>;
51
51
  /** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
52
52
  stopCapturing(): Promise<void>;
53
+ /**
54
+ * Registers a retry attempt in real-time when a request temporarily fails.
55
+ * Optional - crawlers call this when a request is reclaimed back to the queue or list.
56
+ */
57
+ registerRetry?(retryCount: number): void;
53
58
  /**
54
59
  * Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
55
60
  * with no persistence of its own can omit it.
@@ -155,6 +160,11 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
155
160
  * Stops logging and remove event listeners, then persist
156
161
  */
157
162
  stopCapturing(): Promise<void>;
163
+ /**
164
+ * Registers a retry attempt in real-time when a request temporarily fails.
165
+ * Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
166
+ */
167
+ registerRetry(retryCount: number): void;
158
168
  /**
159
169
  * Persist internal state to the key value store.
160
170
  *
@@ -410,11 +410,45 @@ export class Statistics {
410
410
  this.state.crawlerFinishedAt = new Date();
411
411
  await this.#recoverableState.teardown();
412
412
  }
413
- #saveRetryCountForRequest(retryCount) {
414
- if (retryCount > 0)
413
+ /**
414
+ * Registers a retry attempt in real-time when a request temporarily fails.
415
+ * Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
416
+ */
417
+ registerRetry(retryCount) {
418
+ if (!Number.isInteger(retryCount) || retryCount < 0)
419
+ return;
420
+ if (retryCount === 1) {
415
421
  this.state.requestsRetries++;
416
- this.requestRetryHistogram[retryCount] ??= 0;
422
+ }
423
+ // Fill intermediate indices with 0 to prevent sparse arrays and JSON serialization null values
424
+ while (this.requestRetryHistogram.length <= retryCount) {
425
+ this.requestRetryHistogram.push(0);
426
+ }
417
427
  this.requestRetryHistogram[retryCount]++;
428
+ const previousRetryCount = retryCount - 1;
429
+ if (previousRetryCount > 0) {
430
+ while (this.requestRetryHistogram.length <= previousRetryCount) {
431
+ this.requestRetryHistogram.push(0);
432
+ }
433
+ // Decrement previous count only if it is strictly greater than 0
434
+ if (this.requestRetryHistogram[previousRetryCount] > 0) {
435
+ this.requestRetryHistogram[previousRetryCount]--;
436
+ }
437
+ }
438
+ }
439
+ #saveRetryCountForRequest(retryCount) {
440
+ if (!Number.isInteger(retryCount) || retryCount < 0)
441
+ return;
442
+ while (this.requestRetryHistogram.length <= retryCount) {
443
+ this.requestRetryHistogram.push(0);
444
+ }
445
+ if (retryCount === 0) {
446
+ this.requestRetryHistogram[0]++;
447
+ }
448
+ else if (this.requestRetryHistogram[retryCount] === 0) {
449
+ this.requestRetryHistogram[retryCount]++;
450
+ this.state.requestsRetries += retryCount;
451
+ }
418
452
  }
419
453
  /**
420
454
  * Persist internal state to the key value store.
@@ -66,3 +66,8 @@ export declare class RequestHandlerError extends Error {
66
66
  */
67
67
  export declare class NavigationSkippedError extends NonRetryableError {
68
68
  }
69
+ /**
70
+ * Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
71
+ * @internal
72
+ */
73
+ export declare function skippedNavigationRequest<R extends object>(request: R): R;
@@ -81,3 +81,28 @@ export class RequestHandlerError extends Error {
81
81
  */
82
82
  export class NavigationSkippedError extends NonRetryableError {
83
83
  }
84
+ /**
85
+ * Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
86
+ * @internal
87
+ */
88
+ export function skippedNavigationRequest(request) {
89
+ return new Proxy(request, {
90
+ get(target, propertyName) {
91
+ if (propertyName === 'loadedUrl') {
92
+ throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
93
+ }
94
+ // `target` as receiver and bound methods, or `#` members throw on the proxy
95
+ const value = Reflect.get(target, propertyName, target);
96
+ return typeof value === 'function' ? value.bind(target) : value;
97
+ },
98
+ // Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.
99
+ getOwnPropertyDescriptor(target, propertyName) {
100
+ const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);
101
+ if (propertyName === 'loadedUrl' && descriptor) {
102
+ return { ...descriptor, enumerable: false };
103
+ }
104
+ return descriptor;
105
+ },
106
+ set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
107
+ });
108
+ }
@@ -526,6 +526,7 @@ export class ThrottlingRequestManager {
526
526
  waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
527
527
  waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
528
528
  maxNewRequests: options.maxNewRequests,
529
+ log: this.#log,
529
530
  // Routing is the only thing this manager adds; the targets do the batching, validation and
530
531
  // deduplication themselves.
531
532
  processChunk: async (chunk) => {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.254",
3
+ "version": "4.0.0-beta.256",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.254",
46
- "@crawlee/http-client": "4.0.0-beta.254",
47
- "@crawlee/types": "4.0.0-beta.254",
48
- "@crawlee/utils": "4.0.0-beta.254",
45
+ "@crawlee/core": "4.0.0-beta.256",
46
+ "@crawlee/http-client": "4.0.0-beta.256",
47
+ "@crawlee/types": "4.0.0-beta.256",
48
+ "@crawlee/utils": "4.0.0-beta.256",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.254"
60
+ "@crawlee/impit-client": "4.0.0-beta.256"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "fd5e81ed20e33a43a281602f76be93169980bcd5"
69
+ "gitHead": "06cbfcc77b373a93f1a6a048e8115433db98c862"
70
70
  }