@crawlee/basic 4.0.0-beta.253 → 4.0.0-beta.255
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -654,8 +654,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
654
654
|
protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
|
|
655
655
|
/**
|
|
656
656
|
* Handles errors thrown by user provided requestHandler()
|
|
657
|
-
*
|
|
658
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
659
657
|
*/
|
|
660
658
|
private requestFunctionErrorHandler;
|
|
661
659
|
private handleFailedRequestHandler;
|
|
@@ -471,14 +471,14 @@ export class BasicCrawler {
|
|
|
471
471
|
// Navigation, the navigation hooks and the request handler are timed individually, but the
|
|
472
472
|
// phases between them are not, so a request could still get stuck indefinitely. This is the
|
|
473
473
|
// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
|
|
474
|
-
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source
|
|
474
|
+
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
|
|
475
475
|
}
|
|
476
476
|
catch (error) {
|
|
477
477
|
// The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
|
|
478
478
|
// take the normal error flow. Anything else is fatal.
|
|
479
479
|
if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
|
|
480
480
|
const unwrappedError = this.#unwrapError(error);
|
|
481
|
-
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext,
|
|
481
|
+
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
|
|
482
482
|
if (!this.#errorAbsolvesSession(unwrappedError)) {
|
|
483
483
|
crawlingContext.session?.markBad();
|
|
484
484
|
}
|
|
@@ -1500,13 +1500,14 @@ export class BasicCrawler {
|
|
|
1500
1500
|
}
|
|
1501
1501
|
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
1502
1502
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1503
|
-
async handleRequest(crawlingContext, requestSource
|
|
1503
|
+
async handleRequest(crawlingContext, requestSource) {
|
|
1504
1504
|
// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
|
|
1505
1505
|
// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
|
|
1506
1506
|
// execute (and re-report) on top of a request the crawler has already moved past.
|
|
1507
1507
|
if (crawlingContext[timeoutExpiredKey]?.()) {
|
|
1508
1508
|
return;
|
|
1509
1509
|
}
|
|
1510
|
+
const { request } = crawlingContext;
|
|
1510
1511
|
request.state = RequestState.REQUEST_HANDLER;
|
|
1511
1512
|
await this.runRequestHandler(crawlingContext);
|
|
1512
1513
|
// Commit *before* marking the request as handled, so a commit failure fails the request and
|
|
@@ -1524,7 +1525,8 @@ export class BasicCrawler {
|
|
|
1524
1525
|
* or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
|
|
1525
1526
|
* error handler has seen the context.
|
|
1526
1527
|
*/
|
|
1527
|
-
async #handleRequestFailure(rawError, crawlingContext, requestSource
|
|
1528
|
+
async #handleRequestFailure(rawError, crawlingContext, requestSource) {
|
|
1529
|
+
const { request } = crawlingContext;
|
|
1528
1530
|
// Roll back *before* any error handler runs - error handlers write to real storage precisely
|
|
1529
1531
|
// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
|
|
1530
1532
|
currentStorageTransaction()?.rollback();
|
|
@@ -1536,18 +1538,19 @@ export class BasicCrawler {
|
|
|
1536
1538
|
? await this.#trySkipRequest(unwrappedError, request, requestSource)
|
|
1537
1539
|
: unwrappedError;
|
|
1538
1540
|
if (err) {
|
|
1539
|
-
await this.#runErrorHandler(err, crawlingContext, requestSource
|
|
1541
|
+
await this.#runErrorHandler(err, crawlingContext, requestSource);
|
|
1540
1542
|
}
|
|
1541
1543
|
}
|
|
1542
1544
|
finally {
|
|
1543
1545
|
resumeRequestTimeout?.();
|
|
1544
1546
|
}
|
|
1545
1547
|
}
|
|
1546
|
-
async #runErrorHandler(err, context, requestSource
|
|
1548
|
+
async #runErrorHandler(err, context, requestSource) {
|
|
1549
|
+
const { request } = context;
|
|
1547
1550
|
const retryCountBefore = request.retryCount;
|
|
1548
1551
|
try {
|
|
1549
1552
|
request.state = RequestState.ERROR_HANDLER;
|
|
1550
|
-
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context,
|
|
1553
|
+
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1551
1554
|
// The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
|
|
1552
1555
|
if (request.state !== RequestState.SKIPPED) {
|
|
1553
1556
|
request.state = RequestState.DONE;
|
|
@@ -1656,11 +1659,10 @@ export class BasicCrawler {
|
|
|
1656
1659
|
}
|
|
1657
1660
|
/**
|
|
1658
1661
|
* Handles errors thrown by user provided requestHandler()
|
|
1659
|
-
*
|
|
1660
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
1661
1662
|
*/
|
|
1662
1663
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1663
|
-
async requestFunctionErrorHandler(error, crawlingContext,
|
|
1664
|
+
async requestFunctionErrorHandler(error, crawlingContext, source) {
|
|
1665
|
+
const { request } = crawlingContext;
|
|
1664
1666
|
if (error instanceof RequestThrottledError) {
|
|
1665
1667
|
// The domain told us to come back later, so the request was never really attempted. Put it back where
|
|
1666
1668
|
// it was, without recording a failure - it costs neither a retry nor session reputation.
|
|
@@ -1703,6 +1705,7 @@ export class BasicCrawler {
|
|
|
1703
1705
|
}
|
|
1704
1706
|
if (!request.noRetry) {
|
|
1705
1707
|
request.retryCount++;
|
|
1708
|
+
this.statistics.registerRetry?.(request.retryCount);
|
|
1706
1709
|
const { url, retryCount, id } = request;
|
|
1707
1710
|
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
|
|
1708
1711
|
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
|
|
@@ -50,6 +50,11 @@ export interface IStatistics<StateExtension extends object = {}> {
|
|
|
50
50
|
startCapturing(): Promise<void>;
|
|
51
51
|
/** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
|
|
52
52
|
stopCapturing(): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
55
|
+
* Optional - crawlers call this when a request is reclaimed back to the queue or list.
|
|
56
|
+
*/
|
|
57
|
+
registerRetry?(retryCount: number): void;
|
|
53
58
|
/**
|
|
54
59
|
* Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
|
|
55
60
|
* with no persistence of its own can omit it.
|
|
@@ -155,6 +160,11 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
|
|
|
155
160
|
* Stops logging and remove event listeners, then persist
|
|
156
161
|
*/
|
|
157
162
|
stopCapturing(): Promise<void>;
|
|
163
|
+
/**
|
|
164
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
165
|
+
* Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
|
|
166
|
+
*/
|
|
167
|
+
registerRetry(retryCount: number): void;
|
|
158
168
|
/**
|
|
159
169
|
* Persist internal state to the key value store.
|
|
160
170
|
*
|
|
@@ -410,11 +410,45 @@ export class Statistics {
|
|
|
410
410
|
this.state.crawlerFinishedAt = new Date();
|
|
411
411
|
await this.#recoverableState.teardown();
|
|
412
412
|
}
|
|
413
|
-
|
|
414
|
-
|
|
413
|
+
/**
|
|
414
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
415
|
+
* Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
|
|
416
|
+
*/
|
|
417
|
+
registerRetry(retryCount) {
|
|
418
|
+
if (!Number.isInteger(retryCount) || retryCount < 0)
|
|
419
|
+
return;
|
|
420
|
+
if (retryCount === 1) {
|
|
415
421
|
this.state.requestsRetries++;
|
|
416
|
-
|
|
422
|
+
}
|
|
423
|
+
// Fill intermediate indices with 0 to prevent sparse arrays and JSON serialization null values
|
|
424
|
+
while (this.requestRetryHistogram.length <= retryCount) {
|
|
425
|
+
this.requestRetryHistogram.push(0);
|
|
426
|
+
}
|
|
417
427
|
this.requestRetryHistogram[retryCount]++;
|
|
428
|
+
const previousRetryCount = retryCount - 1;
|
|
429
|
+
if (previousRetryCount > 0) {
|
|
430
|
+
while (this.requestRetryHistogram.length <= previousRetryCount) {
|
|
431
|
+
this.requestRetryHistogram.push(0);
|
|
432
|
+
}
|
|
433
|
+
// Decrement previous count only if it is strictly greater than 0
|
|
434
|
+
if (this.requestRetryHistogram[previousRetryCount] > 0) {
|
|
435
|
+
this.requestRetryHistogram[previousRetryCount]--;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
#saveRetryCountForRequest(retryCount) {
|
|
440
|
+
if (!Number.isInteger(retryCount) || retryCount < 0)
|
|
441
|
+
return;
|
|
442
|
+
while (this.requestRetryHistogram.length <= retryCount) {
|
|
443
|
+
this.requestRetryHistogram.push(0);
|
|
444
|
+
}
|
|
445
|
+
if (retryCount === 0) {
|
|
446
|
+
this.requestRetryHistogram[0]++;
|
|
447
|
+
}
|
|
448
|
+
else if (this.requestRetryHistogram[retryCount] === 0) {
|
|
449
|
+
this.requestRetryHistogram[retryCount]++;
|
|
450
|
+
this.state.requestsRetries += retryCount;
|
|
451
|
+
}
|
|
418
452
|
}
|
|
419
453
|
/**
|
|
420
454
|
* Persist internal state to the key value store.
|
package/internals/errors.d.ts
CHANGED
|
@@ -66,3 +66,8 @@ export declare class RequestHandlerError extends Error {
|
|
|
66
66
|
*/
|
|
67
67
|
export declare class NavigationSkippedError extends NonRetryableError {
|
|
68
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
71
|
+
* @internal
|
|
72
|
+
*/
|
|
73
|
+
export declare function skippedNavigationRequest<R extends object>(request: R): R;
|
package/internals/errors.js
CHANGED
|
@@ -81,3 +81,28 @@ export class RequestHandlerError extends Error {
|
|
|
81
81
|
*/
|
|
82
82
|
export class NavigationSkippedError extends NonRetryableError {
|
|
83
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
86
|
+
* @internal
|
|
87
|
+
*/
|
|
88
|
+
export function skippedNavigationRequest(request) {
|
|
89
|
+
return new Proxy(request, {
|
|
90
|
+
get(target, propertyName) {
|
|
91
|
+
if (propertyName === 'loadedUrl') {
|
|
92
|
+
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
93
|
+
}
|
|
94
|
+
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
95
|
+
const value = Reflect.get(target, propertyName, target);
|
|
96
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
97
|
+
},
|
|
98
|
+
// Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.
|
|
99
|
+
getOwnPropertyDescriptor(target, propertyName) {
|
|
100
|
+
const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);
|
|
101
|
+
if (propertyName === 'loadedUrl' && descriptor) {
|
|
102
|
+
return { ...descriptor, enumerable: false };
|
|
103
|
+
}
|
|
104
|
+
return descriptor;
|
|
105
|
+
},
|
|
106
|
+
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
107
|
+
});
|
|
108
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.255",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.255",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.255",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.255",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.255",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.255"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "e1b2f6c415f90b3eca1686a39a8af5272b54046e"
|
|
70
70
|
}
|