@crawlee/basic 4.0.0-beta.254 → 4.0.0-beta.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/basic-crawler.d.ts +0 -2
- package/internals/basic-crawler.js +14 -10
- package/internals/crawlers/statistics.d.ts +10 -0
- package/internals/crawlers/statistics.js +37 -3
- package/internals/errors.d.ts +5 -0
- package/internals/errors.js +25 -0
- package/internals/throttling-request-manager.js +1 -0
- package/package.json +7 -7
|
@@ -654,8 +654,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
654
654
|
protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
|
|
655
655
|
/**
|
|
656
656
|
* Handles errors thrown by user provided requestHandler()
|
|
657
|
-
*
|
|
658
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
659
657
|
*/
|
|
660
658
|
private requestFunctionErrorHandler;
|
|
661
659
|
private handleFailedRequestHandler;
|
|
@@ -471,14 +471,14 @@ export class BasicCrawler {
|
|
|
471
471
|
// Navigation, the navigation hooks and the request handler are timed individually, but the
|
|
472
472
|
// phases between them are not, so a request could still get stuck indefinitely. This is the
|
|
473
473
|
// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
|
|
474
|
-
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source
|
|
474
|
+
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
|
|
475
475
|
}
|
|
476
476
|
catch (error) {
|
|
477
477
|
// The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
|
|
478
478
|
// take the normal error flow. Anything else is fatal.
|
|
479
479
|
if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
|
|
480
480
|
const unwrappedError = this.#unwrapError(error);
|
|
481
|
-
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext,
|
|
481
|
+
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
|
|
482
482
|
if (!this.#errorAbsolvesSession(unwrappedError)) {
|
|
483
483
|
crawlingContext.session?.markBad();
|
|
484
484
|
}
|
|
@@ -1202,6 +1202,7 @@ export class BasicCrawler {
|
|
|
1202
1202
|
});
|
|
1203
1203
|
// Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
|
|
1204
1204
|
// when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
|
|
1205
|
+
// Background failures are already logged by `drainRequestBatches`.
|
|
1205
1206
|
void waitForAllRequestsToBeAdded.catch(() => { });
|
|
1206
1207
|
return { ...result, waitForAllRequestsToBeAdded };
|
|
1207
1208
|
}
|
|
@@ -1500,13 +1501,14 @@ export class BasicCrawler {
|
|
|
1500
1501
|
}
|
|
1501
1502
|
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
1502
1503
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1503
|
-
async handleRequest(crawlingContext, requestSource
|
|
1504
|
+
async handleRequest(crawlingContext, requestSource) {
|
|
1504
1505
|
// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
|
|
1505
1506
|
// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
|
|
1506
1507
|
// execute (and re-report) on top of a request the crawler has already moved past.
|
|
1507
1508
|
if (crawlingContext[timeoutExpiredKey]?.()) {
|
|
1508
1509
|
return;
|
|
1509
1510
|
}
|
|
1511
|
+
const { request } = crawlingContext;
|
|
1510
1512
|
request.state = RequestState.REQUEST_HANDLER;
|
|
1511
1513
|
await this.runRequestHandler(crawlingContext);
|
|
1512
1514
|
// Commit *before* marking the request as handled, so a commit failure fails the request and
|
|
@@ -1524,7 +1526,8 @@ export class BasicCrawler {
|
|
|
1524
1526
|
* or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
|
|
1525
1527
|
* error handler has seen the context.
|
|
1526
1528
|
*/
|
|
1527
|
-
async #handleRequestFailure(rawError, crawlingContext, requestSource
|
|
1529
|
+
async #handleRequestFailure(rawError, crawlingContext, requestSource) {
|
|
1530
|
+
const { request } = crawlingContext;
|
|
1528
1531
|
// Roll back *before* any error handler runs - error handlers write to real storage precisely
|
|
1529
1532
|
// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
|
|
1530
1533
|
currentStorageTransaction()?.rollback();
|
|
@@ -1536,18 +1539,19 @@ export class BasicCrawler {
|
|
|
1536
1539
|
? await this.#trySkipRequest(unwrappedError, request, requestSource)
|
|
1537
1540
|
: unwrappedError;
|
|
1538
1541
|
if (err) {
|
|
1539
|
-
await this.#runErrorHandler(err, crawlingContext, requestSource
|
|
1542
|
+
await this.#runErrorHandler(err, crawlingContext, requestSource);
|
|
1540
1543
|
}
|
|
1541
1544
|
}
|
|
1542
1545
|
finally {
|
|
1543
1546
|
resumeRequestTimeout?.();
|
|
1544
1547
|
}
|
|
1545
1548
|
}
|
|
1546
|
-
async #runErrorHandler(err, context, requestSource
|
|
1549
|
+
async #runErrorHandler(err, context, requestSource) {
|
|
1550
|
+
const { request } = context;
|
|
1547
1551
|
const retryCountBefore = request.retryCount;
|
|
1548
1552
|
try {
|
|
1549
1553
|
request.state = RequestState.ERROR_HANDLER;
|
|
1550
|
-
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context,
|
|
1554
|
+
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1551
1555
|
// The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
|
|
1552
1556
|
if (request.state !== RequestState.SKIPPED) {
|
|
1553
1557
|
request.state = RequestState.DONE;
|
|
@@ -1656,11 +1660,10 @@ export class BasicCrawler {
|
|
|
1656
1660
|
}
|
|
1657
1661
|
/**
|
|
1658
1662
|
* Handles errors thrown by user provided requestHandler()
|
|
1659
|
-
*
|
|
1660
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
1661
1663
|
*/
|
|
1662
1664
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1663
|
-
async requestFunctionErrorHandler(error, crawlingContext,
|
|
1665
|
+
async requestFunctionErrorHandler(error, crawlingContext, source) {
|
|
1666
|
+
const { request } = crawlingContext;
|
|
1664
1667
|
if (error instanceof RequestThrottledError) {
|
|
1665
1668
|
// The domain told us to come back later, so the request was never really attempted. Put it back where
|
|
1666
1669
|
// it was, without recording a failure - it costs neither a retry nor session reputation.
|
|
@@ -1703,6 +1706,7 @@ export class BasicCrawler {
|
|
|
1703
1706
|
}
|
|
1704
1707
|
if (!request.noRetry) {
|
|
1705
1708
|
request.retryCount++;
|
|
1709
|
+
this.statistics.registerRetry?.(request.retryCount);
|
|
1706
1710
|
const { url, retryCount, id } = request;
|
|
1707
1711
|
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
|
|
1708
1712
|
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
|
|
@@ -50,6 +50,11 @@ export interface IStatistics<StateExtension extends object = {}> {
|
|
|
50
50
|
startCapturing(): Promise<void>;
|
|
51
51
|
/** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
|
|
52
52
|
stopCapturing(): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
55
|
+
* Optional - crawlers call this when a request is reclaimed back to the queue or list.
|
|
56
|
+
*/
|
|
57
|
+
registerRetry?(retryCount: number): void;
|
|
53
58
|
/**
|
|
54
59
|
* Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
|
|
55
60
|
* with no persistence of its own can omit it.
|
|
@@ -155,6 +160,11 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
|
|
|
155
160
|
* Stops logging and remove event listeners, then persist
|
|
156
161
|
*/
|
|
157
162
|
stopCapturing(): Promise<void>;
|
|
163
|
+
/**
|
|
164
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
165
|
+
* Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
|
|
166
|
+
*/
|
|
167
|
+
registerRetry(retryCount: number): void;
|
|
158
168
|
/**
|
|
159
169
|
* Persist internal state to the key value store.
|
|
160
170
|
*
|
|
@@ -410,11 +410,45 @@ export class Statistics {
|
|
|
410
410
|
this.state.crawlerFinishedAt = new Date();
|
|
411
411
|
await this.#recoverableState.teardown();
|
|
412
412
|
}
|
|
413
|
-
|
|
414
|
-
|
|
413
|
+
/**
|
|
414
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
415
|
+
* Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
|
|
416
|
+
*/
|
|
417
|
+
registerRetry(retryCount) {
|
|
418
|
+
if (!Number.isInteger(retryCount) || retryCount < 0)
|
|
419
|
+
return;
|
|
420
|
+
if (retryCount === 1) {
|
|
415
421
|
this.state.requestsRetries++;
|
|
416
|
-
|
|
422
|
+
}
|
|
423
|
+
// Fill intermediate indices with 0 to prevent sparse arrays and JSON serialization null values
|
|
424
|
+
while (this.requestRetryHistogram.length <= retryCount) {
|
|
425
|
+
this.requestRetryHistogram.push(0);
|
|
426
|
+
}
|
|
417
427
|
this.requestRetryHistogram[retryCount]++;
|
|
428
|
+
const previousRetryCount = retryCount - 1;
|
|
429
|
+
if (previousRetryCount > 0) {
|
|
430
|
+
while (this.requestRetryHistogram.length <= previousRetryCount) {
|
|
431
|
+
this.requestRetryHistogram.push(0);
|
|
432
|
+
}
|
|
433
|
+
// Decrement previous count only if it is strictly greater than 0
|
|
434
|
+
if (this.requestRetryHistogram[previousRetryCount] > 0) {
|
|
435
|
+
this.requestRetryHistogram[previousRetryCount]--;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
#saveRetryCountForRequest(retryCount) {
|
|
440
|
+
if (!Number.isInteger(retryCount) || retryCount < 0)
|
|
441
|
+
return;
|
|
442
|
+
while (this.requestRetryHistogram.length <= retryCount) {
|
|
443
|
+
this.requestRetryHistogram.push(0);
|
|
444
|
+
}
|
|
445
|
+
if (retryCount === 0) {
|
|
446
|
+
this.requestRetryHistogram[0]++;
|
|
447
|
+
}
|
|
448
|
+
else if (this.requestRetryHistogram[retryCount] === 0) {
|
|
449
|
+
this.requestRetryHistogram[retryCount]++;
|
|
450
|
+
this.state.requestsRetries += retryCount;
|
|
451
|
+
}
|
|
418
452
|
}
|
|
419
453
|
/**
|
|
420
454
|
* Persist internal state to the key value store.
|
package/internals/errors.d.ts
CHANGED
|
@@ -66,3 +66,8 @@ export declare class RequestHandlerError extends Error {
|
|
|
66
66
|
*/
|
|
67
67
|
export declare class NavigationSkippedError extends NonRetryableError {
|
|
68
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
71
|
+
* @internal
|
|
72
|
+
*/
|
|
73
|
+
export declare function skippedNavigationRequest<R extends object>(request: R): R;
|
package/internals/errors.js
CHANGED
|
@@ -81,3 +81,28 @@ export class RequestHandlerError extends Error {
|
|
|
81
81
|
*/
|
|
82
82
|
export class NavigationSkippedError extends NonRetryableError {
|
|
83
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
86
|
+
* @internal
|
|
87
|
+
*/
|
|
88
|
+
export function skippedNavigationRequest(request) {
|
|
89
|
+
return new Proxy(request, {
|
|
90
|
+
get(target, propertyName) {
|
|
91
|
+
if (propertyName === 'loadedUrl') {
|
|
92
|
+
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
93
|
+
}
|
|
94
|
+
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
95
|
+
const value = Reflect.get(target, propertyName, target);
|
|
96
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
97
|
+
},
|
|
98
|
+
// Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.
|
|
99
|
+
getOwnPropertyDescriptor(target, propertyName) {
|
|
100
|
+
const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);
|
|
101
|
+
if (propertyName === 'loadedUrl' && descriptor) {
|
|
102
|
+
return { ...descriptor, enumerable: false };
|
|
103
|
+
}
|
|
104
|
+
return descriptor;
|
|
105
|
+
},
|
|
106
|
+
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
@@ -526,6 +526,7 @@ export class ThrottlingRequestManager {
|
|
|
526
526
|
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
|
|
527
527
|
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
|
|
528
528
|
maxNewRequests: options.maxNewRequests,
|
|
529
|
+
log: this.#log,
|
|
529
530
|
// Routing is the only thing this manager adds; the targets do the batching, validation and
|
|
530
531
|
// deduplication themselves.
|
|
531
532
|
processChunk: async (chunk) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.256",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.256",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.256",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.256",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.256",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.256"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "06cbfcc77b373a93f1a6a048e8115433db98c862"
|
|
70
70
|
}
|