@crawlee/basic 4.0.0-beta.29 → 4.0.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +11 -2
- package/index.js +11 -1
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +370 -374
- package/internals/basic-crawler.js +1416 -857
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +54 -0
- package/internals/request-timeout.js +98 -0
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.d.ts +4 -4
- package/internals/send-request.js +18 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +165 -0
- package/internals/sitemap-request-loader.js +492 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +23 -17
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/basic-crawler.d.ts.map +0 -1
- package/internals/basic-crawler.js.map +0 -1
- package/internals/send-request.d.ts.map +0 -1
- package/internals/send-request.js.map +0 -1
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { CriticalError, NonRetryableError } from '@crawlee/core';
|
|
2
|
+
import type { SkippedRequestReason } from './crawling-request.js';
|
|
3
|
+
/**
|
|
4
|
+
* @ignore
|
|
5
|
+
*/
|
|
6
|
+
export declare class MissingRouteError extends CriticalError {
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
10
|
+
*
|
|
11
|
+
* *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
|
|
12
|
+
*/
|
|
13
|
+
export declare class RetryRequestError extends Error {
|
|
14
|
+
constructor(message?: string);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Thrown when a domain has rate-limited us and the request should simply be attempted again later.
|
|
18
|
+
*
|
|
19
|
+
* The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
|
|
20
|
+
* nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
|
|
21
|
+
* the domain's backoff expires, so retries are paced rather than immediate.
|
|
22
|
+
*/
|
|
23
|
+
export declare class RequestThrottledError extends RetryRequestError {
|
|
24
|
+
constructor(message?: string);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
|
|
28
|
+
* abandoned rather than kept waiting.
|
|
29
|
+
*
|
|
30
|
+
* Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
|
|
31
|
+
* The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
|
|
32
|
+
* resumes them once the domain recovers.
|
|
33
|
+
*/
|
|
34
|
+
export declare class PersistentRateLimitError extends CriticalError {
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Thrown when a requested session is not found in the referenced SessionPool.
|
|
38
|
+
*/
|
|
39
|
+
export declare class MissingSessionError extends Error {
|
|
40
|
+
constructor(sessionId?: string);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
|
|
44
|
+
* Use {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
|
|
45
|
+
* @internal
|
|
46
|
+
*/
|
|
47
|
+
export declare class ContextPipelineInterruptedError extends Error {
|
|
48
|
+
readonly reason: SkippedRequestReason;
|
|
49
|
+
readonly skipMessage?: string | undefined;
|
|
50
|
+
constructor(reason: SkippedRequestReason, skipMessage?: string | undefined);
|
|
51
|
+
}
|
|
52
|
+
export declare class ContextPipelineCleanupError extends CriticalError {
|
|
53
|
+
constructor(error: unknown, options?: ErrorOptions);
|
|
54
|
+
}
|
|
55
|
+
/** Wraps a failure of a context pipeline middleware, telling it apart from a {@link RequestHandlerError}. */
|
|
56
|
+
export declare class ContextPipelineInitializationError extends Error {
|
|
57
|
+
constructor(error: unknown, options?: ErrorOptions);
|
|
58
|
+
}
|
|
59
|
+
/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
|
|
60
|
+
export declare class RequestHandlerError extends Error {
|
|
61
|
+
constructor(error: unknown, options?: ErrorOptions);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Thrown by crawlers when `skipNavigation` is used on a request.
|
|
65
|
+
* Subclasses can catch this error to skip their own navigation-dependent logic.
|
|
66
|
+
*/
|
|
67
|
+
export declare class NavigationSkippedError extends NonRetryableError {
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
71
|
+
* @internal
|
|
72
|
+
*/
|
|
73
|
+
export declare function skippedNavigationRequest<R extends object>(request: R): R;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { CriticalError, NonRetryableError } from '@crawlee/core';
|
|
2
|
+
/**
|
|
3
|
+
* @ignore
|
|
4
|
+
*/
|
|
5
|
+
export class MissingRouteError extends CriticalError {
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
9
|
+
*
|
|
10
|
+
* *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
|
|
11
|
+
*/
|
|
12
|
+
export class RetryRequestError extends Error {
|
|
13
|
+
constructor(message) {
|
|
14
|
+
super(message ?? "Request is being retried at the user's request");
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Thrown when a domain has rate-limited us and the request should simply be attempted again later.
|
|
19
|
+
*
|
|
20
|
+
* The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
|
|
21
|
+
* nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
|
|
22
|
+
* the domain's backoff expires, so retries are paced rather than immediate.
|
|
23
|
+
*/
|
|
24
|
+
export class RequestThrottledError extends RetryRequestError {
|
|
25
|
+
constructor(message) {
|
|
26
|
+
super(message ?? 'Request is being retried later because its domain is rate-limiting us');
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
|
|
31
|
+
* abandoned rather than kept waiting.
|
|
32
|
+
*
|
|
33
|
+
* Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
|
|
34
|
+
* The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
|
|
35
|
+
* resumes them once the domain recovers.
|
|
36
|
+
*/
|
|
37
|
+
export class PersistentRateLimitError extends CriticalError {
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Thrown when a requested session is not found in the referenced SessionPool.
|
|
41
|
+
*/
|
|
42
|
+
export class MissingSessionError extends Error {
|
|
43
|
+
constructor(sessionId) {
|
|
44
|
+
super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
|
|
49
|
+
* Use {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
|
|
50
|
+
* @internal
|
|
51
|
+
*/
|
|
52
|
+
export class ContextPipelineInterruptedError extends Error {
|
|
53
|
+
reason;
|
|
54
|
+
skipMessage;
|
|
55
|
+
constructor(reason, skipMessage) {
|
|
56
|
+
super(skipMessage ?? 'Request was skipped');
|
|
57
|
+
this.reason = reason;
|
|
58
|
+
this.skipMessage = skipMessage;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export class ContextPipelineCleanupError extends CriticalError {
|
|
62
|
+
constructor(error, options) {
|
|
63
|
+
super(undefined, { cause: error, ...options });
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** Wraps a failure of a context pipeline middleware, telling it apart from a {@link RequestHandlerError}. */
|
|
67
|
+
export class ContextPipelineInitializationError extends Error {
|
|
68
|
+
constructor(error, options) {
|
|
69
|
+
super(undefined, { cause: error, ...options });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
|
|
73
|
+
export class RequestHandlerError extends Error {
|
|
74
|
+
constructor(error, options) {
|
|
75
|
+
super(undefined, { cause: error, ...options });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Thrown by crawlers when `skipNavigation` is used on a request.
|
|
80
|
+
* Subclasses can catch this error to skip their own navigation-dependent logic.
|
|
81
|
+
*/
|
|
82
|
+
export class NavigationSkippedError extends NonRetryableError {
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
86
|
+
* @internal
|
|
87
|
+
*/
|
|
88
|
+
export function skippedNavigationRequest(request) {
|
|
89
|
+
return new Proxy(request, {
|
|
90
|
+
get(target, propertyName) {
|
|
91
|
+
if (propertyName === 'loadedUrl') {
|
|
92
|
+
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
93
|
+
}
|
|
94
|
+
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
95
|
+
const value = Reflect.get(target, propertyName, target);
|
|
96
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
97
|
+
},
|
|
98
|
+
// Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.
|
|
99
|
+
getOwnPropertyDescriptor(target, propertyName) {
|
|
100
|
+
const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);
|
|
101
|
+
if (propertyName === 'loadedUrl' && descriptor) {
|
|
102
|
+
return { ...descriptor, enumerable: false };
|
|
103
|
+
}
|
|
104
|
+
return descriptor;
|
|
105
|
+
},
|
|
106
|
+
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parses a `Retry-After` response header into a delay in milliseconds.
|
|
3
|
+
*
|
|
4
|
+
* The header holds either a non-negative number of seconds or an HTTP-date.
|
|
5
|
+
* See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
|
|
6
|
+
*
|
|
7
|
+
* @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
|
|
8
|
+
*/
|
|
9
|
+
export declare function parseRetryAfterHeader(value?: string | null): number | null;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parses a `Retry-After` response header into a delay in milliseconds.
|
|
3
|
+
*
|
|
4
|
+
* The header holds either a non-negative number of seconds or an HTTP-date.
|
|
5
|
+
* See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
|
|
6
|
+
*
|
|
7
|
+
* @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
|
|
8
|
+
*/
|
|
9
|
+
export function parseRetryAfterHeader(value) {
|
|
10
|
+
if (!value) {
|
|
11
|
+
return null;
|
|
12
|
+
}
|
|
13
|
+
const trimmed = value.trim();
|
|
14
|
+
// Per the spec this is a `delay-seconds`: digits only, so a negative or fractional value is not one.
|
|
15
|
+
if (/^\d+$/.test(trimmed)) {
|
|
16
|
+
// `Retry-After: 0` names no future deadline, same as an HTTP-date that has already passed. Reporting it
|
|
17
|
+
// as a zero delay would leave the domain unthrottled while still counting as a rate-limit event, so the
|
|
18
|
+
// caller would defer the request for free and re-send it immediately.
|
|
19
|
+
const delayMs = Number(trimmed) * 1000;
|
|
20
|
+
return delayMs > 0 ? delayMs : null;
|
|
21
|
+
}
|
|
22
|
+
const date = Date.parse(trimmed);
|
|
23
|
+
if (!Number.isNaN(date)) {
|
|
24
|
+
const delayMs = date - Date.now();
|
|
25
|
+
return delayMs > 0 ? delayMs : null;
|
|
26
|
+
}
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shared navigation-window deadline (epoch millis), stored on the in-flight context so the pre- and
|
|
3
|
+
* post-navigation hooks and the navigation itself all draw from one budget - and so `context.extendTimeout`
|
|
4
|
+
* can push the whole window, not just the current step.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
7
|
+
export declare const navigationDeadlineKey: unique symbol;
|
|
8
|
+
/**
|
|
9
|
+
* Lets `context.extendTimeout` push back the internal request timeout as well. That timeout is a bare timer
|
|
10
|
+
* rather than an `addTimeoutToPromise` frame, so `extendTimeout` from `@apify/timeout` cannot reach it on its own.
|
|
11
|
+
*/
|
|
12
|
+
export declare const extendTimeoutKey: unique symbol;
|
|
13
|
+
/**
|
|
14
|
+
* Returns `true` once the internal request timeout has fired for this request. It cannot cancel work stuck
|
|
15
|
+
* somewhere we do not control, but the phases we do run (e.g. the request handler) check this at their start and
|
|
16
|
+
* bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
|
|
17
|
+
*/
|
|
18
|
+
export declare const timeoutExpiredKey: unique symbol;
|
|
19
|
+
/**
|
|
20
|
+
* Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
|
|
21
|
+
* of its own, and must not be cut short (and handled again) by the request timeout.
|
|
22
|
+
*/
|
|
23
|
+
export declare const suspendTimeoutKey: unique symbol;
|
|
24
|
+
/** The slots the internal request timeout hangs on the in-flight crawling context. */
|
|
25
|
+
export interface RequestTimeoutContext {
|
|
26
|
+
[extendTimeoutKey]?: (extraMillis: number) => void;
|
|
27
|
+
[timeoutExpiredKey]?: () => boolean;
|
|
28
|
+
[suspendTimeoutKey]?: () => () => void;
|
|
29
|
+
[navigationDeadlineKey]?: number;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.
|
|
33
|
+
* @internal
|
|
34
|
+
*/
|
|
35
|
+
export declare function remainingNavigationWindowMillis(ctx: object, windowMillis: number): number;
|
|
36
|
+
/**
|
|
37
|
+
* Races `work` against a `timeoutMillis` timeout, so a request stuck in a phase that has no timeout of its own
|
|
38
|
+
* (anything that is not the navigation, a navigation hook or the request handler) still fails instead of stalling
|
|
39
|
+
* the crawler.
|
|
40
|
+
*
|
|
41
|
+
* This is a bare timer, not an `addTimeoutToPromise` frame: nested `addTimeoutToPromise` calls share one
|
|
42
|
+
* `AbortController`, so wrapping the request in one would let the request handler timing out abort this outer
|
|
43
|
+
* context too, cancelling the error handling that follows it.
|
|
44
|
+
*
|
|
45
|
+
* `Promise.race` attaches a handler to both sides, so a late rejection from `work` cannot go unhandled. The losing
|
|
46
|
+
* side is not cancelled - by definition it is stuck somewhere we do not control - but the phases we do run check
|
|
47
|
+
* {@link timeoutExpiredKey} at their start and bail, so a request whose timeout already fired does not carry
|
|
48
|
+
* on (e.g. run the handler) after the crawler has moved past it. `context.extendTimeout` pushes the deadline back
|
|
49
|
+
* via {@link extendTimeoutKey}.
|
|
50
|
+
*/
|
|
51
|
+
export declare function raceWithTimeout(context: RequestTimeoutContext, work: Promise<void>, { timeoutMillis, requestId }: {
|
|
52
|
+
timeoutMillis: number;
|
|
53
|
+
requestId?: string;
|
|
54
|
+
}): Promise<void>;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { TimeoutError } from '@apify/timeout';
|
|
2
|
+
/**
|
|
3
|
+
* The shared navigation-window deadline (epoch millis), stored on the in-flight context so the pre- and
|
|
4
|
+
* post-navigation hooks and the navigation itself all draw from one budget - and so `context.extendTimeout`
|
|
5
|
+
* can push the whole window, not just the current step.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export const navigationDeadlineKey = Symbol('navigationDeadline');
|
|
9
|
+
/**
|
|
10
|
+
* Lets `context.extendTimeout` push back the internal request timeout as well. That timeout is a bare timer
|
|
11
|
+
* rather than an `addTimeoutToPromise` frame, so `extendTimeout` from `@apify/timeout` cannot reach it on its own.
|
|
12
|
+
*/
|
|
13
|
+
export const extendTimeoutKey = Symbol('extendTimeout');
|
|
14
|
+
/**
|
|
15
|
+
* Returns `true` once the internal request timeout has fired for this request. It cannot cancel work stuck
|
|
16
|
+
* somewhere we do not control, but the phases we do run (e.g. the request handler) check this at their start and
|
|
17
|
+
* bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
|
|
18
|
+
*/
|
|
19
|
+
export const timeoutExpiredKey = Symbol('timeoutExpired');
|
|
20
|
+
/**
|
|
21
|
+
* Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
|
|
22
|
+
* of its own, and must not be cut short (and handled again) by the request timeout.
|
|
23
|
+
*/
|
|
24
|
+
export const suspendTimeoutKey = Symbol('suspendTimeout');
|
|
25
|
+
/**
|
|
26
|
+
* Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.
|
|
27
|
+
* @internal
|
|
28
|
+
*/
|
|
29
|
+
export function remainingNavigationWindowMillis(ctx, windowMillis) {
|
|
30
|
+
const store = ctx;
|
|
31
|
+
store[navigationDeadlineKey] ??= Date.now() + windowMillis;
|
|
32
|
+
return store[navigationDeadlineKey] - Date.now();
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Races `work` against a `timeoutMillis` timeout, so a request stuck in a phase that has no timeout of its own
|
|
36
|
+
* (anything that is not the navigation, a navigation hook or the request handler) still fails instead of stalling
|
|
37
|
+
* the crawler.
|
|
38
|
+
*
|
|
39
|
+
* This is a bare timer, not an `addTimeoutToPromise` frame: nested `addTimeoutToPromise` calls share one
|
|
40
|
+
* `AbortController`, so wrapping the request in one would let the request handler timing out abort this outer
|
|
41
|
+
* context too, cancelling the error handling that follows it.
|
|
42
|
+
*
|
|
43
|
+
* `Promise.race` attaches a handler to both sides, so a late rejection from `work` cannot go unhandled. The losing
|
|
44
|
+
* side is not cancelled - by definition it is stuck somewhere we do not control - but the phases we do run check
|
|
45
|
+
* {@link timeoutExpiredKey} at their start and bail, so a request whose timeout already fired does not carry
|
|
46
|
+
* on (e.g. run the handler) after the crawler has moved past it. `context.extendTimeout` pushes the deadline back
|
|
47
|
+
* via {@link extendTimeoutKey}.
|
|
48
|
+
*/
|
|
49
|
+
export async function raceWithTimeout(context, work, { timeoutMillis, requestId }) {
|
|
50
|
+
let timer;
|
|
51
|
+
let deadline = Date.now() + timeoutMillis;
|
|
52
|
+
let settled = false;
|
|
53
|
+
let firedByTimeout = false;
|
|
54
|
+
let suspendedAt;
|
|
55
|
+
const timeout = new Promise((_, reject) => {
|
|
56
|
+
const fire = () => {
|
|
57
|
+
settled = true;
|
|
58
|
+
firedByTimeout = true;
|
|
59
|
+
reject(new TimeoutError(`Request timed out after ${timeoutMillis / 1e3} seconds (${requestId}).`));
|
|
60
|
+
};
|
|
61
|
+
const schedule = () => {
|
|
62
|
+
clearTimeout(timer);
|
|
63
|
+
timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
|
|
64
|
+
};
|
|
65
|
+
timer = setTimeout(fire, timeoutMillis);
|
|
66
|
+
context[timeoutExpiredKey] = () => firedByTimeout;
|
|
67
|
+
context[extendTimeoutKey] = (extraMillis) => {
|
|
68
|
+
if (settled) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
deadline += extraMillis;
|
|
72
|
+
if (suspendedAt === undefined) {
|
|
73
|
+
schedule();
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
context[suspendTimeoutKey] = () => {
|
|
77
|
+
if (settled || suspendedAt !== undefined) {
|
|
78
|
+
return () => { };
|
|
79
|
+
}
|
|
80
|
+
clearTimeout(timer);
|
|
81
|
+
suspendedAt = Date.now();
|
|
82
|
+
return () => {
|
|
83
|
+
deadline += Date.now() - suspendedAt;
|
|
84
|
+
suspendedAt = undefined;
|
|
85
|
+
if (!settled) {
|
|
86
|
+
schedule();
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
};
|
|
90
|
+
});
|
|
91
|
+
try {
|
|
92
|
+
await Promise.race([work, timeout]);
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
settled = true;
|
|
96
|
+
clearTimeout(timer);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
import type { CrawlingRequest } from './crawling-request.js';
|
|
2
|
+
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
4
|
+
import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler-commons.js';
|
|
5
|
+
/**
|
|
6
|
+
* The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
|
|
7
|
+
* Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
|
|
8
|
+
* that falls through to the default handler (i.e. whose label has no route of its own).
|
|
9
|
+
*/
|
|
10
|
+
export declare const defaultRoute: unique symbol;
|
|
11
|
+
/**
|
|
12
|
+
* The crawling context received by a route handler, with `request.userData` narrowed to `UserData`, and
|
|
13
|
+
* `addRequests`/`enqueueLinks` typed according to the router's route map (`Routes`) so that enqueuing a
|
|
14
|
+
* request under a declared label requires the matching `userData` shape.
|
|
15
|
+
*/
|
|
16
|
+
export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
|
|
17
|
+
request: LoadedRequest<CrawlingRequest<UserData>>;
|
|
18
|
+
addRequests: TypedContextAddRequests<Routes>;
|
|
19
|
+
} & (Context extends {
|
|
20
|
+
enqueueLinks: infer EnqueueLinks;
|
|
21
|
+
} ? {
|
|
22
|
+
enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes>;
|
|
23
|
+
} : {});
|
|
24
|
+
/**
|
|
25
|
+
* A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
|
|
26
|
+
* validating that label's `request.userData`. Pass it to {@link Router.create} or a `createXRouter`
|
|
27
|
+
* factory to derive the per-label `request.userData` types *and* validate them at runtime. The optional
|
|
28
|
+
* {@link defaultRoute} key registers a schema for requests handled by the default route.
|
|
29
|
+
*/
|
|
30
|
+
export type RouteSchemas = Record<string, StandardSchemaV1> & {
|
|
31
|
+
[defaultRoute]?: StandardSchemaV1;
|
|
32
|
+
};
|
|
33
|
+
/** Infers a label's `userData` type from its schema, falling back to a plain {@link Dictionary}. */
|
|
34
|
+
type SchemaUserData<Schema extends StandardSchemaV1> = StandardSchemaV1.InferOutput<Schema> extends Dictionary ? StandardSchemaV1.InferOutput<Schema> : Dictionary;
|
|
35
|
+
/**
|
|
36
|
+
* Derives a route map (label → `userData` type) from a {@link RouteSchemas} map by inferring each schema's
|
|
37
|
+
* output type. Outputs that are not object-shaped fall back to a plain {@link Dictionary}. The
|
|
38
|
+
* {@link defaultRoute} schema is kept under its symbol key so {@link Router.addDefaultHandler} can pick it
|
|
39
|
+
* up; string labels (the ones {@link Router.addHandler} and the crawler-level typing accept) ignore it.
|
|
40
|
+
*/
|
|
41
|
+
export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
|
|
42
|
+
[Label in Extract<keyof Schemas, string>]: SchemaUserData<Schemas[Label]>;
|
|
43
|
+
} & (Schemas extends {
|
|
44
|
+
[defaultRoute]: StandardSchemaV1;
|
|
45
|
+
} ? {
|
|
46
|
+
[defaultRoute]: SchemaUserData<Schemas[typeof defaultRoute]>;
|
|
47
|
+
} : {});
|
|
48
|
+
/**
|
|
49
|
+
* The `userData` type of the default route: inferred from the {@link defaultRoute} schema when the route map
|
|
50
|
+
* carries one, otherwise the provided `Fallback`.
|
|
51
|
+
*/
|
|
52
|
+
export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
|
|
53
|
+
[defaultRoute]: infer DefaultUserData extends Dictionary;
|
|
54
|
+
} ? DefaultUserData : Fallback;
|
|
55
|
+
/**
|
|
56
|
+
* Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
|
|
57
|
+
* value. Throws a {@link RequestValidationError} when validation fails.
|
|
58
|
+
* @internal
|
|
59
|
+
*/
|
|
60
|
+
export declare function validateUserData(label: string | symbol, schema: StandardSchemaV1, userData: unknown): Promise<Dictionary>;
|
|
61
|
+
/**
|
|
62
|
+
* The set of labels accepted by {@link Router.addHandler}. When the router declares a concrete
|
|
63
|
+
* route map (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
|
|
64
|
+
* allowed — unknown labels become a compile-time error. When the map is left open (the default
|
|
65
|
+
* `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
|
|
66
|
+
*/
|
|
67
|
+
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
|
|
68
|
+
export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
|
|
69
|
+
(ctx: Context): Awaitable<void>;
|
|
70
|
+
}
|
|
71
|
+
export type GetUserDataFromRequest<T> = T extends CrawlingRequest<infer Y> ? Y : never;
|
|
72
|
+
/**
|
|
73
|
+
* Per-route overrides, passed as the last argument of {@link Router.addHandler|`addHandler`} and
|
|
74
|
+
* {@link Router.addDefaultHandler|`addDefaultHandler`}.
|
|
75
|
+
*/
|
|
76
|
+
export interface RouteOptions {
|
|
77
|
+
/**
|
|
78
|
+
* Overrides the crawler's `requestHandlerTimeoutSecs` for this route only. Useful when one kind of page
|
|
79
|
+
* needs markedly more time than the rest - a listing page behind an infinite scroll, say - and you do not
|
|
80
|
+
* want to raise the timeout for every other page to accommodate it.
|
|
81
|
+
*
|
|
82
|
+
* Applies only to this route's handler. The navigation and the navigation hooks keep their own timeouts.
|
|
83
|
+
*/
|
|
84
|
+
requestHandlerTimeoutSecs?: number;
|
|
85
|
+
}
|
|
86
|
+
export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
|
|
87
|
+
[Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
|
|
88
|
+
request: CrawlingRequest<Routes[Label]>;
|
|
89
|
+
}) => Awaitable<void>;
|
|
90
|
+
};
|
|
91
|
+
/**
|
|
92
|
+
* Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
|
|
93
|
+
*
|
|
94
|
+
* ```ts
|
|
95
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
96
|
+
*
|
|
97
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
98
|
+
*
|
|
99
|
+
* // we can also use factory methods for specific crawling contexts, the above equals to:
|
|
100
|
+
* // import { createCheerioRouter } from 'crawlee';
|
|
101
|
+
* // const router = createCheerioRouter();
|
|
102
|
+
*
|
|
103
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
104
|
+
* ctx.log.info('...');
|
|
105
|
+
* });
|
|
106
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
107
|
+
* ctx.log.info('...');
|
|
108
|
+
* });
|
|
109
|
+
*
|
|
110
|
+
* const crawler = new CheerioCrawler({
|
|
111
|
+
* requestHandler: router,
|
|
112
|
+
* });
|
|
113
|
+
* await crawler.run();
|
|
114
|
+
* ```
|
|
115
|
+
*
|
|
116
|
+
* Alternatively we can use the default router instance from crawler object:
|
|
117
|
+
*
|
|
118
|
+
* ```ts
|
|
119
|
+
* import { CheerioCrawler } from 'crawlee';
|
|
120
|
+
*
|
|
121
|
+
* const crawler = new CheerioCrawler();
|
|
122
|
+
*
|
|
123
|
+
* crawler.router.addHandler('label-a', async (ctx) => {
|
|
124
|
+
* ctx.log.info('...');
|
|
125
|
+
* });
|
|
126
|
+
* crawler.router.addDefaultHandler(async (ctx) => {
|
|
127
|
+
* ctx.log.info('...');
|
|
128
|
+
* });
|
|
129
|
+
*
|
|
130
|
+
* await crawler.run();
|
|
131
|
+
* ```
|
|
132
|
+
*
|
|
133
|
+
* For convenience, we can also define the routes right when creating the router:
|
|
134
|
+
*
|
|
135
|
+
* ```ts
|
|
136
|
+
* import { CheerioCrawler, createCheerioRouter } from 'crawlee';
|
|
137
|
+
* const crawler = new CheerioCrawler({
|
|
138
|
+
* requestHandler: createCheerioRouter({
|
|
139
|
+
* 'label-a': async (ctx) => { ... },
|
|
140
|
+
* 'label-b': async (ctx) => { ... },
|
|
141
|
+
* })},
|
|
142
|
+
* });
|
|
143
|
+
* await crawler.run();
|
|
144
|
+
* ```
|
|
145
|
+
*
|
|
146
|
+
* Middlewares are also supported via the `router.use` method. There can be multiple
|
|
147
|
+
* middlewares for a single router, they will be executed sequentially in the same
|
|
148
|
+
* order as they were registered.
|
|
149
|
+
*
|
|
150
|
+
* ```ts
|
|
151
|
+
* crawler.router.use(async (ctx) => {
|
|
152
|
+
* ctx.log.info('...');
|
|
153
|
+
* });
|
|
154
|
+
* ```
|
|
155
|
+
*
|
|
156
|
+
* To get `request.userData` typed per label, declare a route map and pass it as the second
|
|
157
|
+
* type argument. The label passed to {@link Router.addHandler} then drives the type of
|
|
158
|
+
* `request.userData`, and unknown labels are rejected at compile time:
|
|
159
|
+
*
|
|
160
|
+
* ```ts
|
|
161
|
+
* import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
|
|
162
|
+
*
|
|
163
|
+
* interface Routes {
|
|
164
|
+
* PRODUCT: { sku: string; price: number };
|
|
165
|
+
* CATEGORY: { categoryId: string };
|
|
166
|
+
* }
|
|
167
|
+
*
|
|
168
|
+
* const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
|
|
169
|
+
*
|
|
170
|
+
* router.addHandler('PRODUCT', async ({ request }) => {
|
|
171
|
+
* request.userData.sku; // string
|
|
172
|
+
* request.userData.price; // number
|
|
173
|
+
* });
|
|
174
|
+
*
|
|
175
|
+
* router.addHandler('TYPO', async () => {}); // compile error: not a known label
|
|
176
|
+
* ```
|
|
177
|
+
*
|
|
178
|
+
* Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
|
|
179
|
+
* `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
|
|
180
|
+
* added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
|
|
181
|
+
* throws a {@link RequestValidationError}.
|
|
182
|
+
*
|
|
183
|
+
* ```ts
|
|
184
|
+
* import { z } from 'zod';
|
|
185
|
+
* import { createCheerioRouter } from 'crawlee';
|
|
186
|
+
*
|
|
187
|
+
* const router = createCheerioRouter({
|
|
188
|
+
* PRODUCT: z.object({ sku: z.string(), price: z.number() }),
|
|
189
|
+
* CATEGORY: z.object({ categoryId: z.string() }),
|
|
190
|
+
* });
|
|
191
|
+
*
|
|
192
|
+
* router.addHandler('PRODUCT', async ({ request }) => {
|
|
193
|
+
* request.userData.price; // number, inferred from the schema and validated at runtime
|
|
194
|
+
* });
|
|
195
|
+
* ```
|
|
196
|
+
*
|
|
197
|
+
* A single route can take longer than the rest without raising the crawler-wide
|
|
198
|
+
* `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
|
|
199
|
+
*
|
|
200
|
+
* ```ts
|
|
201
|
+
* // LIST pages scroll through a lot of content, DETAIL pages are quick
|
|
202
|
+
* router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
|
|
203
|
+
* router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
|
|
204
|
+
* ```
|
|
205
|
+
*
|
|
206
|
+
* When the time a route needs is only apparent once it is already running, call
|
|
207
|
+
* {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
|
|
208
|
+
*
|
|
209
|
+
* ```ts
|
|
210
|
+
* router.addHandler('LIST', async ({ page, extendTimeout }) => {
|
|
211
|
+
* const pageCount = await countPages(page);
|
|
212
|
+
* extendTimeout(pageCount * 10); // ask for 10 more seconds per page
|
|
213
|
+
* await scrapeAllPages(page);
|
|
214
|
+
* });
|
|
215
|
+
* ```
|
|
216
|
+
*/
|
|
217
|
+
export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
|
|
218
|
+
#private;
|
|
219
|
+
/**
|
|
220
|
+
* use Router.create() instead!
|
|
221
|
+
* @ignore
|
|
222
|
+
*/
|
|
223
|
+
private constructor();
|
|
224
|
+
/**
|
|
225
|
+
* Registers new route handler for given label. When the router declares a route map, the
|
|
226
|
+
* `label` is restricted to the declared labels and `request.userData` is typed accordingly. Pass
|
|
227
|
+
* {@link RouteOptions|`options`} to give this route its own `requestHandlerTimeoutSecs`,
|
|
228
|
+
* overriding the crawler's default for requests with this label.
|
|
229
|
+
*/
|
|
230
|
+
addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>, options?: RouteOptions): void;
|
|
231
|
+
/**
|
|
232
|
+
* Registers new route handler for given label, explicitly typing `request.userData` via the
|
|
233
|
+
* `UserData` type argument. Useful when the router has no declared route map (the open default)
|
|
234
|
+
* and you want to type a single handler, or to register a handler under a `symbol` label.
|
|
235
|
+
*/
|
|
236
|
+
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
|
|
237
|
+
/**
|
|
238
|
+
* Registers default route handler. As a fallback it can receive any request (including labels not
|
|
239
|
+
* declared in the route map). When the router was created with a {@link defaultRoute} schema,
|
|
240
|
+
* `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
|
|
241
|
+
* Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
|
|
242
|
+
* default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
|
|
243
|
+
* through to it.
|
|
244
|
+
*/
|
|
245
|
+
addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
|
|
246
|
+
/**
|
|
247
|
+
* Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
|
|
248
|
+
* to validate `request.userData` when requests are added.
|
|
249
|
+
* @internal
|
|
250
|
+
*/
|
|
251
|
+
getSchema(label?: string | symbol): StandardSchemaV1 | undefined;
|
|
252
|
+
/**
|
|
253
|
+
* Registers a middleware that will be fired before the matching route handler.
|
|
254
|
+
* Multiple middlewares can be registered, they will be fired in the same order.
|
|
255
|
+
*/
|
|
256
|
+
use(middleware: (ctx: Context) => Awaitable<void>): void;
|
|
257
|
+
/**
|
|
258
|
+
* Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
|
|
259
|
+
* override it and the crawler's own timeout should apply. Falls back to the default route the same way
|
|
260
|
+
* {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
|
|
261
|
+
* the default route asked for. Used by the crawler; not meant to be called directly.
|
|
262
|
+
* @internal
|
|
263
|
+
*/
|
|
264
|
+
getTimeoutSecs(label?: string | symbol): number | undefined;
|
|
265
|
+
/**
|
|
266
|
+
* The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
|
|
267
|
+
* The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
|
|
268
|
+
* @internal
|
|
269
|
+
*/
|
|
270
|
+
getMaxTimeoutSecs(): number | undefined;
|
|
271
|
+
/**
|
|
272
|
+
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
273
|
+
*/
|
|
274
|
+
getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
|
|
275
|
+
/**
|
|
276
|
+
* Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
|
|
277
|
+
*
|
|
278
|
+
* ```ts
|
|
279
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
280
|
+
*
|
|
281
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
282
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
283
|
+
* ctx.log.info('...');
|
|
284
|
+
* });
|
|
285
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
286
|
+
* ctx.log.info('...');
|
|
287
|
+
* });
|
|
288
|
+
*
|
|
289
|
+
* const crawler = new CheerioCrawler({
|
|
290
|
+
* requestHandler: router,
|
|
291
|
+
* });
|
|
292
|
+
* await crawler.run();
|
|
293
|
+
* ```
|
|
294
|
+
*/
|
|
295
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
296
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
297
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
298
|
+
}
|
|
299
|
+
export {};
|