@crawlee/basic 4.0.0-beta.27 → 4.0.0-beta.270
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +11 -2
- package/index.js +11 -1
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +383 -369
- package/internals/basic-crawler.js +1468 -890
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +54 -0
- package/internals/request-timeout.js +98 -0
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.d.ts +4 -4
- package/internals/send-request.js +18 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +158 -0
- package/internals/sitemap-request-loader.js +427 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +23 -17
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/basic-crawler.d.ts.map +0 -1
- package/internals/basic-crawler.js.map +0 -1
- package/internals/send-request.d.ts.map +0 -1
- package/internals/send-request.js.map +0 -1
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
import { RequestValidationError } from '@crawlee/core';
|
|
2
|
+
import { MissingRouteError } from './errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
|
|
5
|
+
* Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
|
|
6
|
+
* that falls through to the default handler (i.e. whose label has no route of its own).
|
|
7
|
+
*/
|
|
8
|
+
export const defaultRoute = Symbol('default-route');
|
|
9
|
+
/** Whether a validation issue points at the top-level `label` key. */
|
|
10
|
+
function isLabelIssue(issue) {
|
|
11
|
+
if (issue.path?.length !== 1) {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
const [segment] = issue.path;
|
|
15
|
+
return (typeof segment === 'object' ? segment.key : segment) === 'label';
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
|
|
19
|
+
* value. Throws a {@link RequestValidationError} when validation fails.
|
|
20
|
+
* @internal
|
|
21
|
+
*/
|
|
22
|
+
export async function validateUserData(label, schema, userData) {
|
|
23
|
+
const { label: _label, ...rest } = (userData ?? {});
|
|
24
|
+
// `label` is a Crawlee-managed key that lives inside `userData`, so validating it is opt-in: we validate
|
|
25
|
+
// without it first, letting schemas that don't describe it pass (including `.strict()` ones). A schema that
|
|
26
|
+
// *does* declare `label` reports an issue for the now-missing key — so we re-validate with it included,
|
|
27
|
+
// honouring the declaration. Unlike `userData.__crawlee`, `label` is enumerable, so schemas do see it.
|
|
28
|
+
let result = await schema['~standard'].validate(rest);
|
|
29
|
+
if (result.issues?.some(isLabelIssue)) {
|
|
30
|
+
result = await schema['~standard'].validate({ ...rest, label });
|
|
31
|
+
}
|
|
32
|
+
if (result.issues) {
|
|
33
|
+
throw new RequestValidationError(label, result.issues);
|
|
34
|
+
}
|
|
35
|
+
// Restore the label so it survives schemas that strip undeclared keys.
|
|
36
|
+
return { ...result.value, label };
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
|
|
40
|
+
*
|
|
41
|
+
* ```ts
|
|
42
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
43
|
+
*
|
|
44
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
45
|
+
*
|
|
46
|
+
* // we can also use factory methods for specific crawling contexts, the above equals to:
|
|
47
|
+
* // import { createCheerioRouter } from 'crawlee';
|
|
48
|
+
* // const router = createCheerioRouter();
|
|
49
|
+
*
|
|
50
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
51
|
+
* ctx.log.info('...');
|
|
52
|
+
* });
|
|
53
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
54
|
+
* ctx.log.info('...');
|
|
55
|
+
* });
|
|
56
|
+
*
|
|
57
|
+
* const crawler = new CheerioCrawler({
|
|
58
|
+
* requestHandler: router,
|
|
59
|
+
* });
|
|
60
|
+
* await crawler.run();
|
|
61
|
+
* ```
|
|
62
|
+
*
|
|
63
|
+
* Alternatively we can use the default router instance from crawler object:
|
|
64
|
+
*
|
|
65
|
+
* ```ts
|
|
66
|
+
* import { CheerioCrawler } from 'crawlee';
|
|
67
|
+
*
|
|
68
|
+
* const crawler = new CheerioCrawler();
|
|
69
|
+
*
|
|
70
|
+
* crawler.router.addHandler('label-a', async (ctx) => {
|
|
71
|
+
* ctx.log.info('...');
|
|
72
|
+
* });
|
|
73
|
+
* crawler.router.addDefaultHandler(async (ctx) => {
|
|
74
|
+
* ctx.log.info('...');
|
|
75
|
+
* });
|
|
76
|
+
*
|
|
77
|
+
* await crawler.run();
|
|
78
|
+
* ```
|
|
79
|
+
*
|
|
80
|
+
* For convenience, we can also define the routes right when creating the router:
|
|
81
|
+
*
|
|
82
|
+
* ```ts
|
|
83
|
+
* import { CheerioCrawler, createCheerioRouter } from 'crawlee';
|
|
84
|
+
* const crawler = new CheerioCrawler({
|
|
85
|
+
* requestHandler: createCheerioRouter({
|
|
86
|
+
* 'label-a': async (ctx) => { ... },
|
|
87
|
+
* 'label-b': async (ctx) => { ... },
|
|
88
|
+
* })},
|
|
89
|
+
* });
|
|
90
|
+
* await crawler.run();
|
|
91
|
+
* ```
|
|
92
|
+
*
|
|
93
|
+
* Middlewares are also supported via the `router.use` method. There can be multiple
|
|
94
|
+
* middlewares for a single router, they will be executed sequentially in the same
|
|
95
|
+
* order as they were registered.
|
|
96
|
+
*
|
|
97
|
+
* ```ts
|
|
98
|
+
* crawler.router.use(async (ctx) => {
|
|
99
|
+
* ctx.log.info('...');
|
|
100
|
+
* });
|
|
101
|
+
* ```
|
|
102
|
+
*
|
|
103
|
+
* To get `request.userData` typed per label, declare a route map and pass it as the second
|
|
104
|
+
* type argument. The label passed to {@link Router.addHandler} then drives the type of
|
|
105
|
+
* `request.userData`, and unknown labels are rejected at compile time:
|
|
106
|
+
*
|
|
107
|
+
* ```ts
|
|
108
|
+
* import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
|
|
109
|
+
*
|
|
110
|
+
* interface Routes {
|
|
111
|
+
* PRODUCT: { sku: string; price: number };
|
|
112
|
+
* CATEGORY: { categoryId: string };
|
|
113
|
+
* }
|
|
114
|
+
*
|
|
115
|
+
* const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
|
|
116
|
+
*
|
|
117
|
+
* router.addHandler('PRODUCT', async ({ request }) => {
|
|
118
|
+
* request.userData.sku; // string
|
|
119
|
+
* request.userData.price; // number
|
|
120
|
+
* });
|
|
121
|
+
*
|
|
122
|
+
* router.addHandler('TYPO', async () => {}); // compile error: not a known label
|
|
123
|
+
* ```
|
|
124
|
+
*
|
|
125
|
+
* Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
|
|
126
|
+
* `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
|
|
127
|
+
* added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
|
|
128
|
+
* throws a {@link RequestValidationError}.
|
|
129
|
+
*
|
|
130
|
+
* ```ts
|
|
131
|
+
* import { z } from 'zod';
|
|
132
|
+
* import { createCheerioRouter } from 'crawlee';
|
|
133
|
+
*
|
|
134
|
+
* const router = createCheerioRouter({
|
|
135
|
+
* PRODUCT: z.object({ sku: z.string(), price: z.number() }),
|
|
136
|
+
* CATEGORY: z.object({ categoryId: z.string() }),
|
|
137
|
+
* });
|
|
138
|
+
*
|
|
139
|
+
* router.addHandler('PRODUCT', async ({ request }) => {
|
|
140
|
+
* request.userData.price; // number, inferred from the schema and validated at runtime
|
|
141
|
+
* });
|
|
142
|
+
* ```
|
|
143
|
+
*
|
|
144
|
+
* A single route can take longer than the rest without raising the crawler-wide
|
|
145
|
+
* `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
|
|
146
|
+
*
|
|
147
|
+
* ```ts
|
|
148
|
+
* // LIST pages scroll through a lot of content, DETAIL pages are quick
|
|
149
|
+
* router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
|
|
150
|
+
* router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
|
|
151
|
+
* ```
|
|
152
|
+
*
|
|
153
|
+
* When the time a route needs is only apparent once it is already running, call
|
|
154
|
+
* {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
|
|
155
|
+
*
|
|
156
|
+
* ```ts
|
|
157
|
+
* router.addHandler('LIST', async ({ page, extendTimeout }) => {
|
|
158
|
+
* const pageCount = await countPages(page);
|
|
159
|
+
* extendTimeout(pageCount * 10); // ask for 10 more seconds per page
|
|
160
|
+
* await scrapeAllPages(page);
|
|
161
|
+
* });
|
|
162
|
+
* ```
|
|
163
|
+
*/
|
|
164
|
+
export class Router {
|
|
165
|
+
#routes = new Map();
|
|
166
|
+
#schemas = new Map();
|
|
167
|
+
#timeouts = new Map();
|
|
168
|
+
#middlewares = [];
|
|
169
|
+
/**
|
|
170
|
+
* use Router.create() instead!
|
|
171
|
+
* @ignore
|
|
172
|
+
*/
|
|
173
|
+
constructor() { }
|
|
174
|
+
addHandler(label, handler, options = {}) {
|
|
175
|
+
this.#validate(label);
|
|
176
|
+
this.#routes.set(label, handler);
|
|
177
|
+
if (options.requestHandlerTimeoutSecs !== undefined) {
|
|
178
|
+
this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Registers default route handler. As a fallback it can receive any request (including labels not
|
|
183
|
+
* declared in the route map). When the router was created with a {@link defaultRoute} schema,
|
|
184
|
+
* `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
|
|
185
|
+
* Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
|
|
186
|
+
* default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
|
|
187
|
+
* through to it.
|
|
188
|
+
*/
|
|
189
|
+
addDefaultHandler(handler, options = {}) {
|
|
190
|
+
this.#validate(defaultRoute);
|
|
191
|
+
this.#routes.set(defaultRoute, handler);
|
|
192
|
+
if (options.requestHandlerTimeoutSecs !== undefined) {
|
|
193
|
+
this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
|
|
198
|
+
* to validate `request.userData` when requests are added.
|
|
199
|
+
* @internal
|
|
200
|
+
*/
|
|
201
|
+
getSchema(label) {
|
|
202
|
+
if (label != null) {
|
|
203
|
+
const schema = this.#schemas.get(label);
|
|
204
|
+
if (schema) {
|
|
205
|
+
return schema;
|
|
206
|
+
}
|
|
207
|
+
// A label with its own route is fully specified; don't fall back to the default-route schema.
|
|
208
|
+
if (this.#routes.has(label)) {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// Requests with no route of their own fall through to the default handler, so validate their
|
|
213
|
+
// `userData` against the default-route schema, if one was registered.
|
|
214
|
+
return this.#schemas.get(defaultRoute);
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Registers a middleware that will be fired before the matching route handler.
|
|
218
|
+
* Multiple middlewares can be registered, they will be fired in the same order.
|
|
219
|
+
*/
|
|
220
|
+
use(middleware) {
|
|
221
|
+
this.#middlewares.push(middleware);
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
|
|
225
|
+
* override it and the crawler's own timeout should apply. Falls back to the default route the same way
|
|
226
|
+
* {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
|
|
227
|
+
* the default route asked for. Used by the crawler; not meant to be called directly.
|
|
228
|
+
* @internal
|
|
229
|
+
*/
|
|
230
|
+
getTimeoutSecs(label) {
|
|
231
|
+
if (label && this.#routes.has(label)) {
|
|
232
|
+
return this.#timeouts.get(label);
|
|
233
|
+
}
|
|
234
|
+
return this.#timeouts.get(defaultRoute);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
|
|
238
|
+
* The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
|
|
239
|
+
* @internal
|
|
240
|
+
*/
|
|
241
|
+
getMaxTimeoutSecs() {
|
|
242
|
+
return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
246
|
+
*/
|
|
247
|
+
getHandler(label) {
|
|
248
|
+
if (label && this.#routes.has(label)) {
|
|
249
|
+
return this.#routes.get(label);
|
|
250
|
+
}
|
|
251
|
+
if (this.#routes.has(defaultRoute)) {
|
|
252
|
+
return this.#routes.get(defaultRoute);
|
|
253
|
+
}
|
|
254
|
+
throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
|
|
255
|
+
' You must set up a route for this label or a default route.' +
|
|
256
|
+
' Use `requestHandler`, `router.addHandler` or `router.addDefaultHandler`.');
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Validates `request.userData` against the schema registered for its label (if any), replacing it with
|
|
260
|
+
* the parsed value. Throws a {@link RequestValidationError} when validation fails.
|
|
261
|
+
*/
|
|
262
|
+
async #validateRequest(context) {
|
|
263
|
+
const label = context.request.label;
|
|
264
|
+
const schema = this.getSchema(label);
|
|
265
|
+
if (schema) {
|
|
266
|
+
context.request.userData = (await validateUserData(label, schema, context.request.userData));
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Throws when the label already exists in our registry.
|
|
271
|
+
*/
|
|
272
|
+
#validate(label) {
|
|
273
|
+
if (this.#routes.has(label)) {
|
|
274
|
+
const message = label === defaultRoute
|
|
275
|
+
? `Default route is already defined!`
|
|
276
|
+
: `Route for label '${String(label)}' is already defined!`;
|
|
277
|
+
throw new Error(message);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
static create(routesOrSchemas) {
|
|
281
|
+
const router = new Router();
|
|
282
|
+
const obj = Object.create(Function.prototype);
|
|
283
|
+
obj.addHandler = router.addHandler.bind(router);
|
|
284
|
+
obj.addDefaultHandler = router.addDefaultHandler.bind(router);
|
|
285
|
+
obj.getSchema = router.getSchema.bind(router);
|
|
286
|
+
obj.getHandler = router.getHandler.bind(router);
|
|
287
|
+
obj.getTimeoutSecs = router.getTimeoutSecs.bind(router);
|
|
288
|
+
obj.getMaxTimeoutSecs = router.getMaxTimeoutSecs.bind(router);
|
|
289
|
+
obj.use = router.use.bind(router);
|
|
290
|
+
// `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.
|
|
291
|
+
for (const label of Reflect.ownKeys(routesOrSchemas ?? {})) {
|
|
292
|
+
const value = routesOrSchemas[label];
|
|
293
|
+
if (typeof value === 'function') {
|
|
294
|
+
router.addHandler(label, value);
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
router.#schemas.set(label, value);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
const func = async function (context) {
|
|
301
|
+
const { url, loadedUrl, label } = context.request;
|
|
302
|
+
context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
|
|
303
|
+
await router.#validateRequest(context);
|
|
304
|
+
for (const middleware of router.#middlewares) {
|
|
305
|
+
await middleware(context);
|
|
306
|
+
}
|
|
307
|
+
return router.getHandler(label)(context);
|
|
308
|
+
};
|
|
309
|
+
Object.setPrototypeOf(func, obj);
|
|
310
|
+
return func;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { Request as CrawleeRequest
|
|
2
|
-
import type { BaseHttpClient
|
|
1
|
+
import type { Request as CrawleeRequest } from '@crawlee/core';
|
|
2
|
+
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
3
|
+
import type { HttpRequestOptions, ISession, SendRequestOptions } from '@crawlee/types';
|
|
3
4
|
/**
|
|
4
5
|
* Prepares a function to be used as the `sendRequest` context helper.
|
|
5
6
|
*
|
|
@@ -8,5 +9,4 @@ import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@cr
|
|
|
8
9
|
* @param originRequest The crawling request being processed.
|
|
9
10
|
* @param session The user session associated with the current request.
|
|
10
11
|
*/
|
|
11
|
-
export declare function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session:
|
|
12
|
-
//# sourceMappingURL=send-request.d.ts.map
|
|
12
|
+
export declare function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: ISession): (overrideRequest?: Partial<HttpRequestOptions>, overrideOptions?: SendRequestOptions) => Promise<Response>;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { processHttpRequestOptions } from '@crawlee/utils/internal';
|
|
1
3
|
/**
|
|
2
4
|
* Prepares a function to be used as the `sendRequest` context helper.
|
|
3
5
|
*
|
|
@@ -9,24 +11,32 @@
|
|
|
9
11
|
export function createSendRequest(httpClient, originRequest, session) {
|
|
10
12
|
return async (overrideRequest = {}, overrideOptions = {}) => {
|
|
11
13
|
const baseRequest = originRequest.intoFetchAPIRequest();
|
|
12
|
-
const mergedUrl = overrideRequest.url ?? baseRequest.url;
|
|
13
|
-
const mergedMethod = overrideRequest.method ?? baseRequest.method;
|
|
14
14
|
const mergedHeaders = new Headers(baseRequest.headers);
|
|
15
15
|
if (overrideRequest.headers) {
|
|
16
16
|
overrideRequest.headers.forEach((value, key) => {
|
|
17
17
|
mergedHeaders.set(key, value);
|
|
18
18
|
});
|
|
19
19
|
}
|
|
20
|
-
const
|
|
21
|
-
|
|
20
|
+
const opts = processHttpRequestOptions({
|
|
21
|
+
...overrideRequest,
|
|
22
|
+
url: overrideRequest.url ?? baseRequest.url,
|
|
23
|
+
method: overrideRequest.method ?? originRequest.method,
|
|
22
24
|
headers: mergedHeaders,
|
|
23
|
-
|
|
25
|
+
});
|
|
26
|
+
const request = new Request(opts.url, {
|
|
27
|
+
method: opts.method,
|
|
28
|
+
headers: opts.headers,
|
|
29
|
+
body: opts.body ? Readable.toWeb(opts.body) : baseRequest.body,
|
|
30
|
+
// Node-specific option to make the request body work with streams
|
|
31
|
+
duplex: 'half',
|
|
24
32
|
});
|
|
25
33
|
return httpClient.sendRequest(request, {
|
|
26
34
|
session,
|
|
27
|
-
cookieJar: overrideOptions
|
|
28
|
-
|
|
35
|
+
cookieJar: overrideOptions.cookieJar ?? opts.cookieJar ?? session.cookieJar,
|
|
36
|
+
proxyUrl: overrideOptions.proxyUrl ?? opts.proxyUrl,
|
|
37
|
+
timeoutMillis: overrideOptions.timeoutMillis ?? opts.timeout,
|
|
38
|
+
signal: overrideOptions.signal ?? opts.signal,
|
|
39
|
+
ignoreTlsErrors: overrideOptions.ignoreTlsErrors,
|
|
29
40
|
});
|
|
30
41
|
};
|
|
31
42
|
}
|
|
32
|
-
//# sourceMappingURL=send-request.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const BLOCKED_STATUS_CODES: readonly number[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const BLOCKED_STATUS_CODES = [401, 403, 429];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @ignore
|
|
3
|
+
*/
|
|
4
|
+
export class CookieParseError extends Error {
|
|
5
|
+
cookieHeaderString;
|
|
6
|
+
constructor(cookieHeaderString) {
|
|
7
|
+
super(`Could not parse cookie header string: ${cookieHeaderString}`);
|
|
8
|
+
this.cookieHeaderString = cookieHeaderString;
|
|
9
|
+
Error.captureStackTrace(this, CookieParseError);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { SessionFingerprint } from '@crawlee/types';
|
|
2
|
+
/**
|
|
3
|
+
* Build a {@link SessionFingerprint} whose `platform` matches the host OS
|
|
4
|
+
* and whose `browser`/`device` are randomized within the realistic profiles for
|
|
5
|
+
* that platform. Used by {@link SessionPool} as the default fingerprint for
|
|
6
|
+
* freshly created sessions; callers can override by passing their own
|
|
7
|
+
* `fingerprint` in `sessionOptions`.
|
|
8
|
+
*/
|
|
9
|
+
export declare function createDefaultSessionFingerprint(): SessionFingerprint;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* (browser, platform, device) combinations that correspond to setups people
|
|
3
|
+
* actually run. Anything not listed here (e.g. `edge` on android, `safari` on
|
|
4
|
+
* windows, `desktop` mobile platforms) is left out so a randomized default
|
|
5
|
+
* never produces a fingerprint that would itself be a giveaway.
|
|
6
|
+
*/
|
|
7
|
+
const PROFILES_BY_PLATFORM = [
|
|
8
|
+
{ browser: 'chrome', platform: 'windows', device: 'desktop' },
|
|
9
|
+
{ browser: 'firefox', platform: 'windows', device: 'desktop' },
|
|
10
|
+
{ browser: 'edge', platform: 'windows', device: 'desktop' },
|
|
11
|
+
{ browser: 'chrome', platform: 'macos', device: 'desktop' },
|
|
12
|
+
{ browser: 'firefox', platform: 'macos', device: 'desktop' },
|
|
13
|
+
{ browser: 'safari', platform: 'macos', device: 'desktop' },
|
|
14
|
+
{ browser: 'edge', platform: 'macos', device: 'desktop' },
|
|
15
|
+
{ browser: 'chrome', platform: 'linux', device: 'desktop' },
|
|
16
|
+
{ browser: 'firefox', platform: 'linux', device: 'desktop' },
|
|
17
|
+
{ browser: 'chrome', platform: 'android', device: 'mobile' },
|
|
18
|
+
{ browser: 'firefox', platform: 'android', device: 'mobile' },
|
|
19
|
+
{ browser: 'safari', platform: 'ios', device: 'mobile' },
|
|
20
|
+
];
|
|
21
|
+
/**
|
|
22
|
+
* Build a {@link SessionFingerprint} whose `platform` matches the host OS
|
|
23
|
+
* and whose `browser`/`device` are randomized within the realistic profiles for
|
|
24
|
+
* that platform. Used by {@link SessionPool} as the default fingerprint for
|
|
25
|
+
* freshly created sessions; callers can override by passing their own
|
|
26
|
+
* `fingerprint` in `sessionOptions`.
|
|
27
|
+
*/
|
|
28
|
+
export function createDefaultSessionFingerprint() {
|
|
29
|
+
return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
|
|
30
|
+
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
+
import type { ISessionPool, SessionState } from '@crawlee/types';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import type { PersistenceOptions } from '../crawlers/statistics.js';
|
|
5
|
+
import type { SessionOptions } from './session.js';
|
|
6
|
+
import { Session } from './session.js';
|
|
7
|
+
export type SessionReuseStrategy = 'random' | 'round-robin' | 'use-until-failure';
|
|
8
|
+
declare const persistedSessionPoolState: z.ZodObject<{
|
|
9
|
+
usableSessionsCount: z.ZodNumber;
|
|
10
|
+
retiredSessionsCount: z.ZodNumber;
|
|
11
|
+
sessions: z.ZodArray<z.ZodCustom<SessionState, SessionState>>;
|
|
12
|
+
}, z.core.$strip>;
|
|
13
|
+
/**
|
|
14
|
+
* The shape of the persisted {@link SessionPool} record, as returned by {@link SessionPool.getState}.
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export type SessionPoolPersistedState = z.infer<typeof persistedSessionPoolState>;
|
|
18
|
+
/**
|
|
19
|
+
* Factory user-function which creates customized {@link Session} instances.
|
|
20
|
+
*/
|
|
21
|
+
export interface CreateSession {
|
|
22
|
+
/**
|
|
23
|
+
* @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
|
|
24
|
+
*/
|
|
25
|
+
(options?: {
|
|
26
|
+
sessionOptions?: SessionOptions;
|
|
27
|
+
}): Session | Promise<Session>;
|
|
28
|
+
}
|
|
29
|
+
export interface SessionPoolOptions {
|
|
30
|
+
/**
|
|
31
|
+
* Unique identifier for this session pool instance. Used to generate a unique
|
|
32
|
+
* persistence key when `persistStateKey` is not provided.
|
|
33
|
+
* If not provided, an auto-incrementing ID is used.
|
|
34
|
+
*/
|
|
35
|
+
id?: string | number;
|
|
36
|
+
/**
|
|
37
|
+
* Maximum size of the pool. Indicates how many sessions are rotated.
|
|
38
|
+
* @default 1000
|
|
39
|
+
*/
|
|
40
|
+
maxPoolSize?: number;
|
|
41
|
+
/** The configuration options for {@link Session} instances. */
|
|
42
|
+
sessionOptions?: SessionOptions;
|
|
43
|
+
/** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
|
|
44
|
+
persistStateKeyValueStoreId?: string;
|
|
45
|
+
/**
|
|
46
|
+
* Session pool persists its state under this key in Key value store.
|
|
47
|
+
* @default CRAWLEE_SESSION_POOL_STATE_{id}
|
|
48
|
+
*/
|
|
49
|
+
persistStateKey?: string;
|
|
50
|
+
/**
|
|
51
|
+
* Custom function that should return a `Session` instance, or a promise resolving to such instance.
|
|
52
|
+
* Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
|
|
53
|
+
* already merged from the pool-wide defaults and the per-call overrides.
|
|
54
|
+
*/
|
|
55
|
+
createSessionFunction?: CreateSession;
|
|
56
|
+
/**
|
|
57
|
+
* Strategy for picking sessions from the pool.
|
|
58
|
+
* - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
|
|
59
|
+
* - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
|
|
60
|
+
* - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
|
|
61
|
+
* @default 'random'
|
|
62
|
+
*/
|
|
63
|
+
sessionReuseStrategy?: SessionReuseStrategy;
|
|
64
|
+
/** @internal */
|
|
65
|
+
log?: CrawleeLogger;
|
|
66
|
+
/**
|
|
67
|
+
* Control how and when to persist the state of the session pool.
|
|
68
|
+
*/
|
|
69
|
+
persistenceOptions?: PersistenceOptions;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Handles the rotation, creation and persistence of user-like sessions.
|
|
73
|
+
* Creates a pool of {@link Session} instances, that are randomly rotated.
|
|
74
|
+
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
75
|
+
* Learn more in the {@doclink guides/session-management | Session management guide}.
|
|
76
|
+
*
|
|
77
|
+
* Session pool is already integrated into crawlers and is always active.
|
|
78
|
+
* All public methods are lazy-initialized — the pool initializes itself on first use.
|
|
79
|
+
*
|
|
80
|
+
* You can configure the pool with many options. See the {@link SessionPoolOptions}.
|
|
81
|
+
* Session pool is by default persisted in default {@link KeyValueStore}.
|
|
82
|
+
* If you want to have one pool for all runs you have to specify
|
|
83
|
+
* {@link SessionPoolOptions.persistStateKeyValueStoreId}.
|
|
84
|
+
*
|
|
85
|
+
* **Advanced usage:**
|
|
86
|
+
*
|
|
87
|
+
* ```javascript
|
|
88
|
+
* const sessionPool = new SessionPool({
|
|
89
|
+
* maxPoolSize: 25,
|
|
90
|
+
* sessionOptions:{
|
|
91
|
+
* maxAgeSecs: 10,
|
|
92
|
+
* maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
|
|
93
|
+
* },
|
|
94
|
+
* persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
|
|
95
|
+
* persistStateKey: 'my-session-pool',
|
|
96
|
+
* });
|
|
97
|
+
*
|
|
98
|
+
* // Get random session from the pool
|
|
99
|
+
* const session1 = await sessionPool.getSession();
|
|
100
|
+
* const session2 = await sessionPool.getSession();
|
|
101
|
+
* const session3 = await sessionPool.getSession();
|
|
102
|
+
*
|
|
103
|
+
* // Now you can mark the session either failed or successful
|
|
104
|
+
*
|
|
105
|
+
* // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
|
|
106
|
+
* session1.markBad()
|
|
107
|
+
*
|
|
108
|
+
* // Marks as successful.
|
|
109
|
+
* session2.markGood()
|
|
110
|
+
*
|
|
111
|
+
* // Retires session -> session is removed from the pool
|
|
112
|
+
* session3.retire()
|
|
113
|
+
*
|
|
114
|
+
* ```
|
|
115
|
+
*
|
|
116
|
+
* **Default session allocation flow:*
|
|
117
|
+
* 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
|
|
118
|
+
* 2. Blocked/retired sessions stay in the pool but are never provided to the user
|
|
119
|
+
* 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
|
|
120
|
+
* 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
|
|
121
|
+
*
|
|
122
|
+
* @category Scaling
|
|
123
|
+
*/
|
|
124
|
+
export declare class SessionPool implements ISessionPool {
|
|
125
|
+
#private;
|
|
126
|
+
readonly id: string;
|
|
127
|
+
constructor(options?: SessionPoolOptions);
|
|
128
|
+
/**
|
|
129
|
+
* Gets count of usable sessions in the pool.
|
|
130
|
+
*/
|
|
131
|
+
usableSessionsCount(): Promise<number>;
|
|
132
|
+
/**
|
|
133
|
+
* Gets count of retired sessions in the pool.
|
|
134
|
+
*/
|
|
135
|
+
retiredSessionsCount(): Promise<number>;
|
|
136
|
+
/**
|
|
137
|
+
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
138
|
+
* but this allows you to add more sessions once the max pool size is reached.
|
|
139
|
+
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
140
|
+
*
|
|
141
|
+
* If the pool already contains a session with the same id, it is replaced when it is no longer usable
|
|
142
|
+
* (e.g. retired), and an error is thrown otherwise.
|
|
143
|
+
* @param [options] The configuration options for the session being added to the session pool.
|
|
144
|
+
*/
|
|
145
|
+
addSession(options?: Session | SessionOptions): Promise<void>;
|
|
146
|
+
/**
|
|
147
|
+
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
148
|
+
* but this allows you to add more sessions once the max pool size is reached.
|
|
149
|
+
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
150
|
+
* @param [options] The configuration options for the session being added to the session pool.
|
|
151
|
+
*/
|
|
152
|
+
newSession(sessionOptions?: SessionOptions): Promise<Session>;
|
|
153
|
+
/**
|
|
154
|
+
* Gets session.
|
|
155
|
+
* If there is space for new session, it creates and returns new session.
|
|
156
|
+
* If the session pool is full, it picks a session from the pool,
|
|
157
|
+
* If the picked session is usable it is returned, otherwise it creates and returns a new one.
|
|
158
|
+
* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
|
|
159
|
+
*/
|
|
160
|
+
getSession(sessionId?: string): Promise<Session | undefined>;
|
|
161
|
+
/**
|
|
162
|
+
* Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
|
|
163
|
+
*
|
|
164
|
+
* The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
|
|
165
|
+
*/
|
|
166
|
+
reset(): void;
|
|
167
|
+
/**
|
|
168
|
+
* Clears the persisted pool record, leaving the in-memory sessions alone.
|
|
169
|
+
*
|
|
170
|
+
* Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
|
|
171
|
+
* the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
|
|
172
|
+
* to discard the sessions themselves. A no-op if persistence is disabled.
|
|
173
|
+
*/
|
|
174
|
+
resetStore(): Promise<void>;
|
|
175
|
+
/**
|
|
176
|
+
* Returns an object representing the internal state of the `SessionPool` instance.
|
|
177
|
+
* @internal
|
|
178
|
+
*/
|
|
179
|
+
getState(): Promise<SessionPoolPersistedState>;
|
|
180
|
+
/**
|
|
181
|
+
* Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
|
|
182
|
+
* The state is persisted automatically in regular intervals.
|
|
183
|
+
*/
|
|
184
|
+
persistState(): Promise<void>;
|
|
185
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
186
|
+
/**
|
|
187
|
+
* Stops the periodic state persistence and persists the state one last time.
|
|
188
|
+
* This function should be called after you are done with using the `SessionPool` instance.
|
|
189
|
+
* Using the pool again afterwards initializes it anew.
|
|
190
|
+
*/
|
|
191
|
+
teardown(): Promise<void>;
|
|
192
|
+
}
|
|
193
|
+
export {};
|