@crawlee/core 4.0.0-beta.98 → 4.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +3 -21
- package/autoscaling/autoscaled_pool.js +105 -97
- package/autoscaling/concurrency_system.d.ts +7 -22
- package/autoscaling/concurrency_system.js +112 -100
- package/autoscaling/cpu_load_signal.d.ts +1 -2
- package/autoscaling/cpu_load_signal.js +11 -10
- package/autoscaling/event_loop_load_signal.d.ts +1 -4
- package/autoscaling/event_loop_load_signal.js +18 -18
- package/autoscaling/index.d.ts +1 -1
- package/autoscaling/index.js +1 -1
- package/autoscaling/load_signal.d.ts +8 -7
- package/autoscaling/load_signal.js +14 -12
- package/autoscaling/memory_load_signal.d.ts +3 -12
- package/autoscaling/memory_load_signal.js +41 -41
- package/autoscaling/snapshotter.d.ts +7 -10
- package/autoscaling/snapshotter.js +18 -18
- package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts} +14 -17
- package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js} +27 -27
- package/autoscaling/system_status.d.ts +9 -11
- package/autoscaling/system_status.js +13 -13
- package/configuration.d.ts +16 -16
- package/configuration.js +6 -6
- package/crawlers/context_pipeline.js +6 -6
- package/crawlers/crawler_commons.d.ts +14 -116
- package/crawlers/crawler_commons.js +1 -107
- package/crawlers/error_snapshotter.d.ts +2 -4
- package/crawlers/error_snapshotter.js +5 -5
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.js +0 -1
- package/crawlers/statistics.d.ts +164 -45
- package/crawlers/statistics.js +339 -153
- package/debug.js +4 -4
- package/enqueue_links/enqueue_links.d.ts +51 -159
- package/enqueue_links/enqueue_links.js +38 -231
- package/enqueue_links/shared.d.ts +34 -24
- package/enqueue_links/shared.js +54 -36
- package/errors.d.ts +43 -12
- package/errors.js +48 -9
- package/events/event_manager.d.ts +2 -2
- package/events/event_manager.js +7 -6
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +5 -5
- package/http.d.ts +9 -0
- package/http.js +28 -0
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/log.js +5 -1
- package/memory-storage/memory-storage.d.ts +3 -11
- package/memory-storage/memory-storage.js +42 -48
- package/memory-storage/resource-clients/dataset.d.ts +1 -1
- package/memory-storage/resource-clients/dataset.js +8 -13
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
- package/memory-storage/resource-clients/key-value-store.js +36 -38
- package/memory-storage/resource-clients/request-queue.d.ts +4 -23
- package/memory-storage/resource-clients/request-queue.js +68 -80
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +8 -9
- package/proxy_configuration.d.ts +1 -3
- package/proxy_configuration.js +18 -14
- package/recoverable_state.d.ts +80 -45
- package/recoverable_state.js +163 -75
- package/request.d.ts +3 -4
- package/request.js +57 -44
- package/router.d.ts +6 -10
- package/router.js +23 -23
- package/serialization.js +14 -13
- package/service_locator.d.ts +19 -10
- package/service_locator.js +57 -48
- package/session_pool/session.d.ts +3 -14
- package/session_pool/session.js +76 -74
- package/session_pool/session_pool.d.ts +6 -12
- package/session_pool/session_pool.js +85 -77
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +12 -1
- package/storages/dataset.js +133 -32
- package/storages/index.d.ts +2 -1
- package/storages/index.js +2 -1
- package/storages/key_value_store.d.ts +19 -4
- package/storages/key_value_store.js +204 -75
- package/storages/request_dedup_cache.d.ts +1 -2
- package/storages/request_dedup_cache.js +9 -9
- package/storages/request_list.d.ts +4 -23
- package/storages/request_list.js +103 -96
- package/storages/request_loader.d.ts +5 -0
- package/storages/request_manager_tandem.d.ts +1 -10
- package/storages/request_manager_tandem.js +27 -27
- package/storages/request_queue.d.ts +21 -21
- package/storages/request_queue.js +318 -177
- package/storages/sitemap_request_loader.d.ts +22 -70
- package/storages/sitemap_request_loader.js +122 -116
- package/storages/storage_instance_manager.d.ts +1 -4
- package/storages/storage_instance_manager.js +17 -19
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/throttling_request_manager.d.ts +239 -0
- package/storages/throttling_request_manager.js +646 -0
- package/storages/transaction.d.ts +252 -0
- package/storages/transaction.js +251 -0
- package/storages/utils.d.ts +6 -3
- package/storages/utils.js +11 -2
- package/system-info/runtime.js +7 -7
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +22 -25
- package/validators.js +13 -25
- package/storages/access_checking.d.ts +0 -12
- package/storages/access_checking.js +0 -17
package/request.js
CHANGED
|
@@ -1,32 +1,13 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
|
|
5
5
|
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
uniqueKey: ow.optional.string,
|
|
12
|
-
method: ow.optional.string,
|
|
13
|
-
payload: ow.optional.any(ow.string, ow.uint8Array),
|
|
14
|
-
noRetry: ow.optional.boolean,
|
|
15
|
-
retryCount: ow.optional.number,
|
|
16
|
-
sessionId: ow.optional.string,
|
|
17
|
-
maxRetries: ow.optional.number,
|
|
18
|
-
errorMessages: ow.optional.array.ofType(ow.string),
|
|
19
|
-
headers: ow.optional.object,
|
|
20
|
-
userData: ow.optional.object,
|
|
21
|
-
label: ow.optional.string,
|
|
22
|
-
handledAt: ow.optional.any(ow.string.date, ow.date),
|
|
23
|
-
keepUrlFragment: ow.optional.boolean,
|
|
24
|
-
useExtendedUniqueKey: ow.optional.boolean,
|
|
25
|
-
alwaysEnqueue: ow.optional.boolean,
|
|
26
|
-
skipNavigation: ow.optional.boolean,
|
|
27
|
-
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
28
|
-
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
29
|
-
};
|
|
7
|
+
import { parseArgument, schemas } from './validators.js';
|
|
8
|
+
const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
|
|
9
|
+
message: 'Invalid input: expected a date string',
|
|
10
|
+
});
|
|
30
11
|
export var RequestState;
|
|
31
12
|
(function (RequestState) {
|
|
32
13
|
RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
|
|
@@ -38,6 +19,34 @@ export var RequestState;
|
|
|
38
19
|
RequestState[RequestState["ERROR"] = 6] = "ERROR";
|
|
39
20
|
RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
|
|
40
21
|
})(RequestState || (RequestState = {}));
|
|
22
|
+
const requestUrlSchema = z.object({ url: z.string() });
|
|
23
|
+
// new properties on the Request object breaks serialization
|
|
24
|
+
const requestOptionalSchemaShapes = {
|
|
25
|
+
id: z.string().optional(),
|
|
26
|
+
loadedUrl: z.url().optional(),
|
|
27
|
+
uniqueKey: z.string().optional(),
|
|
28
|
+
method: z.string().optional(),
|
|
29
|
+
payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
|
|
30
|
+
noRetry: z.boolean().optional(),
|
|
31
|
+
retryCount: schemas.anyNumber.optional(),
|
|
32
|
+
sessionId: z.string().optional(),
|
|
33
|
+
maxRetries: schemas.anyNumber.optional(),
|
|
34
|
+
errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
|
|
35
|
+
headers: z.looseObject({}).optional(),
|
|
36
|
+
userData: z.looseObject({}).optional(),
|
|
37
|
+
label: z.string().optional(),
|
|
38
|
+
handledAt: z.union([dateString, z.date()]).optional(),
|
|
39
|
+
keepUrlFragment: z.boolean().optional(),
|
|
40
|
+
useExtendedUniqueKey: z.boolean().optional(),
|
|
41
|
+
alwaysEnqueue: z.boolean().optional(),
|
|
42
|
+
skipNavigation: z.boolean().optional(),
|
|
43
|
+
crawlDepth: schemas.anyNumber
|
|
44
|
+
.refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
|
|
45
|
+
.optional(),
|
|
46
|
+
state: z.enum(RequestState).optional(),
|
|
47
|
+
};
|
|
48
|
+
// Each schema is wrapped in a single-key object so validation errors carry the property name.
|
|
49
|
+
const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
|
|
41
50
|
/**
|
|
42
51
|
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
43
52
|
* The `Request` object also stores information about errors that occurred during processing of the request.
|
|
@@ -101,7 +110,7 @@ class CrawleeRequest {
|
|
|
101
110
|
/** Object with HTTP headers. Key is header name, value is the value. */
|
|
102
111
|
headers;
|
|
103
112
|
/** Private store for the custom user data assigned to the request. */
|
|
104
|
-
|
|
113
|
+
#userData = {};
|
|
105
114
|
/**
|
|
106
115
|
* Custom user data assigned to the request.
|
|
107
116
|
*
|
|
@@ -118,22 +127,26 @@ class CrawleeRequest {
|
|
|
118
127
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
119
128
|
*/
|
|
120
129
|
constructor(options) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
130
|
+
// A bare URL is a common slip — point at the object form instead of a generic type error.
|
|
131
|
+
if (typeof options === 'string') {
|
|
132
|
+
throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
|
|
133
|
+
'Did you mean `new Request({ url })`?');
|
|
134
|
+
}
|
|
135
|
+
parseArgument(options, schemas.anyObject, 'RequestOptions');
|
|
136
|
+
parseArgument(options, requestUrlSchema, 'RequestOptions');
|
|
137
|
+
// Full-shape validation is slow, because it checks all predicates
|
|
124
138
|
// even if the validated object has only 1 property.
|
|
125
139
|
// This custom validation loop iterates only over existing
|
|
126
140
|
// properties and speeds up the validation cca 3-fold.
|
|
127
|
-
// See https://github.com/sindresorhus/ow/issues/193
|
|
128
141
|
keys(options).forEach((prop) => {
|
|
129
142
|
// skip url, because it is validated above
|
|
130
143
|
if (prop === 'url') {
|
|
131
144
|
return;
|
|
132
145
|
}
|
|
133
|
-
const
|
|
146
|
+
const schema = requestOptionalSchemas[prop];
|
|
134
147
|
const value = options[prop];
|
|
135
|
-
if (
|
|
136
|
-
|
|
148
|
+
if (schema) {
|
|
149
|
+
parseArgument({ [prop]: value }, schema, 'RequestOptions');
|
|
137
150
|
}
|
|
138
151
|
});
|
|
139
152
|
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
@@ -167,36 +180,36 @@ class CrawleeRequest {
|
|
|
167
180
|
if (label) {
|
|
168
181
|
userData.label = label;
|
|
169
182
|
}
|
|
183
|
+
// Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
|
|
184
|
+
// bag is non-enumerable, so the spread alone would silently drop the internal state
|
|
185
|
+
// (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
|
|
186
|
+
this.#userData = { __crawlee: userData.__crawlee ?? {}, ...userData };
|
|
187
|
+
// `userData` must stay an enumerable own accessor — serialization in the storages relies on it
|
|
170
188
|
Object.defineProperties(this, {
|
|
171
|
-
_userData: {
|
|
172
|
-
value: { __crawlee: {}, ...userData },
|
|
173
|
-
enumerable: false,
|
|
174
|
-
writable: true,
|
|
175
|
-
},
|
|
176
189
|
userData: {
|
|
177
|
-
get: () => this
|
|
190
|
+
get: () => this.#userData,
|
|
178
191
|
set: (value) => {
|
|
179
192
|
Object.defineProperties(value, {
|
|
180
193
|
__crawlee: {
|
|
181
|
-
value: this.
|
|
194
|
+
value: this.#userData.__crawlee,
|
|
182
195
|
enumerable: false,
|
|
183
196
|
writable: true,
|
|
184
197
|
},
|
|
185
198
|
toJSON: {
|
|
186
199
|
value: () => {
|
|
187
|
-
if (Object.keys(this.
|
|
200
|
+
if (Object.keys(this.#userData.__crawlee).length > 0) {
|
|
188
201
|
return {
|
|
189
|
-
...this
|
|
190
|
-
__crawlee: this.
|
|
202
|
+
...this.#userData,
|
|
203
|
+
__crawlee: this.#userData.__crawlee,
|
|
191
204
|
};
|
|
192
205
|
}
|
|
193
|
-
return this
|
|
206
|
+
return this.#userData;
|
|
194
207
|
},
|
|
195
208
|
enumerable: false,
|
|
196
209
|
writable: true,
|
|
197
210
|
},
|
|
198
211
|
});
|
|
199
|
-
this
|
|
212
|
+
this.#userData = value;
|
|
200
213
|
},
|
|
201
214
|
enumerable: true,
|
|
202
215
|
},
|
package/router.d.ts
CHANGED
|
@@ -48,7 +48,6 @@ export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
|
|
|
48
48
|
/**
|
|
49
49
|
* The `userData` type of the default route: inferred from the {@link defaultRoute} schema when the route map
|
|
50
50
|
* carries one, otherwise the provided `Fallback`.
|
|
51
|
-
* @internal
|
|
52
51
|
*/
|
|
53
52
|
export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
|
|
54
53
|
[defaultRoute]: infer DefaultUserData extends Dictionary;
|
|
@@ -66,7 +65,7 @@ export declare function validateUserData(label: string | symbol, schema: Standar
|
|
|
66
65
|
* `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
|
|
67
66
|
*/
|
|
68
67
|
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
|
|
69
|
-
export interface RouterHandler<Context extends
|
|
68
|
+
export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
|
|
70
69
|
(ctx: Context): Awaitable<void>;
|
|
71
70
|
}
|
|
72
71
|
export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
|
|
@@ -215,11 +214,8 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
|
|
|
215
214
|
* });
|
|
216
215
|
* ```
|
|
217
216
|
*/
|
|
218
|
-
export declare class Router<Context extends
|
|
219
|
-
private
|
|
220
|
-
private readonly schemas;
|
|
221
|
-
private readonly timeouts;
|
|
222
|
-
private readonly middlewares;
|
|
217
|
+
export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
|
|
218
|
+
#private;
|
|
223
219
|
/**
|
|
224
220
|
* use Router.create() instead!
|
|
225
221
|
* @ignore
|
|
@@ -303,8 +299,8 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
|
|
|
303
299
|
* await crawler.run();
|
|
304
300
|
* ```
|
|
305
301
|
*/
|
|
306
|
-
static create<Context extends
|
|
307
|
-
static create<Context extends
|
|
308
|
-
static create<Context extends
|
|
302
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
303
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
304
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
309
305
|
}
|
|
310
306
|
export {};
|
package/router.js
CHANGED
|
@@ -161,10 +161,10 @@ export async function validateUserData(label, schema, userData) {
|
|
|
161
161
|
* ```
|
|
162
162
|
*/
|
|
163
163
|
export class Router {
|
|
164
|
-
routes = new Map();
|
|
165
|
-
schemas = new Map();
|
|
166
|
-
timeouts = new Map();
|
|
167
|
-
middlewares = [];
|
|
164
|
+
#routes = new Map();
|
|
165
|
+
#schemas = new Map();
|
|
166
|
+
#timeouts = new Map();
|
|
167
|
+
#middlewares = [];
|
|
168
168
|
/**
|
|
169
169
|
* use Router.create() instead!
|
|
170
170
|
* @ignore
|
|
@@ -172,9 +172,9 @@ export class Router {
|
|
|
172
172
|
constructor() { }
|
|
173
173
|
addHandler(label, handler, options = {}) {
|
|
174
174
|
this.validate(label);
|
|
175
|
-
this
|
|
175
|
+
this.#routes.set(label, handler);
|
|
176
176
|
if (options.requestHandlerTimeoutSecs !== undefined) {
|
|
177
|
-
this
|
|
177
|
+
this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
|
|
178
178
|
}
|
|
179
179
|
}
|
|
180
180
|
/**
|
|
@@ -187,9 +187,9 @@ export class Router {
|
|
|
187
187
|
*/
|
|
188
188
|
addDefaultHandler(handler, options = {}) {
|
|
189
189
|
this.validate(defaultRoute);
|
|
190
|
-
this
|
|
190
|
+
this.#routes.set(defaultRoute, handler);
|
|
191
191
|
if (options.requestHandlerTimeoutSecs !== undefined) {
|
|
192
|
-
this
|
|
192
|
+
this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
|
|
193
193
|
}
|
|
194
194
|
}
|
|
195
195
|
/**
|
|
@@ -199,25 +199,25 @@ export class Router {
|
|
|
199
199
|
*/
|
|
200
200
|
getSchema(label) {
|
|
201
201
|
if (label != null) {
|
|
202
|
-
const schema = this
|
|
202
|
+
const schema = this.#schemas.get(label);
|
|
203
203
|
if (schema) {
|
|
204
204
|
return schema;
|
|
205
205
|
}
|
|
206
206
|
// A label with its own route is fully specified; don't fall back to the default-route schema.
|
|
207
|
-
if (this
|
|
207
|
+
if (this.#routes.has(label)) {
|
|
208
208
|
return undefined;
|
|
209
209
|
}
|
|
210
210
|
}
|
|
211
211
|
// Requests with no route of their own fall through to the default handler, so validate their
|
|
212
212
|
// `userData` against the default-route schema, if one was registered.
|
|
213
|
-
return this
|
|
213
|
+
return this.#schemas.get(defaultRoute);
|
|
214
214
|
}
|
|
215
215
|
/**
|
|
216
216
|
* Registers a middleware that will be fired before the matching route handler.
|
|
217
217
|
* Multiple middlewares can be registered, they will be fired in the same order.
|
|
218
218
|
*/
|
|
219
219
|
use(middleware) {
|
|
220
|
-
this
|
|
220
|
+
this.#middlewares.push(middleware);
|
|
221
221
|
}
|
|
222
222
|
/**
|
|
223
223
|
* Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
|
|
@@ -226,27 +226,27 @@ export class Router {
|
|
|
226
226
|
* the default route asked for. Used by the crawler; not meant to be called directly.
|
|
227
227
|
*/
|
|
228
228
|
getTimeoutSecs(label) {
|
|
229
|
-
if (label && this
|
|
230
|
-
return this
|
|
229
|
+
if (label && this.#routes.has(label)) {
|
|
230
|
+
return this.#timeouts.get(label);
|
|
231
231
|
}
|
|
232
|
-
return this
|
|
232
|
+
return this.#timeouts.get(defaultRoute);
|
|
233
233
|
}
|
|
234
234
|
/**
|
|
235
235
|
* The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
|
|
236
236
|
* The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
|
|
237
237
|
*/
|
|
238
238
|
getMaxTimeoutSecs() {
|
|
239
|
-
return this
|
|
239
|
+
return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
|
|
240
240
|
}
|
|
241
241
|
/**
|
|
242
242
|
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
243
243
|
*/
|
|
244
244
|
getHandler(label) {
|
|
245
|
-
if (label && this
|
|
246
|
-
return this
|
|
245
|
+
if (label && this.#routes.has(label)) {
|
|
246
|
+
return this.#routes.get(label);
|
|
247
247
|
}
|
|
248
|
-
if (this
|
|
249
|
-
return this
|
|
248
|
+
if (this.#routes.has(defaultRoute)) {
|
|
249
|
+
return this.#routes.get(defaultRoute);
|
|
250
250
|
}
|
|
251
251
|
throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
|
|
252
252
|
' You must set up a route for this label or a default route.' +
|
|
@@ -267,7 +267,7 @@ export class Router {
|
|
|
267
267
|
* Throws when the label already exists in our registry.
|
|
268
268
|
*/
|
|
269
269
|
validate(label) {
|
|
270
|
-
if (this
|
|
270
|
+
if (this.#routes.has(label)) {
|
|
271
271
|
const message = label === defaultRoute
|
|
272
272
|
? `Default route is already defined!`
|
|
273
273
|
: `Route for label '${String(label)}' is already defined!`;
|
|
@@ -291,14 +291,14 @@ export class Router {
|
|
|
291
291
|
router.addHandler(label, value);
|
|
292
292
|
}
|
|
293
293
|
else {
|
|
294
|
-
router
|
|
294
|
+
router.#schemas.set(label, value);
|
|
295
295
|
}
|
|
296
296
|
}
|
|
297
297
|
const func = async function (context) {
|
|
298
298
|
const { url, loadedUrl, label } = context.request;
|
|
299
299
|
context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
|
|
300
300
|
await router.validateRequest(context);
|
|
301
|
-
for (const middleware of router
|
|
301
|
+
for (const middleware of router.#middlewares) {
|
|
302
302
|
await middleware(context);
|
|
303
303
|
}
|
|
304
304
|
return router.getHandler(label)(context);
|
package/serialization.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import zlib from 'node:zlib';
|
|
4
|
-
import ow from 'ow';
|
|
5
4
|
import StreamArray from 'stream-json/streamers/StreamArray.js';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import { parseArgument, schemas } from './validators.js';
|
|
6
7
|
const pipeline = util.promisify(streamPipeline);
|
|
8
|
+
const uint8ArraySchema = z.instanceof(Uint8Array);
|
|
7
9
|
/**
|
|
8
10
|
* Transforms an array of items to a JSON in a streaming
|
|
9
11
|
* fashion to save memory. It operates in batches to speed
|
|
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
|
|
|
11
13
|
* @internal
|
|
12
14
|
*/
|
|
13
15
|
class ArrayToJson extends Readable {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
#offset = 0;
|
|
17
|
+
#batchSize;
|
|
18
|
+
#data;
|
|
17
19
|
constructor(data, options = {}) {
|
|
18
20
|
super({
|
|
19
21
|
...options,
|
|
20
22
|
autoDestroy: true,
|
|
21
23
|
emitClose: true,
|
|
22
24
|
});
|
|
23
|
-
this.data = data;
|
|
24
25
|
const { batchSize = 10000 } = options;
|
|
25
|
-
this
|
|
26
|
-
this
|
|
26
|
+
this.#batchSize = batchSize;
|
|
27
|
+
this.#data = data;
|
|
27
28
|
this.push('[');
|
|
28
29
|
}
|
|
29
30
|
_read() {
|
|
30
31
|
try {
|
|
31
|
-
const items = this
|
|
32
|
+
const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
|
|
32
33
|
if (items.length) {
|
|
33
34
|
const json = JSON.stringify(items);
|
|
34
35
|
// Strip brackets to flatten the batch.
|
|
35
36
|
const itemString = json.substring(1, json.length - 1);
|
|
36
|
-
if (this
|
|
37
|
+
if (this.#offset > 0)
|
|
37
38
|
this.push(',', 'utf8');
|
|
38
39
|
this.push(itemString, 'utf8');
|
|
39
|
-
this
|
|
40
|
+
this.#offset += this.#batchSize;
|
|
40
41
|
}
|
|
41
42
|
else {
|
|
42
43
|
this.push(']');
|
|
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
|
|
|
58
59
|
* @internal
|
|
59
60
|
*/
|
|
60
61
|
export async function serializeArray(data) {
|
|
61
|
-
|
|
62
|
+
parseArgument(data, schemas.anyArray);
|
|
62
63
|
const { chunks, collector } = createChunkCollector();
|
|
63
64
|
await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
|
|
64
65
|
return Buffer.concat(chunks);
|
|
@@ -73,7 +74,7 @@ export async function serializeArray(data) {
|
|
|
73
74
|
* @internal
|
|
74
75
|
*/
|
|
75
76
|
export async function deserializeArray(compressedData) {
|
|
76
|
-
|
|
77
|
+
parseArgument(compressedData, uint8ArraySchema);
|
|
77
78
|
const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
|
|
78
79
|
await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
|
|
79
80
|
return chunks;
|
|
@@ -88,7 +89,7 @@ export async function deserializeArray(compressedData) {
|
|
|
88
89
|
* @internal
|
|
89
90
|
*/
|
|
90
91
|
export function createDeserialize(compressedData) {
|
|
91
|
-
|
|
92
|
+
parseArgument(compressedData, uint8ArraySchema);
|
|
92
93
|
const streamArray = StreamArray.withParser();
|
|
93
94
|
const destination = pluckValue(streamArray);
|
|
94
95
|
streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
|
package/service_locator.d.ts
CHANGED
|
@@ -62,6 +62,17 @@ interface ServiceLocatorInterface {
|
|
|
62
62
|
* Get the storage instance manager (shared across all storage types).
|
|
63
63
|
*/
|
|
64
64
|
getStorageInstanceManager(): StorageInstanceManager;
|
|
65
|
+
/**
|
|
66
|
+
* Returns the currently set services without triggering the implicit creation of defaults.
|
|
67
|
+
* Used to inherit already-materialized services into crawler-scoped service locators.
|
|
68
|
+
* @internal
|
|
69
|
+
*/
|
|
70
|
+
getServicesIfSet(): {
|
|
71
|
+
configuration?: Configuration;
|
|
72
|
+
eventManager?: EventManager;
|
|
73
|
+
storageBackend?: StorageBackend;
|
|
74
|
+
logger?: CrawleeLogger;
|
|
75
|
+
};
|
|
65
76
|
/**
|
|
66
77
|
* Resets the service locator to its initial state.
|
|
67
78
|
* Used mainly for testing purposes.
|
|
@@ -101,16 +112,7 @@ interface ServiceLocatorInterface {
|
|
|
101
112
|
* ```
|
|
102
113
|
*/
|
|
103
114
|
export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
104
|
-
private
|
|
105
|
-
private eventManager?;
|
|
106
|
-
private storageBackend?;
|
|
107
|
-
private logger?;
|
|
108
|
-
/**
|
|
109
|
-
* Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
|
|
110
|
-
* Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
|
|
111
|
-
* Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
|
|
112
|
-
*/
|
|
113
|
-
private static storageInstanceManager?;
|
|
115
|
+
#private;
|
|
114
116
|
/**
|
|
115
117
|
* Creates a new ServiceLocator instance.
|
|
116
118
|
*
|
|
@@ -120,6 +122,13 @@ export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
|
120
122
|
* @param logger Optional logger instance to use
|
|
121
123
|
*/
|
|
122
124
|
constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
|
|
125
|
+
/** @internal */
|
|
126
|
+
getServicesIfSet(): {
|
|
127
|
+
configuration?: Configuration;
|
|
128
|
+
eventManager?: EventManager;
|
|
129
|
+
storageBackend?: StorageBackend;
|
|
130
|
+
logger?: CrawleeLogger;
|
|
131
|
+
};
|
|
123
132
|
getConfiguration(): Configuration;
|
|
124
133
|
setConfiguration(configuration: Configuration): void;
|
|
125
134
|
getEventManager(): EventManager;
|
package/service_locator.js
CHANGED
|
@@ -39,16 +39,16 @@ import { StorageInstanceManager } from './storages/storage_instance_manager.js';
|
|
|
39
39
|
* ```
|
|
40
40
|
*/
|
|
41
41
|
export class ServiceLocator {
|
|
42
|
-
configuration;
|
|
43
|
-
eventManager;
|
|
44
|
-
storageBackend;
|
|
45
|
-
logger;
|
|
42
|
+
#configuration;
|
|
43
|
+
#eventManager;
|
|
44
|
+
#storageBackend;
|
|
45
|
+
#logger;
|
|
46
46
|
/**
|
|
47
47
|
* Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
|
|
48
48
|
* Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
|
|
49
49
|
* Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
|
|
50
50
|
*/
|
|
51
|
-
static storageInstanceManager;
|
|
51
|
+
static #storageInstanceManager;
|
|
52
52
|
/**
|
|
53
53
|
* Creates a new ServiceLocator instance.
|
|
54
54
|
*
|
|
@@ -58,61 +58,70 @@ export class ServiceLocator {
|
|
|
58
58
|
* @param logger Optional logger instance to use
|
|
59
59
|
*/
|
|
60
60
|
constructor(configuration, eventManager, storageBackend, logger) {
|
|
61
|
-
this
|
|
62
|
-
this
|
|
63
|
-
this
|
|
64
|
-
this
|
|
61
|
+
this.#configuration = configuration;
|
|
62
|
+
this.#eventManager = eventManager;
|
|
63
|
+
this.#storageBackend = storageBackend;
|
|
64
|
+
this.#logger = logger;
|
|
65
|
+
}
|
|
66
|
+
/** @internal */
|
|
67
|
+
getServicesIfSet() {
|
|
68
|
+
return {
|
|
69
|
+
configuration: this.#configuration,
|
|
70
|
+
eventManager: this.#eventManager,
|
|
71
|
+
storageBackend: this.#storageBackend,
|
|
72
|
+
logger: this.#logger,
|
|
73
|
+
};
|
|
65
74
|
}
|
|
66
75
|
getConfiguration() {
|
|
67
|
-
if (!this
|
|
76
|
+
if (!this.#configuration) {
|
|
68
77
|
this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
|
|
69
|
-
this
|
|
78
|
+
this.#configuration = new Configuration();
|
|
70
79
|
}
|
|
71
|
-
return this
|
|
80
|
+
return this.#configuration;
|
|
72
81
|
}
|
|
73
82
|
setConfiguration(configuration) {
|
|
74
83
|
// Same instance, no need to do anything
|
|
75
|
-
if (this
|
|
84
|
+
if (this.#configuration === configuration) {
|
|
76
85
|
return;
|
|
77
86
|
}
|
|
78
87
|
// Already have a different configuration that was retrieved
|
|
79
|
-
if (this
|
|
80
|
-
throw new ServiceConflictError('Configuration', configuration, this
|
|
88
|
+
if (this.#configuration) {
|
|
89
|
+
throw new ServiceConflictError('Configuration', configuration, this.#configuration);
|
|
81
90
|
}
|
|
82
|
-
this
|
|
91
|
+
this.#configuration = configuration;
|
|
83
92
|
}
|
|
84
93
|
getEventManager() {
|
|
85
|
-
if (!this
|
|
94
|
+
if (!this.#eventManager) {
|
|
86
95
|
this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
|
|
87
|
-
if (!this
|
|
96
|
+
if (!this.#configuration) {
|
|
88
97
|
this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
|
|
89
98
|
'It is advised to explicitly first set the configuration instead.');
|
|
90
99
|
}
|
|
91
|
-
this
|
|
100
|
+
this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
|
|
92
101
|
}
|
|
93
|
-
return this
|
|
102
|
+
return this.#eventManager;
|
|
94
103
|
}
|
|
95
104
|
setEventManager(eventManager) {
|
|
96
105
|
// Same instance, no need to do anything
|
|
97
|
-
if (this
|
|
106
|
+
if (this.#eventManager === eventManager) {
|
|
98
107
|
return;
|
|
99
108
|
}
|
|
100
109
|
// Already have a different event manager that was retrieved
|
|
101
|
-
if (this
|
|
102
|
-
throw new ServiceConflictError('EventManager', eventManager, this
|
|
110
|
+
if (this.#eventManager) {
|
|
111
|
+
throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
|
|
103
112
|
}
|
|
104
|
-
this
|
|
113
|
+
this.#eventManager = eventManager;
|
|
105
114
|
}
|
|
106
115
|
getStorageBackend() {
|
|
107
|
-
if (!this
|
|
116
|
+
if (!this.#storageBackend) {
|
|
108
117
|
this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
|
|
109
118
|
'(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
|
|
110
|
-
if (!this
|
|
119
|
+
if (!this.#configuration) {
|
|
111
120
|
this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
|
|
112
121
|
'It is advised to explicitly first set the configuration instead.');
|
|
113
122
|
}
|
|
114
123
|
const configuration = this.getConfiguration();
|
|
115
|
-
this
|
|
124
|
+
this.#storageBackend = configuration.persistStorage
|
|
116
125
|
? new FileSystemStorageBackend({
|
|
117
126
|
localDataDirectory: configuration.storageDir,
|
|
118
127
|
logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
|
|
@@ -121,50 +130,50 @@ export class ServiceLocator {
|
|
|
121
130
|
logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
|
|
122
131
|
});
|
|
123
132
|
}
|
|
124
|
-
return this
|
|
133
|
+
return this.#storageBackend;
|
|
125
134
|
}
|
|
126
135
|
setStorageBackend(storageBackend) {
|
|
127
136
|
// Same instance, no need to do anything
|
|
128
|
-
if (this
|
|
137
|
+
if (this.#storageBackend === storageBackend) {
|
|
129
138
|
return;
|
|
130
139
|
}
|
|
131
140
|
// Already have a different storage backend that was retrieved
|
|
132
|
-
if (this
|
|
133
|
-
throw new ServiceConflictError('StorageBackend', storageBackend, this
|
|
141
|
+
if (this.#storageBackend) {
|
|
142
|
+
throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
|
|
134
143
|
}
|
|
135
|
-
this
|
|
144
|
+
this.#storageBackend = storageBackend;
|
|
136
145
|
}
|
|
137
146
|
getLogger() {
|
|
138
|
-
if (!this
|
|
139
|
-
this
|
|
147
|
+
if (!this.#logger) {
|
|
148
|
+
this.#logger = new ApifyLogAdapter(log);
|
|
140
149
|
}
|
|
141
|
-
return this
|
|
150
|
+
return this.#logger;
|
|
142
151
|
}
|
|
143
152
|
setLogger(logger) {
|
|
144
|
-
if (this
|
|
153
|
+
if (this.#logger === logger) {
|
|
145
154
|
return;
|
|
146
155
|
}
|
|
147
|
-
if (this
|
|
148
|
-
throw new ServiceConflictError('Logger', logger, this
|
|
156
|
+
if (this.#logger) {
|
|
157
|
+
throw new ServiceConflictError('Logger', logger, this.#logger);
|
|
149
158
|
}
|
|
150
|
-
this
|
|
159
|
+
this.#logger = logger;
|
|
151
160
|
}
|
|
152
161
|
getChildLog(prefix) {
|
|
153
162
|
return this.getLogger().child({ prefix });
|
|
154
163
|
}
|
|
155
164
|
getStorageInstanceManager() {
|
|
156
|
-
if (!ServiceLocator
|
|
157
|
-
ServiceLocator
|
|
165
|
+
if (!ServiceLocator.#storageInstanceManager) {
|
|
166
|
+
ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
|
|
158
167
|
}
|
|
159
|
-
return ServiceLocator
|
|
168
|
+
return ServiceLocator.#storageInstanceManager;
|
|
160
169
|
}
|
|
161
170
|
reset() {
|
|
162
|
-
this
|
|
163
|
-
this
|
|
164
|
-
this
|
|
165
|
-
this
|
|
166
|
-
ServiceLocator
|
|
167
|
-
ServiceLocator
|
|
171
|
+
this.#configuration = undefined;
|
|
172
|
+
this.#eventManager = undefined;
|
|
173
|
+
this.#storageBackend = undefined;
|
|
174
|
+
this.#logger = undefined;
|
|
175
|
+
ServiceLocator.#storageInstanceManager?.clearCache();
|
|
176
|
+
ServiceLocator.#storageInstanceManager = undefined;
|
|
168
177
|
}
|
|
169
178
|
}
|
|
170
179
|
/**
|