@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +3 -21
  2. package/autoscaling/autoscaled_pool.js +105 -97
  3. package/autoscaling/concurrency_system.d.ts +7 -22
  4. package/autoscaling/concurrency_system.js +112 -100
  5. package/autoscaling/cpu_load_signal.d.ts +1 -2
  6. package/autoscaling/cpu_load_signal.js +11 -10
  7. package/autoscaling/event_loop_load_signal.d.ts +1 -4
  8. package/autoscaling/event_loop_load_signal.js +18 -18
  9. package/autoscaling/index.d.ts +1 -1
  10. package/autoscaling/index.js +1 -1
  11. package/autoscaling/load_signal.d.ts +8 -7
  12. package/autoscaling/load_signal.js +14 -12
  13. package/autoscaling/memory_load_signal.d.ts +3 -12
  14. package/autoscaling/memory_load_signal.js +41 -41
  15. package/autoscaling/snapshotter.d.ts +7 -10
  16. package/autoscaling/snapshotter.js +18 -18
  17. package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts} +14 -17
  18. package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js} +27 -27
  19. package/autoscaling/system_status.d.ts +9 -11
  20. package/autoscaling/system_status.js +13 -13
  21. package/configuration.d.ts +16 -16
  22. package/configuration.js +6 -6
  23. package/crawlers/context_pipeline.js +6 -6
  24. package/crawlers/crawler_commons.d.ts +14 -112
  25. package/crawlers/crawler_commons.js +1 -107
  26. package/crawlers/error_snapshotter.d.ts +2 -4
  27. package/crawlers/error_snapshotter.js +5 -5
  28. package/crawlers/index.d.ts +1 -1
  29. package/crawlers/index.js +0 -1
  30. package/crawlers/statistics.d.ts +164 -45
  31. package/crawlers/statistics.js +339 -153
  32. package/debug.js +4 -4
  33. package/enqueue_links/enqueue_links.d.ts +51 -159
  34. package/enqueue_links/enqueue_links.js +38 -231
  35. package/enqueue_links/shared.d.ts +34 -24
  36. package/enqueue_links/shared.js +54 -36
  37. package/errors.d.ts +43 -12
  38. package/errors.js +48 -9
  39. package/events/event_manager.d.ts +2 -2
  40. package/events/event_manager.js +7 -6
  41. package/events/local_event_manager.d.ts +1 -1
  42. package/events/local_event_manager.js +5 -5
  43. package/http.d.ts +9 -0
  44. package/http.js +28 -0
  45. package/index.d.ts +1 -1
  46. package/index.js +1 -1
  47. package/log.js +5 -1
  48. package/memory-storage/memory-storage.d.ts +3 -11
  49. package/memory-storage/memory-storage.js +42 -48
  50. package/memory-storage/resource-clients/dataset.d.ts +1 -1
  51. package/memory-storage/resource-clients/dataset.js +8 -13
  52. package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
  53. package/memory-storage/resource-clients/key-value-store.js +36 -38
  54. package/memory-storage/resource-clients/request-queue.d.ts +4 -23
  55. package/memory-storage/resource-clients/request-queue.js +68 -80
  56. package/owned_or_injected.d.ts +1 -3
  57. package/owned_or_injected.js +17 -17
  58. package/package.json +8 -9
  59. package/proxy_configuration.d.ts +1 -3
  60. package/proxy_configuration.js +18 -14
  61. package/recoverable_state.d.ts +80 -45
  62. package/recoverable_state.js +163 -75
  63. package/request.d.ts +3 -4
  64. package/request.js +57 -44
  65. package/router.d.ts +6 -9
  66. package/router.js +23 -23
  67. package/serialization.js +14 -13
  68. package/service_locator.d.ts +19 -10
  69. package/service_locator.js +57 -48
  70. package/session_pool/session.d.ts +3 -14
  71. package/session_pool/session.js +76 -74
  72. package/session_pool/session_pool.d.ts +6 -12
  73. package/session_pool/session_pool.js +85 -77
  74. package/storages/batched_adds.d.ts +37 -0
  75. package/storages/batched_adds.js +73 -0
  76. package/storages/dataset.d.ts +12 -1
  77. package/storages/dataset.js +133 -32
  78. package/storages/index.d.ts +2 -1
  79. package/storages/index.js +2 -1
  80. package/storages/key_value_store.d.ts +19 -4
  81. package/storages/key_value_store.js +204 -75
  82. package/storages/request_dedup_cache.d.ts +1 -2
  83. package/storages/request_dedup_cache.js +9 -9
  84. package/storages/request_list.d.ts +4 -23
  85. package/storages/request_list.js +103 -96
  86. package/storages/request_loader.d.ts +5 -0
  87. package/storages/request_manager_tandem.d.ts +1 -10
  88. package/storages/request_manager_tandem.js +27 -27
  89. package/storages/request_queue.d.ts +21 -18
  90. package/storages/request_queue.js +318 -177
  91. package/storages/sitemap_request_loader.d.ts +22 -70
  92. package/storages/sitemap_request_loader.js +122 -116
  93. package/storages/storage_instance_manager.d.ts +1 -2
  94. package/storages/storage_instance_manager.js +17 -17
  95. package/storages/storage_stats.d.ts +1 -1
  96. package/storages/storage_stats.js +4 -4
  97. package/storages/throttling_request_manager.d.ts +239 -0
  98. package/storages/throttling_request_manager.js +646 -0
  99. package/storages/transaction.d.ts +252 -0
  100. package/storages/transaction.js +251 -0
  101. package/storages/utils.d.ts +6 -3
  102. package/storages/utils.js +11 -2
  103. package/system-info/runtime.js +7 -7
  104. package/url.d.ts +9 -0
  105. package/url.js +11 -0
  106. package/validators.d.ts +22 -25
  107. package/validators.js +13 -25
  108. package/storages/access_checking.d.ts +0 -12
  109. package/storages/access_checking.js +0 -17
package/request.js CHANGED
@@ -1,32 +1,13 @@
1
1
  import crypto from 'node:crypto';
2
2
  import util from 'node:util';
3
- import ow from 'ow';
3
+ import { z } from 'zod';
4
4
  import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
5
5
  import { serviceLocator } from './service_locator.js';
6
6
  import { keys } from './typedefs.js';
7
- // new properties on the Request object breaks serialization
8
- const requestOptionalPredicates = {
9
- id: ow.optional.string,
10
- loadedUrl: ow.optional.string.url,
11
- uniqueKey: ow.optional.string,
12
- method: ow.optional.string,
13
- payload: ow.optional.any(ow.string, ow.uint8Array),
14
- noRetry: ow.optional.boolean,
15
- retryCount: ow.optional.number,
16
- sessionId: ow.optional.string,
17
- maxRetries: ow.optional.number,
18
- errorMessages: ow.optional.array.ofType(ow.string),
19
- headers: ow.optional.object,
20
- userData: ow.optional.object,
21
- label: ow.optional.string,
22
- handledAt: ow.optional.any(ow.string.date, ow.date),
23
- keepUrlFragment: ow.optional.boolean,
24
- useExtendedUniqueKey: ow.optional.boolean,
25
- alwaysEnqueue: ow.optional.boolean,
26
- skipNavigation: ow.optional.boolean,
27
- crawlDepth: ow.optional.number.greaterThanOrEqual(0),
28
- state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
29
- };
7
+ import { parseArgument, schemas } from './validators.js';
8
+ const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
9
+ message: 'Invalid input: expected a date string',
10
+ });
30
11
  export var RequestState;
31
12
  (function (RequestState) {
32
13
  RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
@@ -38,6 +19,34 @@ export var RequestState;
38
19
  RequestState[RequestState["ERROR"] = 6] = "ERROR";
39
20
  RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
40
21
  })(RequestState || (RequestState = {}));
22
+ const requestUrlSchema = z.object({ url: z.string() });
23
+ // new properties on the Request object breaks serialization
24
+ const requestOptionalSchemaShapes = {
25
+ id: z.string().optional(),
26
+ loadedUrl: z.url().optional(),
27
+ uniqueKey: z.string().optional(),
28
+ method: z.string().optional(),
29
+ payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
30
+ noRetry: z.boolean().optional(),
31
+ retryCount: schemas.anyNumber.optional(),
32
+ sessionId: z.string().optional(),
33
+ maxRetries: schemas.anyNumber.optional(),
34
+ errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
35
+ headers: z.looseObject({}).optional(),
36
+ userData: z.looseObject({}).optional(),
37
+ label: z.string().optional(),
38
+ handledAt: z.union([dateString, z.date()]).optional(),
39
+ keepUrlFragment: z.boolean().optional(),
40
+ useExtendedUniqueKey: z.boolean().optional(),
41
+ alwaysEnqueue: z.boolean().optional(),
42
+ skipNavigation: z.boolean().optional(),
43
+ crawlDepth: schemas.anyNumber
44
+ .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
45
+ .optional(),
46
+ state: z.enum(RequestState).optional(),
47
+ };
48
+ // Each schema is wrapped in a single-key object so validation errors carry the property name.
49
+ const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
41
50
  /**
42
51
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
43
52
  * The `Request` object also stores information about errors that occurred during processing of the request.
@@ -101,7 +110,7 @@ class CrawleeRequest {
101
110
  /** Object with HTTP headers. Key is header name, value is the value. */
102
111
  headers;
103
112
  /** Private store for the custom user data assigned to the request. */
104
- _userData = {};
113
+ #userData = {};
105
114
  /**
106
115
  * Custom user data assigned to the request.
107
116
  *
@@ -118,22 +127,26 @@ class CrawleeRequest {
118
127
  * `Request` parameters including the URL, HTTP method and headers, and others.
119
128
  */
120
129
  constructor(options) {
121
- ow(options, 'RequestOptions', ow.object);
122
- ow(options.url, 'RequestOptions.url', ow.string);
123
- // 'ow' validation is slow, because it checks all predicates
130
+ // A bare URL is a common slip — point at the object form instead of a generic type error.
131
+ if (typeof options === 'string') {
132
+ throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
133
+ 'Did you mean `new Request({ url })`?');
134
+ }
135
+ parseArgument(options, schemas.anyObject, 'RequestOptions');
136
+ parseArgument(options, requestUrlSchema, 'RequestOptions');
137
+ // Full-shape validation is slow, because it checks all predicates
124
138
  // even if the validated object has only 1 property.
125
139
  // This custom validation loop iterates only over existing
126
140
  // properties and speeds up the validation cca 3-fold.
127
- // See https://github.com/sindresorhus/ow/issues/193
128
141
  keys(options).forEach((prop) => {
129
142
  // skip url, because it is validated above
130
143
  if (prop === 'url') {
131
144
  return;
132
145
  }
133
- const predicate = requestOptionalPredicates[prop];
146
+ const schema = requestOptionalSchemas[prop];
134
147
  const value = options[prop];
135
- if (predicate) {
136
- ow(value, `RequestOptions.${prop}`, predicate);
148
+ if (schema) {
149
+ parseArgument({ [prop]: value }, schema, 'RequestOptions');
137
150
  }
138
151
  });
139
152
  const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
@@ -167,36 +180,36 @@ class CrawleeRequest {
167
180
  if (label) {
168
181
  userData.label = label;
169
182
  }
183
+ // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
184
+ // bag is non-enumerable, so the spread alone would silently drop the internal state
185
+ // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
186
+ this.#userData = { __crawlee: userData.__crawlee ?? {}, ...userData };
187
+ // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
170
188
  Object.defineProperties(this, {
171
- _userData: {
172
- value: { __crawlee: {}, ...userData },
173
- enumerable: false,
174
- writable: true,
175
- },
176
189
  userData: {
177
- get: () => this._userData,
190
+ get: () => this.#userData,
178
191
  set: (value) => {
179
192
  Object.defineProperties(value, {
180
193
  __crawlee: {
181
- value: this._userData.__crawlee,
194
+ value: this.#userData.__crawlee,
182
195
  enumerable: false,
183
196
  writable: true,
184
197
  },
185
198
  toJSON: {
186
199
  value: () => {
187
- if (Object.keys(this._userData.__crawlee).length > 0) {
200
+ if (Object.keys(this.#userData.__crawlee).length > 0) {
188
201
  return {
189
- ...this._userData,
190
- __crawlee: this._userData.__crawlee,
202
+ ...this.#userData,
203
+ __crawlee: this.#userData.__crawlee,
191
204
  };
192
205
  }
193
- return this._userData;
206
+ return this.#userData;
194
207
  },
195
208
  enumerable: false,
196
209
  writable: true,
197
210
  },
198
211
  });
199
- this._userData = value;
212
+ this.#userData = value;
200
213
  },
201
214
  enumerable: true,
202
215
  },
package/router.d.ts CHANGED
@@ -65,7 +65,7 @@ export declare function validateUserData(label: string | symbol, schema: Standar
65
65
  * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
66
66
  */
67
67
  export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
68
- export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
68
+ export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
69
69
  (ctx: Context): Awaitable<void>;
70
70
  }
71
71
  export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
@@ -214,11 +214,8 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
214
214
  * });
215
215
  * ```
216
216
  */
217
- export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
218
- private readonly routes;
219
- private readonly schemas;
220
- private readonly timeouts;
221
- private readonly middlewares;
217
+ export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
218
+ #private;
222
219
  /**
223
220
  * use Router.create() instead!
224
221
  * @ignore
@@ -302,8 +299,8 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
302
299
  * await crawler.run();
303
300
  * ```
304
301
  */
305
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
306
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
307
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
302
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
303
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
304
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
308
305
  }
309
306
  export {};
package/router.js CHANGED
@@ -161,10 +161,10 @@ export async function validateUserData(label, schema, userData) {
161
161
  * ```
162
162
  */
163
163
  export class Router {
164
- routes = new Map();
165
- schemas = new Map();
166
- timeouts = new Map();
167
- middlewares = [];
164
+ #routes = new Map();
165
+ #schemas = new Map();
166
+ #timeouts = new Map();
167
+ #middlewares = [];
168
168
  /**
169
169
  * use Router.create() instead!
170
170
  * @ignore
@@ -172,9 +172,9 @@ export class Router {
172
172
  constructor() { }
173
173
  addHandler(label, handler, options = {}) {
174
174
  this.validate(label);
175
- this.routes.set(label, handler);
175
+ this.#routes.set(label, handler);
176
176
  if (options.requestHandlerTimeoutSecs !== undefined) {
177
- this.timeouts.set(label, options.requestHandlerTimeoutSecs);
177
+ this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
178
178
  }
179
179
  }
180
180
  /**
@@ -187,9 +187,9 @@ export class Router {
187
187
  */
188
188
  addDefaultHandler(handler, options = {}) {
189
189
  this.validate(defaultRoute);
190
- this.routes.set(defaultRoute, handler);
190
+ this.#routes.set(defaultRoute, handler);
191
191
  if (options.requestHandlerTimeoutSecs !== undefined) {
192
- this.timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
192
+ this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
193
193
  }
194
194
  }
195
195
  /**
@@ -199,25 +199,25 @@ export class Router {
199
199
  */
200
200
  getSchema(label) {
201
201
  if (label != null) {
202
- const schema = this.schemas.get(label);
202
+ const schema = this.#schemas.get(label);
203
203
  if (schema) {
204
204
  return schema;
205
205
  }
206
206
  // A label with its own route is fully specified; don't fall back to the default-route schema.
207
- if (this.routes.has(label)) {
207
+ if (this.#routes.has(label)) {
208
208
  return undefined;
209
209
  }
210
210
  }
211
211
  // Requests with no route of their own fall through to the default handler, so validate their
212
212
  // `userData` against the default-route schema, if one was registered.
213
- return this.schemas.get(defaultRoute);
213
+ return this.#schemas.get(defaultRoute);
214
214
  }
215
215
  /**
216
216
  * Registers a middleware that will be fired before the matching route handler.
217
217
  * Multiple middlewares can be registered, they will be fired in the same order.
218
218
  */
219
219
  use(middleware) {
220
- this.middlewares.push(middleware);
220
+ this.#middlewares.push(middleware);
221
221
  }
222
222
  /**
223
223
  * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
@@ -226,27 +226,27 @@ export class Router {
226
226
  * the default route asked for. Used by the crawler; not meant to be called directly.
227
227
  */
228
228
  getTimeoutSecs(label) {
229
- if (label && this.routes.has(label)) {
230
- return this.timeouts.get(label);
229
+ if (label && this.#routes.has(label)) {
230
+ return this.#timeouts.get(label);
231
231
  }
232
- return this.timeouts.get(defaultRoute);
232
+ return this.#timeouts.get(defaultRoute);
233
233
  }
234
234
  /**
235
235
  * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
236
236
  * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
237
237
  */
238
238
  getMaxTimeoutSecs() {
239
- return this.timeouts.size > 0 ? Math.max(...this.timeouts.values()) : undefined;
239
+ return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
240
240
  }
241
241
  /**
242
242
  * Returns route handler for given label. If no label is provided, the default request handler will be returned.
243
243
  */
244
244
  getHandler(label) {
245
- if (label && this.routes.has(label)) {
246
- return this.routes.get(label);
245
+ if (label && this.#routes.has(label)) {
246
+ return this.#routes.get(label);
247
247
  }
248
- if (this.routes.has(defaultRoute)) {
249
- return this.routes.get(defaultRoute);
248
+ if (this.#routes.has(defaultRoute)) {
249
+ return this.#routes.get(defaultRoute);
250
250
  }
251
251
  throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
252
252
  ' You must set up a route for this label or a default route.' +
@@ -267,7 +267,7 @@ export class Router {
267
267
  * Throws when the label already exists in our registry.
268
268
  */
269
269
  validate(label) {
270
- if (this.routes.has(label)) {
270
+ if (this.#routes.has(label)) {
271
271
  const message = label === defaultRoute
272
272
  ? `Default route is already defined!`
273
273
  : `Route for label '${String(label)}' is already defined!`;
@@ -291,14 +291,14 @@ export class Router {
291
291
  router.addHandler(label, value);
292
292
  }
293
293
  else {
294
- router.schemas.set(label, value);
294
+ router.#schemas.set(label, value);
295
295
  }
296
296
  }
297
297
  const func = async function (context) {
298
298
  const { url, loadedUrl, label } = context.request;
299
299
  context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
300
300
  await router.validateRequest(context);
301
- for (const middleware of router.middlewares) {
301
+ for (const middleware of router.#middlewares) {
302
302
  await middleware(context);
303
303
  }
304
304
  return router.getHandler(label)(context);
package/serialization.js CHANGED
@@ -1,9 +1,11 @@
1
1
  import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
2
2
  import util from 'node:util';
3
3
  import zlib from 'node:zlib';
4
- import ow from 'ow';
5
4
  import StreamArray from 'stream-json/streamers/StreamArray.js';
5
+ import { z } from 'zod';
6
+ import { parseArgument, schemas } from './validators.js';
6
7
  const pipeline = util.promisify(streamPipeline);
8
+ const uint8ArraySchema = z.instanceof(Uint8Array);
7
9
  /**
8
10
  * Transforms an array of items to a JSON in a streaming
9
11
  * fashion to save memory. It operates in batches to speed
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
11
13
  * @internal
12
14
  */
13
15
  class ArrayToJson extends Readable {
14
- data;
15
- offset = 0;
16
- batchSize;
16
+ #offset = 0;
17
+ #batchSize;
18
+ #data;
17
19
  constructor(data, options = {}) {
18
20
  super({
19
21
  ...options,
20
22
  autoDestroy: true,
21
23
  emitClose: true,
22
24
  });
23
- this.data = data;
24
25
  const { batchSize = 10000 } = options;
25
- this.batchSize = batchSize;
26
- this.data = data;
26
+ this.#batchSize = batchSize;
27
+ this.#data = data;
27
28
  this.push('[');
28
29
  }
29
30
  _read() {
30
31
  try {
31
- const items = this.data.slice(this.offset, this.offset + this.batchSize);
32
+ const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
32
33
  if (items.length) {
33
34
  const json = JSON.stringify(items);
34
35
  // Strip brackets to flatten the batch.
35
36
  const itemString = json.substring(1, json.length - 1);
36
- if (this.offset > 0)
37
+ if (this.#offset > 0)
37
38
  this.push(',', 'utf8');
38
39
  this.push(itemString, 'utf8');
39
- this.offset += this.batchSize;
40
+ this.#offset += this.#batchSize;
40
41
  }
41
42
  else {
42
43
  this.push(']');
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
58
59
  * @internal
59
60
  */
60
61
  export async function serializeArray(data) {
61
- ow(data, ow.array);
62
+ parseArgument(data, schemas.anyArray);
62
63
  const { chunks, collector } = createChunkCollector();
63
64
  await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
64
65
  return Buffer.concat(chunks);
@@ -73,7 +74,7 @@ export async function serializeArray(data) {
73
74
  * @internal
74
75
  */
75
76
  export async function deserializeArray(compressedData) {
76
- ow(compressedData, ow.uint8Array);
77
+ parseArgument(compressedData, uint8ArraySchema);
77
78
  const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
78
79
  await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
79
80
  return chunks;
@@ -88,7 +89,7 @@ export async function deserializeArray(compressedData) {
88
89
  * @internal
89
90
  */
90
91
  export function createDeserialize(compressedData) {
91
- ow(compressedData, ow.uint8Array);
92
+ parseArgument(compressedData, uint8ArraySchema);
92
93
  const streamArray = StreamArray.withParser();
93
94
  const destination = pluckValue(streamArray);
94
95
  streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
@@ -62,6 +62,17 @@ interface ServiceLocatorInterface {
62
62
  * Get the storage instance manager (shared across all storage types).
63
63
  */
64
64
  getStorageInstanceManager(): StorageInstanceManager;
65
+ /**
66
+ * Returns the currently set services without triggering the implicit creation of defaults.
67
+ * Used to inherit already-materialized services into crawler-scoped service locators.
68
+ * @internal
69
+ */
70
+ getServicesIfSet(): {
71
+ configuration?: Configuration;
72
+ eventManager?: EventManager;
73
+ storageBackend?: StorageBackend;
74
+ logger?: CrawleeLogger;
75
+ };
65
76
  /**
66
77
  * Resets the service locator to its initial state.
67
78
  * Used mainly for testing purposes.
@@ -101,16 +112,7 @@ interface ServiceLocatorInterface {
101
112
  * ```
102
113
  */
103
114
  export declare class ServiceLocator implements ServiceLocatorInterface {
104
- private configuration?;
105
- private eventManager?;
106
- private storageBackend?;
107
- private logger?;
108
- /**
109
- * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
110
- * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
111
- * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
112
- */
113
- private static storageInstanceManager?;
115
+ #private;
114
116
  /**
115
117
  * Creates a new ServiceLocator instance.
116
118
  *
@@ -120,6 +122,13 @@ export declare class ServiceLocator implements ServiceLocatorInterface {
120
122
  * @param logger Optional logger instance to use
121
123
  */
122
124
  constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
125
+ /** @internal */
126
+ getServicesIfSet(): {
127
+ configuration?: Configuration;
128
+ eventManager?: EventManager;
129
+ storageBackend?: StorageBackend;
130
+ logger?: CrawleeLogger;
131
+ };
123
132
  getConfiguration(): Configuration;
124
133
  setConfiguration(configuration: Configuration): void;
125
134
  getEventManager(): EventManager;
@@ -39,16 +39,16 @@ import { StorageInstanceManager } from './storages/storage_instance_manager.js';
39
39
  * ```
40
40
  */
41
41
  export class ServiceLocator {
42
- configuration;
43
- eventManager;
44
- storageBackend;
45
- logger;
42
+ #configuration;
43
+ #eventManager;
44
+ #storageBackend;
45
+ #logger;
46
46
  /**
47
47
  * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
48
48
  * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
49
49
  * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
50
50
  */
51
- static storageInstanceManager;
51
+ static #storageInstanceManager;
52
52
  /**
53
53
  * Creates a new ServiceLocator instance.
54
54
  *
@@ -58,61 +58,70 @@ export class ServiceLocator {
58
58
  * @param logger Optional logger instance to use
59
59
  */
60
60
  constructor(configuration, eventManager, storageBackend, logger) {
61
- this.configuration = configuration;
62
- this.eventManager = eventManager;
63
- this.storageBackend = storageBackend;
64
- this.logger = logger;
61
+ this.#configuration = configuration;
62
+ this.#eventManager = eventManager;
63
+ this.#storageBackend = storageBackend;
64
+ this.#logger = logger;
65
+ }
66
+ /** @internal */
67
+ getServicesIfSet() {
68
+ return {
69
+ configuration: this.#configuration,
70
+ eventManager: this.#eventManager,
71
+ storageBackend: this.#storageBackend,
72
+ logger: this.#logger,
73
+ };
65
74
  }
66
75
  getConfiguration() {
67
- if (!this.configuration) {
76
+ if (!this.#configuration) {
68
77
  this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
69
- this.configuration = new Configuration();
78
+ this.#configuration = new Configuration();
70
79
  }
71
- return this.configuration;
80
+ return this.#configuration;
72
81
  }
73
82
  setConfiguration(configuration) {
74
83
  // Same instance, no need to do anything
75
- if (this.configuration === configuration) {
84
+ if (this.#configuration === configuration) {
76
85
  return;
77
86
  }
78
87
  // Already have a different configuration that was retrieved
79
- if (this.configuration) {
80
- throw new ServiceConflictError('Configuration', configuration, this.configuration);
88
+ if (this.#configuration) {
89
+ throw new ServiceConflictError('Configuration', configuration, this.#configuration);
81
90
  }
82
- this.configuration = configuration;
91
+ this.#configuration = configuration;
83
92
  }
84
93
  getEventManager() {
85
- if (!this.eventManager) {
94
+ if (!this.#eventManager) {
86
95
  this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
87
- if (!this.configuration) {
96
+ if (!this.#configuration) {
88
97
  this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
89
98
  'It is advised to explicitly first set the configuration instead.');
90
99
  }
91
- this.eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
100
+ this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
92
101
  }
93
- return this.eventManager;
102
+ return this.#eventManager;
94
103
  }
95
104
  setEventManager(eventManager) {
96
105
  // Same instance, no need to do anything
97
- if (this.eventManager === eventManager) {
106
+ if (this.#eventManager === eventManager) {
98
107
  return;
99
108
  }
100
109
  // Already have a different event manager that was retrieved
101
- if (this.eventManager) {
102
- throw new ServiceConflictError('EventManager', eventManager, this.eventManager);
110
+ if (this.#eventManager) {
111
+ throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
103
112
  }
104
- this.eventManager = eventManager;
113
+ this.#eventManager = eventManager;
105
114
  }
106
115
  getStorageBackend() {
107
- if (!this.storageBackend) {
116
+ if (!this.#storageBackend) {
108
117
  this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
109
118
  '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
110
- if (!this.configuration) {
119
+ if (!this.#configuration) {
111
120
  this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
112
121
  'It is advised to explicitly first set the configuration instead.');
113
122
  }
114
123
  const configuration = this.getConfiguration();
115
- this.storageBackend = configuration.persistStorage
124
+ this.#storageBackend = configuration.persistStorage
116
125
  ? new FileSystemStorageBackend({
117
126
  localDataDirectory: configuration.storageDir,
118
127
  logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
@@ -121,50 +130,50 @@ export class ServiceLocator {
121
130
  logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
122
131
  });
123
132
  }
124
- return this.storageBackend;
133
+ return this.#storageBackend;
125
134
  }
126
135
  setStorageBackend(storageBackend) {
127
136
  // Same instance, no need to do anything
128
- if (this.storageBackend === storageBackend) {
137
+ if (this.#storageBackend === storageBackend) {
129
138
  return;
130
139
  }
131
140
  // Already have a different storage backend that was retrieved
132
- if (this.storageBackend) {
133
- throw new ServiceConflictError('StorageBackend', storageBackend, this.storageBackend);
141
+ if (this.#storageBackend) {
142
+ throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
134
143
  }
135
- this.storageBackend = storageBackend;
144
+ this.#storageBackend = storageBackend;
136
145
  }
137
146
  getLogger() {
138
- if (!this.logger) {
139
- this.logger = new ApifyLogAdapter(log);
147
+ if (!this.#logger) {
148
+ this.#logger = new ApifyLogAdapter(log);
140
149
  }
141
- return this.logger;
150
+ return this.#logger;
142
151
  }
143
152
  setLogger(logger) {
144
- if (this.logger === logger) {
153
+ if (this.#logger === logger) {
145
154
  return;
146
155
  }
147
- if (this.logger) {
148
- throw new ServiceConflictError('Logger', logger, this.logger);
156
+ if (this.#logger) {
157
+ throw new ServiceConflictError('Logger', logger, this.#logger);
149
158
  }
150
- this.logger = logger;
159
+ this.#logger = logger;
151
160
  }
152
161
  getChildLog(prefix) {
153
162
  return this.getLogger().child({ prefix });
154
163
  }
155
164
  getStorageInstanceManager() {
156
- if (!ServiceLocator.storageInstanceManager) {
157
- ServiceLocator.storageInstanceManager = new StorageInstanceManager();
165
+ if (!ServiceLocator.#storageInstanceManager) {
166
+ ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
158
167
  }
159
- return ServiceLocator.storageInstanceManager;
168
+ return ServiceLocator.#storageInstanceManager;
160
169
  }
161
170
  reset() {
162
- this.configuration = undefined;
163
- this.eventManager = undefined;
164
- this.storageBackend = undefined;
165
- this.logger = undefined;
166
- ServiceLocator.storageInstanceManager?.clearCache();
167
- ServiceLocator.storageInstanceManager = undefined;
171
+ this.#configuration = undefined;
172
+ this.#eventManager = undefined;
173
+ this.#storageBackend = undefined;
174
+ this.#logger = undefined;
175
+ ServiceLocator.#storageInstanceManager?.clearCache();
176
+ ServiceLocator.#storageInstanceManager = undefined;
168
177
  }
169
178
  }
170
179
  /**