@crawlee/core 4.0.0-beta.121 → 4.0.0-beta.123

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/autoscaling/autoscaled_pool.js +20 -12
  2. package/autoscaling/concurrency_system.d.ts +2 -2
  3. package/autoscaling/concurrency_system.js +31 -20
  4. package/autoscaling/index.d.ts +1 -1
  5. package/autoscaling/index.js +1 -1
  6. package/autoscaling/load_signal.d.ts +7 -6
  7. package/autoscaling/load_signal.js +2 -1
  8. package/autoscaling/snapshotter.d.ts +6 -6
  9. package/autoscaling/snapshotter.js +9 -9
  10. package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts} +13 -12
  11. package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js} +11 -11
  12. package/autoscaling/system_status.d.ts +8 -8
  13. package/autoscaling/system_status.js +2 -2
  14. package/configuration.d.ts +15 -15
  15. package/configuration.js +3 -3
  16. package/crawlers/crawler_commons.d.ts +8 -54
  17. package/crawlers/statistics.d.ts +1 -1
  18. package/crawlers/statistics.js +14 -14
  19. package/debug.js +4 -4
  20. package/enqueue_links/enqueue_links.d.ts +33 -61
  21. package/enqueue_links/enqueue_links.js +35 -152
  22. package/enqueue_links/shared.d.ts +17 -4
  23. package/enqueue_links/shared.js +28 -1
  24. package/memory-storage/resource-clients/dataset.js +2 -8
  25. package/memory-storage/resource-clients/key-value-store.js +23 -26
  26. package/memory-storage/resource-clients/request-queue.js +9 -22
  27. package/package.json +7 -8
  28. package/proxy_configuration.js +10 -6
  29. package/request.d.ts +2 -2
  30. package/request.js +44 -31
  31. package/router.d.ts +5 -5
  32. package/serialization.js +6 -4
  33. package/session_pool/session.js +22 -20
  34. package/session_pool/session_pool.js +20 -17
  35. package/storages/dataset.js +11 -9
  36. package/storages/key_value_store.js +30 -27
  37. package/storages/request_list.d.ts +2 -1
  38. package/storages/request_list.js +26 -21
  39. package/storages/request_queue.js +64 -59
  40. package/storages/sitemap_request_loader.d.ts +1 -1
  41. package/storages/sitemap_request_loader.js +22 -22
  42. package/storages/throttling_request_manager.js +11 -9
  43. package/storages/utils.d.ts +2 -1
  44. package/validators.d.ts +22 -25
  45. package/validators.js +13 -25
@@ -1,9 +1,25 @@
1
1
  import { randomUUID } from 'node:crypto';
2
- import { s } from '@sapphire/shapeshift';
2
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
+ import { z } from 'zod';
3
4
  import { isStream, toBuffer } from '../utils.js';
4
5
  import { BaseClient } from './common/base-client.js';
5
6
  import mime from 'mime-types';
6
7
  const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
8
+ const keySchema = z.string();
9
+ const inputRecordSchema = z.object({
10
+ key: z.string().min(1),
11
+ value: z.union([
12
+ z.null(),
13
+ z.string(),
14
+ z.number(),
15
+ z.instanceof(Buffer),
16
+ z.instanceof(ArrayBuffer),
17
+ schemas.typedArray,
18
+ // only checks the value is an actual object, not null, nor array
19
+ schemas.plainObject,
20
+ ]),
21
+ contentType: z.string().min(1).optional(),
22
+ });
7
23
  /**
8
24
  * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
9
25
  * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
@@ -58,13 +74,7 @@ export class KeyValueStoreBackend extends BaseClient {
58
74
  this.updateTimestamps(true);
59
75
  }
60
76
  async listKeys(options = {}) {
61
- const { prefix, exclusiveStartKey, limit } = s
62
- .object({
63
- prefix: s.string().optional(),
64
- exclusiveStartKey: s.string().optional(),
65
- limit: s.number().int().greaterThan(0).optional(),
66
- })
67
- .parse(options);
77
+ const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
68
78
  const items = [];
69
79
  for (const record of this.#keyValueEntries.values()) {
70
80
  const size = Buffer.byteLength(record.value);
@@ -104,7 +114,7 @@ export class KeyValueStoreBackend extends BaseClient {
104
114
  * @param key The key of the record to generate the public URL for.
105
115
  */
106
116
  async getPublicUrl(key) {
107
- s.string().parse(key);
117
+ parseArgument(key, keySchema);
108
118
  return undefined;
109
119
  }
110
120
  /**
@@ -114,11 +124,11 @@ export class KeyValueStoreBackend extends BaseClient {
114
124
  * @returns `true` if the record exists, `false` if it does not.
115
125
  */
116
126
  async recordExists(key) {
117
- s.string().parse(key);
127
+ parseArgument(key, keySchema);
118
128
  return this.#keyValueEntries.has(key);
119
129
  }
120
130
  async getValue(key) {
121
- s.string().parse(key);
131
+ parseArgument(key, keySchema);
122
132
  const entry = this.#keyValueEntries.get(key);
123
133
  if (!entry) {
124
134
  return undefined;
@@ -136,20 +146,7 @@ export class KeyValueStoreBackend extends BaseClient {
136
146
  return record;
137
147
  }
138
148
  async setValue(record) {
139
- s.object({
140
- key: s.string().lengthGreaterThan(0),
141
- value: s.union([
142
- s.null(),
143
- s.string(),
144
- s.number(),
145
- s.instance(Buffer),
146
- s.instance(ArrayBuffer),
147
- s.typedArray(),
148
- // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
149
- s.object({}).setValidationEnabled(false),
150
- ]),
151
- contentType: s.string().lengthGreaterThan(0).optional(),
152
- }).parse(record);
149
+ parseArgument(record, inputRecordSchema);
153
150
  const { key } = record;
154
151
  let { value } = record;
155
152
  // The frontend (KeyValueStore codec) serializes the value and resolves its content type
@@ -180,7 +177,7 @@ export class KeyValueStoreBackend extends BaseClient {
180
177
  this.updateTimestamps(true);
181
178
  }
182
179
  async deleteValue(key) {
183
- s.string().parse(key);
180
+ parseArgument(key, keySchema);
184
181
  if (this.#keyValueEntries.has(key)) {
185
182
  this.#keyValueEntries.delete(key);
186
183
  this.updateTimestamps(true);
@@ -1,23 +1,10 @@
1
1
  import { randomUUID } from 'node:crypto';
2
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
2
3
  import { AsyncQueue } from '@sapphire/async-queue';
3
- import { s } from '@sapphire/shapeshift';
4
+ import { z } from 'zod';
4
5
  import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils.js';
5
6
  import { BaseClient } from './common/base-client.js';
6
- const requestShape = s
7
- .object({
8
- id: s.string(),
9
- url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
10
- uniqueKey: s.string(),
11
- method: s.string().optional(),
12
- retryCount: s.number().int().optional(),
13
- handledAt: s.union([s.string(), s.date().valid()]).optional(),
14
- })
15
- .passthrough();
16
- const requestShapeWithoutId = requestShape.omit(['id']);
17
- const batchRequestShapeWithoutId = requestShapeWithoutId.array();
18
- const requestOptionsShape = s.object({
19
- forefront: s.boolean().optional(),
20
- });
7
+ const uniqueKeySchema = z.string();
21
8
  export class RequestQueueBackend extends BaseClient {
22
9
  name;
23
10
  /**
@@ -183,8 +170,8 @@ export class RequestQueueBackend extends BaseClient {
183
170
  }
184
171
  }
185
172
  async addBatchOfRequests(requests, options = {}) {
186
- batchRequestShapeWithoutId.parse(requests);
187
- requestOptionsShape.parse(options);
173
+ parseArgument(requests, schemas.storageRequestBatch);
174
+ parseArgument(options, schemas.requestQueueOperationOptions);
188
175
  // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
189
176
  // shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
190
177
  // concurrent operation interleaving at one of the `await` points below.
@@ -233,14 +220,14 @@ export class RequestQueueBackend extends BaseClient {
233
220
  }
234
221
  }
235
222
  async getRequest(uniqueKey) {
236
- s.string().parse(uniqueKey);
223
+ parseArgument(uniqueKey, uniqueKeySchema);
237
224
  this.updateTimestamps(false);
238
225
  const id = uniqueKeyToRequestId(uniqueKey);
239
226
  const json = this.#requests.get(id)?.json;
240
227
  return this.jsonToRequest(json);
241
228
  }
242
229
  async markRequestAsHandled(request) {
243
- requestShape.parse(request);
230
+ parseArgument(request, schemas.storageRequest);
244
231
  this.updateTimestamps(false);
245
232
  // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
246
233
  // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
@@ -278,8 +265,8 @@ export class RequestQueueBackend extends BaseClient {
278
265
  }
279
266
  }
280
267
  async reclaimRequest(request, options = {}) {
281
- requestShape.parse(request);
282
- requestOptionsShape.parse(options);
268
+ parseArgument(request, schemas.storageRequest);
269
+ parseArgument(options, schemas.requestQueueOperationOptions);
283
270
  this.updateTimestamps(false);
284
271
  // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
285
272
  // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/core",
3
- "version": "4.0.0-beta.121",
3
+ "version": "4.0.0-beta.123",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -52,24 +52,23 @@
52
52
  "@apify/log": "^2.5.18",
53
53
  "@apify/timeout": "^0.4.4",
54
54
  "@apify/utilities": "^2.15.5",
55
- "@crawlee/fs-storage": "4.0.0-beta.121",
56
- "@crawlee/types": "4.0.0-beta.121",
57
- "@crawlee/utils": "4.0.0-beta.121",
55
+ "@crawlee/fs-storage": "4.0.0-beta.123",
56
+ "@crawlee/http-client": "4.0.0-beta.123",
57
+ "@crawlee/types": "4.0.0-beta.123",
58
+ "@crawlee/utils": "4.0.0-beta.123",
58
59
  "@sapphire/async-queue": "^1.5.5",
59
- "@sapphire/shapeshift": "^4.0.0",
60
60
  "@vladfrangu/async_event_emitter": "^2.4.6",
61
61
  "content-type": "^1.0.5",
62
62
  "csv-stringify": "^6.5.2",
63
63
  "json5": "^2.2.3",
64
64
  "mime-types": "^3.0.1",
65
65
  "minimatch": "^10.0.1",
66
- "ow": "^2.0.0",
67
66
  "stream-json": "^1.9.1",
68
67
  "tldts": "^7.0.6",
69
68
  "tough-cookie": "^6.0.0",
70
69
  "tslib": "^2.8.1",
71
70
  "type-fest": "^4.41.0",
72
- "zod": "^4.1.0"
71
+ "zod": "^4.4.3"
73
72
  },
74
73
  "lerna": {
75
74
  "command": {
@@ -78,5 +77,5 @@
78
77
  }
79
78
  }
80
79
  },
81
- "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
80
+ "gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
82
81
  }
@@ -1,4 +1,12 @@
1
- import ow from 'ow';
1
+ import { z } from 'zod';
2
+ import { parseArgument, schemas } from './validators.js';
3
+ const proxyConfigurationOptionsSchema = z.strictObject({
4
+ proxyUrls: z
5
+ .array(z.union([z.url(), z.null()]))
6
+ .nonempty()
7
+ .optional(),
8
+ newUrlFunction: schemas.anyFunction.optional(),
9
+ });
2
10
  /**
3
11
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
4
12
  * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -58,11 +66,7 @@ export class ProxyConfiguration {
58
66
  throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
59
67
  'See the v4 upgrading guide for the recommended migration to named sessions.');
60
68
  }
61
- ow(rest, ow.object.exactShape({
62
- proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
63
- newUrlFunction: ow.optional.function,
64
- }));
65
- const { proxyUrls, newUrlFunction } = options;
69
+ const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
66
70
  if (proxyUrls && newUrlFunction)
67
71
  this.throwCannotCombineCustomMethods();
68
72
  if (!proxyUrls && !newUrlFunction && validateRequired)
package/request.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { BinaryLike } from 'node:crypto';
2
2
  import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
3
- import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
3
+ import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
4
4
  import type { SkippedRequestReason } from './enqueue_links/shared.js';
5
5
  export declare enum RequestState {
6
6
  UNPROCESSED = 0,
@@ -281,7 +281,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
281
281
  /** @internal */
282
282
  lockExpiresAt?: Date;
283
283
  /** @internal */
284
- enqueueStrategy?: EnqueueLinksOptions['strategy'];
284
+ enqueueStrategy?: EnqueueStrategyOption;
285
285
  }
286
286
  export interface PushErrorMessageOptions {
287
287
  /**
package/request.js CHANGED
@@ -1,32 +1,13 @@
1
1
  import crypto from 'node:crypto';
2
2
  import util from 'node:util';
3
- import ow from 'ow';
3
+ import { z } from 'zod';
4
4
  import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
5
5
  import { serviceLocator } from './service_locator.js';
6
6
  import { keys } from './typedefs.js';
7
- // new properties on the Request object breaks serialization
8
- const requestOptionalPredicates = {
9
- id: ow.optional.string,
10
- loadedUrl: ow.optional.string.url,
11
- uniqueKey: ow.optional.string,
12
- method: ow.optional.string,
13
- payload: ow.optional.any(ow.string, ow.uint8Array),
14
- noRetry: ow.optional.boolean,
15
- retryCount: ow.optional.number,
16
- sessionId: ow.optional.string,
17
- maxRetries: ow.optional.number,
18
- errorMessages: ow.optional.array.ofType(ow.string),
19
- headers: ow.optional.object,
20
- userData: ow.optional.object,
21
- label: ow.optional.string,
22
- handledAt: ow.optional.any(ow.string.date, ow.date),
23
- keepUrlFragment: ow.optional.boolean,
24
- useExtendedUniqueKey: ow.optional.boolean,
25
- alwaysEnqueue: ow.optional.boolean,
26
- skipNavigation: ow.optional.boolean,
27
- crawlDepth: ow.optional.number.greaterThanOrEqual(0),
28
- state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
29
- };
7
+ import { parseArgument, schemas } from './validators.js';
8
+ const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
9
+ message: 'Invalid input: expected a date string',
10
+ });
30
11
  export var RequestState;
31
12
  (function (RequestState) {
32
13
  RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
@@ -38,6 +19,34 @@ export var RequestState;
38
19
  RequestState[RequestState["ERROR"] = 6] = "ERROR";
39
20
  RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
40
21
  })(RequestState || (RequestState = {}));
22
+ const requestUrlSchema = z.object({ url: z.string() });
23
+ // new properties on the Request object breaks serialization
24
+ const requestOptionalSchemaShapes = {
25
+ id: z.string().optional(),
26
+ loadedUrl: z.url().optional(),
27
+ uniqueKey: z.string().optional(),
28
+ method: z.string().optional(),
29
+ payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
30
+ noRetry: z.boolean().optional(),
31
+ retryCount: schemas.anyNumber.optional(),
32
+ sessionId: z.string().optional(),
33
+ maxRetries: schemas.anyNumber.optional(),
34
+ errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
35
+ headers: z.looseObject({}).optional(),
36
+ userData: z.looseObject({}).optional(),
37
+ label: z.string().optional(),
38
+ handledAt: z.union([dateString, z.date()]).optional(),
39
+ keepUrlFragment: z.boolean().optional(),
40
+ useExtendedUniqueKey: z.boolean().optional(),
41
+ alwaysEnqueue: z.boolean().optional(),
42
+ skipNavigation: z.boolean().optional(),
43
+ crawlDepth: schemas.anyNumber
44
+ .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
45
+ .optional(),
46
+ state: z.enum(RequestState).optional(),
47
+ };
48
+ // Each schema is wrapped in a single-key object so validation errors carry the property name.
49
+ const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
41
50
  /**
42
51
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
43
52
  * The `Request` object also stores information about errors that occurred during processing of the request.
@@ -118,22 +127,26 @@ class CrawleeRequest {
118
127
  * `Request` parameters including the URL, HTTP method and headers, and others.
119
128
  */
120
129
  constructor(options) {
121
- ow(options, 'RequestOptions', ow.object);
122
- ow(options.url, 'RequestOptions.url', ow.string);
123
- // 'ow' validation is slow, because it checks all predicates
130
+ // A bare URL is a common slip — point at the object form instead of a generic type error.
131
+ if (typeof options === 'string') {
132
+ throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
133
+ 'Did you mean `new Request({ url })`?');
134
+ }
135
+ parseArgument(options, schemas.anyObject, 'RequestOptions');
136
+ parseArgument(options, requestUrlSchema, 'RequestOptions');
137
+ // Full-shape validation is slow, because it checks all predicates
124
138
  // even if the validated object has only 1 property.
125
139
  // This custom validation loop iterates only over existing
126
140
  // properties and speeds up the validation cca 3-fold.
127
- // See https://github.com/sindresorhus/ow/issues/193
128
141
  keys(options).forEach((prop) => {
129
142
  // skip url, because it is validated above
130
143
  if (prop === 'url') {
131
144
  return;
132
145
  }
133
- const predicate = requestOptionalPredicates[prop];
146
+ const schema = requestOptionalSchemas[prop];
134
147
  const value = options[prop];
135
- if (predicate) {
136
- ow(value, `RequestOptions.${prop}`, predicate);
148
+ if (schema) {
149
+ parseArgument({ [prop]: value }, schema, 'RequestOptions');
137
150
  }
138
151
  });
139
152
  const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
package/router.d.ts CHANGED
@@ -65,7 +65,7 @@ export declare function validateUserData(label: string | symbol, schema: Standar
65
65
  * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
66
66
  */
67
67
  export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
68
- export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
68
+ export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
69
69
  (ctx: Context): Awaitable<void>;
70
70
  }
71
71
  export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
@@ -214,7 +214,7 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
214
214
  * });
215
215
  * ```
216
216
  */
217
- export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
217
+ export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
218
218
  #private;
219
219
  /**
220
220
  * use Router.create() instead!
@@ -299,8 +299,8 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
299
299
  * await crawler.run();
300
300
  * ```
301
301
  */
302
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
303
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
304
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
302
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
303
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
304
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
305
305
  }
306
306
  export {};
package/serialization.js CHANGED
@@ -1,9 +1,11 @@
1
1
  import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
2
2
  import util from 'node:util';
3
3
  import zlib from 'node:zlib';
4
- import ow from 'ow';
5
4
  import StreamArray from 'stream-json/streamers/StreamArray.js';
5
+ import { z } from 'zod';
6
+ import { parseArgument, schemas } from './validators.js';
6
7
  const pipeline = util.promisify(streamPipeline);
8
+ const uint8ArraySchema = z.instanceof(Uint8Array);
7
9
  /**
8
10
  * Transforms an array of items to a JSON in a streaming
9
11
  * fashion to save memory. It operates in batches to speed
@@ -57,7 +59,7 @@ class ArrayToJson extends Readable {
57
59
  * @internal
58
60
  */
59
61
  export async function serializeArray(data) {
60
- ow(data, ow.array);
62
+ parseArgument(data, schemas.anyArray);
61
63
  const { chunks, collector } = createChunkCollector();
62
64
  await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
63
65
  return Buffer.concat(chunks);
@@ -72,7 +74,7 @@ export async function serializeArray(data) {
72
74
  * @internal
73
75
  */
74
76
  export async function deserializeArray(compressedData) {
75
- ow(compressedData, ow.uint8Array);
77
+ parseArgument(compressedData, uint8ArraySchema);
76
78
  const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
77
79
  await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
78
80
  return chunks;
@@ -87,7 +89,7 @@ export async function deserializeArray(compressedData) {
87
89
  * @internal
88
90
  */
89
91
  export function createDeserialize(compressedData) {
90
- ow(compressedData, ow.uint8Array);
92
+ parseArgument(compressedData, uint8ArraySchema);
91
93
  const streamArray = StreamArray.withParser();
92
94
  const destination = pluckValue(streamArray);
93
95
  streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
@@ -1,8 +1,28 @@
1
- import ow from 'ow';
2
1
  import { CookieJar } from 'tough-cookie';
2
+ import { z } from 'zod';
3
3
  import { cryptoRandomObjectId } from '@apify/utilities';
4
4
  import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
5
5
  import { serviceLocator } from '../service_locator.js';
6
+ import { parseArgument, schemas, validators } from '../validators.js';
7
+ // `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
8
+ // copy), so class instances like cookie jars and loggers keep their prototype.
9
+ const sessionOptionsSchema = z.strictObject({
10
+ id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
11
+ cookieJar: schemas.anyObject.default(() => new CookieJar()),
12
+ proxyInfo: schemas.anyObject.optional(),
13
+ maxAgeSecs: schemas.anyNumber.default(3000),
14
+ userData: schemas.anyObject.default(() => ({})),
15
+ maxErrorScore: schemas.anyNumber.default(3),
16
+ errorScoreDecrement: schemas.anyNumber.default(0.5),
17
+ createdAt: z.date().default(() => new Date()),
18
+ expiresAt: z.date().optional(),
19
+ usageCount: schemas.anyNumber.default(0),
20
+ errorScore: schemas.anyNumber.default(0),
21
+ maxUsageCount: schemas.anyNumber.default(50),
22
+ retired: z.boolean().default(false),
23
+ log: validators.logger.default(() => serviceLocator.getLogger()),
24
+ fingerprint: schemas.anyObject.optional(),
25
+ });
6
26
  /**
7
27
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
8
28
  * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
@@ -68,25 +88,7 @@ export class Session {
68
88
  * Session configuration.
69
89
  */
70
90
  constructor(options = {}) {
71
- ow(options, ow.object.exactShape({
72
- id: ow.optional.string,
73
- cookieJar: ow.optional.object,
74
- proxyInfo: ow.optional.object,
75
- maxAgeSecs: ow.optional.number,
76
- userData: ow.optional.object,
77
- maxErrorScore: ow.optional.number,
78
- errorScoreDecrement: ow.optional.number,
79
- createdAt: ow.optional.date,
80
- expiresAt: ow.optional.date,
81
- usageCount: ow.optional.number,
82
- errorScore: ow.optional.number,
83
- maxUsageCount: ow.optional.number,
84
- retired: ow.optional.boolean,
85
- log: ow.optional.object,
86
- fingerprint: ow.optional.object,
87
- }));
88
- const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
89
- const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
91
+ const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
90
92
  this.#log = log.child({ prefix: 'Session' });
91
93
  this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
92
94
  this.#proxyInfo = proxyInfo;
@@ -1,12 +1,29 @@
1
1
  import { AsyncQueue } from '@sapphire/async-queue';
2
- import ow from 'ow';
2
+ import { z } from 'zod';
3
3
  import { EventType } from '../events/event_manager.js';
4
4
  import { serviceLocator } from '../service_locator.js';
5
5
  import { KeyValueStore } from '../storages/key_value_store.js';
6
+ import { parseArgument, schemas, validators } from '../validators.js';
6
7
  import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
7
8
  import { createDefaultSessionFingerprint } from './fingerprint.js';
8
9
  import { Session } from './session.js';
9
10
  const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
11
+ // `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
12
+ // copy), so class instances like loggers keep their prototype.
13
+ const sessionPoolOptionsSchema = z.strictObject({
14
+ id: z.union([schemas.anyNumber, z.string()]).optional(),
15
+ maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
16
+ persistStateKeyValueStoreId: z.string().optional(),
17
+ persistStateKey: z.string().optional(),
18
+ createSessionFunction: schemas.anyFunction.optional(),
19
+ sessionOptions: schemas.anyObject.default(() => ({})),
20
+ log: validators.logger.default(() => serviceLocator.getLogger()),
21
+ persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
22
+ sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
23
+ });
24
+ const createSessionOptionsSchema = z.strictObject({
25
+ sessionOptions: schemas.anyObject.default(() => ({})),
26
+ });
10
27
  /**
11
28
  * Handles the rotation, creation and persistence of user-like sessions.
12
29
  * Creates a pool of {@link Session} instances, that are randomly rotated.
@@ -81,20 +98,7 @@ export class SessionPool {
81
98
  #queue = new AsyncQueue();
82
99
  #roundRobinIndex = 0;
83
100
  constructor(options = {}) {
84
- ow(options, ow.object.exactShape({
85
- id: ow.optional.any(ow.number, ow.string),
86
- maxPoolSize: ow.optional.number,
87
- persistStateKeyValueStoreId: ow.optional.string,
88
- persistStateKey: ow.optional.string,
89
- createSessionFunction: ow.optional.function,
90
- sessionOptions: ow.optional.object,
91
- log: ow.optional.object,
92
- persistenceOptions: ow.optional.object,
93
- sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
94
- }));
95
- const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
96
- enable: true,
97
- }, sessionReuseStrategy = 'random', } = options;
101
+ const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
98
102
  this.id = id != null ? String(id) : String(SessionPool.#nextId++);
99
103
  this.#sessionReuseStrategy = sessionReuseStrategy;
100
104
  this.#events = serviceLocator.getEventManager();
@@ -303,8 +307,7 @@ export class SessionPool {
303
307
  * @returns New session.
304
308
  */
305
309
  async defaultCreateSessionFunction(options = {}) {
306
- ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
307
- const { sessionOptions = {} } = options;
310
+ const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
308
311
  return new Session(sessionOptions);
309
312
  }
310
313
  /**
@@ -1,12 +1,17 @@
1
- import ow from 'ow';
1
+ import { z } from 'zod';
2
2
  import { tryCancel } from '@apify/timeout';
3
3
  import { Configuration } from '../configuration.js';
4
4
  import { serviceLocator } from '../service_locator.js';
5
+ import { parseArgument, schemas, validators } from '../validators.js';
5
6
  import { activeStorageTransaction, rejectOperationInTransaction, snapshotValue } from './transaction.js';
6
7
  import { KeyValueStore } from './key_value_store.js';
7
8
  import { StorageStatsTracker } from './storage_stats.js';
8
9
  import { resolveStorageIdentifier } from './storage_instance_manager.js';
9
10
  import { createDualIterable, purgeDefaultStorages } from './utils.js';
11
+ const openOptionsSchema = z.strictObject({
12
+ configuration: z.instanceof(Configuration).optional(),
13
+ storageBackend: validators.storageBackend.optional(),
14
+ });
10
15
  /** @internal */
11
16
  export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
12
17
  /**
@@ -122,7 +127,7 @@ export class Dataset {
122
127
  */
123
128
  async pushData(data) {
124
129
  const transaction = activeStorageTransaction();
125
- ow(data, 'data', ow.object);
130
+ parseArgument(data, schemas.anyObject);
126
131
  // Normalize to array and validate each item
127
132
  const items = Array.isArray(data) ? data : [data];
128
133
  for (let i = 0; i < items.length; i++) {
@@ -553,13 +558,10 @@ export class Dataset {
553
558
  */
554
559
  static async open(identifier, options = {}) {
555
560
  tryCancel();
556
- ow(options, ow.object.exactShape({
557
- configuration: ow.optional.object.instanceOf(Configuration),
558
- storageBackend: ow.optional.object,
559
- }));
560
- options.configuration ??= Configuration.getGlobalConfiguration();
561
- const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
562
- await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
561
+ const parsedOptions = parseArgument(options, openOptionsSchema);
562
+ const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
563
+ const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
564
+ await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
563
565
  const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'Dataset');
564
566
  return serviceLocator.getStorageInstanceManager().openStorage(this, {
565
567
  ...resolved,