@crawlee/core 4.0.0-beta.121 → 4.0.0-beta.123
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.js +20 -12
- package/autoscaling/concurrency_system.d.ts +2 -2
- package/autoscaling/concurrency_system.js +31 -20
- package/autoscaling/index.d.ts +1 -1
- package/autoscaling/index.js +1 -1
- package/autoscaling/load_signal.d.ts +7 -6
- package/autoscaling/load_signal.js +2 -1
- package/autoscaling/snapshotter.d.ts +6 -6
- package/autoscaling/snapshotter.js +9 -9
- package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts} +13 -12
- package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js} +11 -11
- package/autoscaling/system_status.d.ts +8 -8
- package/autoscaling/system_status.js +2 -2
- package/configuration.d.ts +15 -15
- package/configuration.js +3 -3
- package/crawlers/crawler_commons.d.ts +8 -54
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +14 -14
- package/debug.js +4 -4
- package/enqueue_links/enqueue_links.d.ts +33 -61
- package/enqueue_links/enqueue_links.js +35 -152
- package/enqueue_links/shared.d.ts +17 -4
- package/enqueue_links/shared.js +28 -1
- package/memory-storage/resource-clients/dataset.js +2 -8
- package/memory-storage/resource-clients/key-value-store.js +23 -26
- package/memory-storage/resource-clients/request-queue.js +9 -22
- package/package.json +7 -8
- package/proxy_configuration.js +10 -6
- package/request.d.ts +2 -2
- package/request.js +44 -31
- package/router.d.ts +5 -5
- package/serialization.js +6 -4
- package/session_pool/session.js +22 -20
- package/session_pool/session_pool.js +20 -17
- package/storages/dataset.js +11 -9
- package/storages/key_value_store.js +30 -27
- package/storages/request_list.d.ts +2 -1
- package/storages/request_list.js +26 -21
- package/storages/request_queue.js +64 -59
- package/storages/sitemap_request_loader.d.ts +1 -1
- package/storages/sitemap_request_loader.js +22 -22
- package/storages/throttling_request_manager.js +11 -9
- package/storages/utils.d.ts +2 -1
- package/validators.d.ts +22 -25
- package/validators.js +13 -25
|
@@ -1,9 +1,25 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import {
|
|
2
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
|
+
import { z } from 'zod';
|
|
3
4
|
import { isStream, toBuffer } from '../utils.js';
|
|
4
5
|
import { BaseClient } from './common/base-client.js';
|
|
5
6
|
import mime from 'mime-types';
|
|
6
7
|
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
|
|
8
|
+
const keySchema = z.string();
|
|
9
|
+
const inputRecordSchema = z.object({
|
|
10
|
+
key: z.string().min(1),
|
|
11
|
+
value: z.union([
|
|
12
|
+
z.null(),
|
|
13
|
+
z.string(),
|
|
14
|
+
z.number(),
|
|
15
|
+
z.instanceof(Buffer),
|
|
16
|
+
z.instanceof(ArrayBuffer),
|
|
17
|
+
schemas.typedArray,
|
|
18
|
+
// only checks the value is an actual object, not null, nor array
|
|
19
|
+
schemas.plainObject,
|
|
20
|
+
]),
|
|
21
|
+
contentType: z.string().min(1).optional(),
|
|
22
|
+
});
|
|
7
23
|
/**
|
|
8
24
|
* Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
|
|
9
25
|
* `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
|
|
@@ -58,13 +74,7 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
58
74
|
this.updateTimestamps(true);
|
|
59
75
|
}
|
|
60
76
|
async listKeys(options = {}) {
|
|
61
|
-
const { prefix, exclusiveStartKey, limit } =
|
|
62
|
-
.object({
|
|
63
|
-
prefix: s.string().optional(),
|
|
64
|
-
exclusiveStartKey: s.string().optional(),
|
|
65
|
-
limit: s.number().int().greaterThan(0).optional(),
|
|
66
|
-
})
|
|
67
|
-
.parse(options);
|
|
77
|
+
const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
|
|
68
78
|
const items = [];
|
|
69
79
|
for (const record of this.#keyValueEntries.values()) {
|
|
70
80
|
const size = Buffer.byteLength(record.value);
|
|
@@ -104,7 +114,7 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
104
114
|
* @param key The key of the record to generate the public URL for.
|
|
105
115
|
*/
|
|
106
116
|
async getPublicUrl(key) {
|
|
107
|
-
|
|
117
|
+
parseArgument(key, keySchema);
|
|
108
118
|
return undefined;
|
|
109
119
|
}
|
|
110
120
|
/**
|
|
@@ -114,11 +124,11 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
114
124
|
* @returns `true` if the record exists, `false` if it does not.
|
|
115
125
|
*/
|
|
116
126
|
async recordExists(key) {
|
|
117
|
-
|
|
127
|
+
parseArgument(key, keySchema);
|
|
118
128
|
return this.#keyValueEntries.has(key);
|
|
119
129
|
}
|
|
120
130
|
async getValue(key) {
|
|
121
|
-
|
|
131
|
+
parseArgument(key, keySchema);
|
|
122
132
|
const entry = this.#keyValueEntries.get(key);
|
|
123
133
|
if (!entry) {
|
|
124
134
|
return undefined;
|
|
@@ -136,20 +146,7 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
136
146
|
return record;
|
|
137
147
|
}
|
|
138
148
|
async setValue(record) {
|
|
139
|
-
|
|
140
|
-
key: s.string().lengthGreaterThan(0),
|
|
141
|
-
value: s.union([
|
|
142
|
-
s.null(),
|
|
143
|
-
s.string(),
|
|
144
|
-
s.number(),
|
|
145
|
-
s.instance(Buffer),
|
|
146
|
-
s.instance(ArrayBuffer),
|
|
147
|
-
s.typedArray(),
|
|
148
|
-
// disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
|
|
149
|
-
s.object({}).setValidationEnabled(false),
|
|
150
|
-
]),
|
|
151
|
-
contentType: s.string().lengthGreaterThan(0).optional(),
|
|
152
|
-
}).parse(record);
|
|
149
|
+
parseArgument(record, inputRecordSchema);
|
|
153
150
|
const { key } = record;
|
|
154
151
|
let { value } = record;
|
|
155
152
|
// The frontend (KeyValueStore codec) serializes the value and resolves its content type
|
|
@@ -180,7 +177,7 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
180
177
|
this.updateTimestamps(true);
|
|
181
178
|
}
|
|
182
179
|
async deleteValue(key) {
|
|
183
|
-
|
|
180
|
+
parseArgument(key, keySchema);
|
|
184
181
|
if (this.#keyValueEntries.has(key)) {
|
|
185
182
|
this.#keyValueEntries.delete(key);
|
|
186
183
|
this.updateTimestamps(true);
|
|
@@ -1,23 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
2
3
|
import { AsyncQueue } from '@sapphire/async-queue';
|
|
3
|
-
import {
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils.js';
|
|
5
6
|
import { BaseClient } from './common/base-client.js';
|
|
6
|
-
const
|
|
7
|
-
.object({
|
|
8
|
-
id: s.string(),
|
|
9
|
-
url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
|
|
10
|
-
uniqueKey: s.string(),
|
|
11
|
-
method: s.string().optional(),
|
|
12
|
-
retryCount: s.number().int().optional(),
|
|
13
|
-
handledAt: s.union([s.string(), s.date().valid()]).optional(),
|
|
14
|
-
})
|
|
15
|
-
.passthrough();
|
|
16
|
-
const requestShapeWithoutId = requestShape.omit(['id']);
|
|
17
|
-
const batchRequestShapeWithoutId = requestShapeWithoutId.array();
|
|
18
|
-
const requestOptionsShape = s.object({
|
|
19
|
-
forefront: s.boolean().optional(),
|
|
20
|
-
});
|
|
7
|
+
const uniqueKeySchema = z.string();
|
|
21
8
|
export class RequestQueueBackend extends BaseClient {
|
|
22
9
|
name;
|
|
23
10
|
/**
|
|
@@ -183,8 +170,8 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
183
170
|
}
|
|
184
171
|
}
|
|
185
172
|
async addBatchOfRequests(requests, options = {}) {
|
|
186
|
-
|
|
187
|
-
|
|
173
|
+
parseArgument(requests, schemas.storageRequestBatch);
|
|
174
|
+
parseArgument(options, schemas.requestQueueOperationOptions);
|
|
188
175
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
|
|
189
176
|
// shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
|
|
190
177
|
// concurrent operation interleaving at one of the `await` points below.
|
|
@@ -233,14 +220,14 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
233
220
|
}
|
|
234
221
|
}
|
|
235
222
|
async getRequest(uniqueKey) {
|
|
236
|
-
|
|
223
|
+
parseArgument(uniqueKey, uniqueKeySchema);
|
|
237
224
|
this.updateTimestamps(false);
|
|
238
225
|
const id = uniqueKeyToRequestId(uniqueKey);
|
|
239
226
|
const json = this.#requests.get(id)?.json;
|
|
240
227
|
return this.jsonToRequest(json);
|
|
241
228
|
}
|
|
242
229
|
async markRequestAsHandled(request) {
|
|
243
|
-
|
|
230
|
+
parseArgument(request, schemas.storageRequest);
|
|
244
231
|
this.updateTimestamps(false);
|
|
245
232
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
|
|
246
233
|
// `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
|
|
@@ -278,8 +265,8 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
278
265
|
}
|
|
279
266
|
}
|
|
280
267
|
async reclaimRequest(request, options = {}) {
|
|
281
|
-
|
|
282
|
-
|
|
268
|
+
parseArgument(request, schemas.storageRequest);
|
|
269
|
+
parseArgument(options, schemas.requestQueueOperationOptions);
|
|
283
270
|
this.updateTimestamps(false);
|
|
284
271
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
|
|
285
272
|
// `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.123",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -52,24 +52,23 @@
|
|
|
52
52
|
"@apify/log": "^2.5.18",
|
|
53
53
|
"@apify/timeout": "^0.4.4",
|
|
54
54
|
"@apify/utilities": "^2.15.5",
|
|
55
|
-
"@crawlee/fs-storage": "4.0.0-beta.
|
|
56
|
-
"@crawlee/
|
|
57
|
-
"@crawlee/
|
|
55
|
+
"@crawlee/fs-storage": "4.0.0-beta.123",
|
|
56
|
+
"@crawlee/http-client": "4.0.0-beta.123",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.123",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.123",
|
|
58
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
59
|
-
"@sapphire/shapeshift": "^4.0.0",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"content-type": "^1.0.5",
|
|
62
62
|
"csv-stringify": "^6.5.2",
|
|
63
63
|
"json5": "^2.2.3",
|
|
64
64
|
"mime-types": "^3.0.1",
|
|
65
65
|
"minimatch": "^10.0.1",
|
|
66
|
-
"ow": "^2.0.0",
|
|
67
66
|
"stream-json": "^1.9.1",
|
|
68
67
|
"tldts": "^7.0.6",
|
|
69
68
|
"tough-cookie": "^6.0.0",
|
|
70
69
|
"tslib": "^2.8.1",
|
|
71
70
|
"type-fest": "^4.41.0",
|
|
72
|
-
"zod": "^4.
|
|
71
|
+
"zod": "^4.4.3"
|
|
73
72
|
},
|
|
74
73
|
"lerna": {
|
|
75
74
|
"command": {
|
|
@@ -78,5 +77,5 @@
|
|
|
78
77
|
}
|
|
79
78
|
}
|
|
80
79
|
},
|
|
81
|
-
"gitHead": "
|
|
80
|
+
"gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
|
|
82
81
|
}
|
package/proxy_configuration.js
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { parseArgument, schemas } from './validators.js';
|
|
3
|
+
const proxyConfigurationOptionsSchema = z.strictObject({
|
|
4
|
+
proxyUrls: z
|
|
5
|
+
.array(z.union([z.url(), z.null()]))
|
|
6
|
+
.nonempty()
|
|
7
|
+
.optional(),
|
|
8
|
+
newUrlFunction: schemas.anyFunction.optional(),
|
|
9
|
+
});
|
|
2
10
|
/**
|
|
3
11
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
4
12
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -58,11 +66,7 @@ export class ProxyConfiguration {
|
|
|
58
66
|
throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
|
|
59
67
|
'See the v4 upgrading guide for the recommended migration to named sessions.');
|
|
60
68
|
}
|
|
61
|
-
|
|
62
|
-
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
|
|
63
|
-
newUrlFunction: ow.optional.function,
|
|
64
|
-
}));
|
|
65
|
-
const { proxyUrls, newUrlFunction } = options;
|
|
69
|
+
const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
|
|
66
70
|
if (proxyUrls && newUrlFunction)
|
|
67
71
|
this.throwCannotCombineCustomMethods();
|
|
68
72
|
if (!proxyUrls && !newUrlFunction && validateRequired)
|
package/request.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
2
|
import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
|
|
3
|
-
import type {
|
|
3
|
+
import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
|
|
4
4
|
import type { SkippedRequestReason } from './enqueue_links/shared.js';
|
|
5
5
|
export declare enum RequestState {
|
|
6
6
|
UNPROCESSED = 0,
|
|
@@ -281,7 +281,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
281
281
|
/** @internal */
|
|
282
282
|
lockExpiresAt?: Date;
|
|
283
283
|
/** @internal */
|
|
284
|
-
enqueueStrategy?:
|
|
284
|
+
enqueueStrategy?: EnqueueStrategyOption;
|
|
285
285
|
}
|
|
286
286
|
export interface PushErrorMessageOptions {
|
|
287
287
|
/**
|
package/request.js
CHANGED
|
@@ -1,32 +1,13 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
|
|
5
5
|
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
uniqueKey: ow.optional.string,
|
|
12
|
-
method: ow.optional.string,
|
|
13
|
-
payload: ow.optional.any(ow.string, ow.uint8Array),
|
|
14
|
-
noRetry: ow.optional.boolean,
|
|
15
|
-
retryCount: ow.optional.number,
|
|
16
|
-
sessionId: ow.optional.string,
|
|
17
|
-
maxRetries: ow.optional.number,
|
|
18
|
-
errorMessages: ow.optional.array.ofType(ow.string),
|
|
19
|
-
headers: ow.optional.object,
|
|
20
|
-
userData: ow.optional.object,
|
|
21
|
-
label: ow.optional.string,
|
|
22
|
-
handledAt: ow.optional.any(ow.string.date, ow.date),
|
|
23
|
-
keepUrlFragment: ow.optional.boolean,
|
|
24
|
-
useExtendedUniqueKey: ow.optional.boolean,
|
|
25
|
-
alwaysEnqueue: ow.optional.boolean,
|
|
26
|
-
skipNavigation: ow.optional.boolean,
|
|
27
|
-
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
28
|
-
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
29
|
-
};
|
|
7
|
+
import { parseArgument, schemas } from './validators.js';
|
|
8
|
+
const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
|
|
9
|
+
message: 'Invalid input: expected a date string',
|
|
10
|
+
});
|
|
30
11
|
export var RequestState;
|
|
31
12
|
(function (RequestState) {
|
|
32
13
|
RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
|
|
@@ -38,6 +19,34 @@ export var RequestState;
|
|
|
38
19
|
RequestState[RequestState["ERROR"] = 6] = "ERROR";
|
|
39
20
|
RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
|
|
40
21
|
})(RequestState || (RequestState = {}));
|
|
22
|
+
const requestUrlSchema = z.object({ url: z.string() });
|
|
23
|
+
// new properties on the Request object breaks serialization
|
|
24
|
+
const requestOptionalSchemaShapes = {
|
|
25
|
+
id: z.string().optional(),
|
|
26
|
+
loadedUrl: z.url().optional(),
|
|
27
|
+
uniqueKey: z.string().optional(),
|
|
28
|
+
method: z.string().optional(),
|
|
29
|
+
payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
|
|
30
|
+
noRetry: z.boolean().optional(),
|
|
31
|
+
retryCount: schemas.anyNumber.optional(),
|
|
32
|
+
sessionId: z.string().optional(),
|
|
33
|
+
maxRetries: schemas.anyNumber.optional(),
|
|
34
|
+
errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
|
|
35
|
+
headers: z.looseObject({}).optional(),
|
|
36
|
+
userData: z.looseObject({}).optional(),
|
|
37
|
+
label: z.string().optional(),
|
|
38
|
+
handledAt: z.union([dateString, z.date()]).optional(),
|
|
39
|
+
keepUrlFragment: z.boolean().optional(),
|
|
40
|
+
useExtendedUniqueKey: z.boolean().optional(),
|
|
41
|
+
alwaysEnqueue: z.boolean().optional(),
|
|
42
|
+
skipNavigation: z.boolean().optional(),
|
|
43
|
+
crawlDepth: schemas.anyNumber
|
|
44
|
+
.refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
|
|
45
|
+
.optional(),
|
|
46
|
+
state: z.enum(RequestState).optional(),
|
|
47
|
+
};
|
|
48
|
+
// Each schema is wrapped in a single-key object so validation errors carry the property name.
|
|
49
|
+
const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
|
|
41
50
|
/**
|
|
42
51
|
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
43
52
|
* The `Request` object also stores information about errors that occurred during processing of the request.
|
|
@@ -118,22 +127,26 @@ class CrawleeRequest {
|
|
|
118
127
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
119
128
|
*/
|
|
120
129
|
constructor(options) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
130
|
+
// A bare URL is a common slip — point at the object form instead of a generic type error.
|
|
131
|
+
if (typeof options === 'string') {
|
|
132
|
+
throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
|
|
133
|
+
'Did you mean `new Request({ url })`?');
|
|
134
|
+
}
|
|
135
|
+
parseArgument(options, schemas.anyObject, 'RequestOptions');
|
|
136
|
+
parseArgument(options, requestUrlSchema, 'RequestOptions');
|
|
137
|
+
// Full-shape validation is slow, because it checks all predicates
|
|
124
138
|
// even if the validated object has only 1 property.
|
|
125
139
|
// This custom validation loop iterates only over existing
|
|
126
140
|
// properties and speeds up the validation cca 3-fold.
|
|
127
|
-
// See https://github.com/sindresorhus/ow/issues/193
|
|
128
141
|
keys(options).forEach((prop) => {
|
|
129
142
|
// skip url, because it is validated above
|
|
130
143
|
if (prop === 'url') {
|
|
131
144
|
return;
|
|
132
145
|
}
|
|
133
|
-
const
|
|
146
|
+
const schema = requestOptionalSchemas[prop];
|
|
134
147
|
const value = options[prop];
|
|
135
|
-
if (
|
|
136
|
-
|
|
148
|
+
if (schema) {
|
|
149
|
+
parseArgument({ [prop]: value }, schema, 'RequestOptions');
|
|
137
150
|
}
|
|
138
151
|
});
|
|
139
152
|
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
package/router.d.ts
CHANGED
|
@@ -65,7 +65,7 @@ export declare function validateUserData(label: string | symbol, schema: Standar
|
|
|
65
65
|
* `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
|
|
66
66
|
*/
|
|
67
67
|
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
|
|
68
|
-
export interface RouterHandler<Context extends
|
|
68
|
+
export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
|
|
69
69
|
(ctx: Context): Awaitable<void>;
|
|
70
70
|
}
|
|
71
71
|
export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
|
|
@@ -214,7 +214,7 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
|
|
|
214
214
|
* });
|
|
215
215
|
* ```
|
|
216
216
|
*/
|
|
217
|
-
export declare class Router<Context extends
|
|
217
|
+
export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
|
|
218
218
|
#private;
|
|
219
219
|
/**
|
|
220
220
|
* use Router.create() instead!
|
|
@@ -299,8 +299,8 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
|
|
|
299
299
|
* await crawler.run();
|
|
300
300
|
* ```
|
|
301
301
|
*/
|
|
302
|
-
static create<Context extends
|
|
303
|
-
static create<Context extends
|
|
304
|
-
static create<Context extends
|
|
302
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
303
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
304
|
+
static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
305
305
|
}
|
|
306
306
|
export {};
|
package/serialization.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import zlib from 'node:zlib';
|
|
4
|
-
import ow from 'ow';
|
|
5
4
|
import StreamArray from 'stream-json/streamers/StreamArray.js';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import { parseArgument, schemas } from './validators.js';
|
|
6
7
|
const pipeline = util.promisify(streamPipeline);
|
|
8
|
+
const uint8ArraySchema = z.instanceof(Uint8Array);
|
|
7
9
|
/**
|
|
8
10
|
* Transforms an array of items to a JSON in a streaming
|
|
9
11
|
* fashion to save memory. It operates in batches to speed
|
|
@@ -57,7 +59,7 @@ class ArrayToJson extends Readable {
|
|
|
57
59
|
* @internal
|
|
58
60
|
*/
|
|
59
61
|
export async function serializeArray(data) {
|
|
60
|
-
|
|
62
|
+
parseArgument(data, schemas.anyArray);
|
|
61
63
|
const { chunks, collector } = createChunkCollector();
|
|
62
64
|
await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
|
|
63
65
|
return Buffer.concat(chunks);
|
|
@@ -72,7 +74,7 @@ export async function serializeArray(data) {
|
|
|
72
74
|
* @internal
|
|
73
75
|
*/
|
|
74
76
|
export async function deserializeArray(compressedData) {
|
|
75
|
-
|
|
77
|
+
parseArgument(compressedData, uint8ArraySchema);
|
|
76
78
|
const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
|
|
77
79
|
await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
|
|
78
80
|
return chunks;
|
|
@@ -87,7 +89,7 @@ export async function deserializeArray(compressedData) {
|
|
|
87
89
|
* @internal
|
|
88
90
|
*/
|
|
89
91
|
export function createDeserialize(compressedData) {
|
|
90
|
-
|
|
92
|
+
parseArgument(compressedData, uint8ArraySchema);
|
|
91
93
|
const streamArray = StreamArray.withParser();
|
|
92
94
|
const destination = pluckValue(streamArray);
|
|
93
95
|
streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
|
package/session_pool/session.js
CHANGED
|
@@ -1,8 +1,28 @@
|
|
|
1
|
-
import ow from 'ow';
|
|
2
1
|
import { CookieJar } from 'tough-cookie';
|
|
2
|
+
import { z } from 'zod';
|
|
3
3
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
4
4
|
import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
|
|
5
5
|
import { serviceLocator } from '../service_locator.js';
|
|
6
|
+
import { parseArgument, schemas, validators } from '../validators.js';
|
|
7
|
+
// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
|
|
8
|
+
// copy), so class instances like cookie jars and loggers keep their prototype.
|
|
9
|
+
const sessionOptionsSchema = z.strictObject({
|
|
10
|
+
id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
|
|
11
|
+
cookieJar: schemas.anyObject.default(() => new CookieJar()),
|
|
12
|
+
proxyInfo: schemas.anyObject.optional(),
|
|
13
|
+
maxAgeSecs: schemas.anyNumber.default(3000),
|
|
14
|
+
userData: schemas.anyObject.default(() => ({})),
|
|
15
|
+
maxErrorScore: schemas.anyNumber.default(3),
|
|
16
|
+
errorScoreDecrement: schemas.anyNumber.default(0.5),
|
|
17
|
+
createdAt: z.date().default(() => new Date()),
|
|
18
|
+
expiresAt: z.date().optional(),
|
|
19
|
+
usageCount: schemas.anyNumber.default(0),
|
|
20
|
+
errorScore: schemas.anyNumber.default(0),
|
|
21
|
+
maxUsageCount: schemas.anyNumber.default(50),
|
|
22
|
+
retired: z.boolean().default(false),
|
|
23
|
+
log: validators.logger.default(() => serviceLocator.getLogger()),
|
|
24
|
+
fingerprint: schemas.anyObject.optional(),
|
|
25
|
+
});
|
|
6
26
|
/**
|
|
7
27
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
8
28
|
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
@@ -68,25 +88,7 @@ export class Session {
|
|
|
68
88
|
* Session configuration.
|
|
69
89
|
*/
|
|
70
90
|
constructor(options = {}) {
|
|
71
|
-
|
|
72
|
-
id: ow.optional.string,
|
|
73
|
-
cookieJar: ow.optional.object,
|
|
74
|
-
proxyInfo: ow.optional.object,
|
|
75
|
-
maxAgeSecs: ow.optional.number,
|
|
76
|
-
userData: ow.optional.object,
|
|
77
|
-
maxErrorScore: ow.optional.number,
|
|
78
|
-
errorScoreDecrement: ow.optional.number,
|
|
79
|
-
createdAt: ow.optional.date,
|
|
80
|
-
expiresAt: ow.optional.date,
|
|
81
|
-
usageCount: ow.optional.number,
|
|
82
|
-
errorScore: ow.optional.number,
|
|
83
|
-
maxUsageCount: ow.optional.number,
|
|
84
|
-
retired: ow.optional.boolean,
|
|
85
|
-
log: ow.optional.object,
|
|
86
|
-
fingerprint: ow.optional.object,
|
|
87
|
-
}));
|
|
88
|
-
const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
|
|
89
|
-
const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
|
|
91
|
+
const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
|
|
90
92
|
this.#log = log.child({ prefix: 'Session' });
|
|
91
93
|
this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
92
94
|
this.#proxyInfo = proxyInfo;
|
|
@@ -1,12 +1,29 @@
|
|
|
1
1
|
import { AsyncQueue } from '@sapphire/async-queue';
|
|
2
|
-
import
|
|
2
|
+
import { z } from 'zod';
|
|
3
3
|
import { EventType } from '../events/event_manager.js';
|
|
4
4
|
import { serviceLocator } from '../service_locator.js';
|
|
5
5
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
6
|
+
import { parseArgument, schemas, validators } from '../validators.js';
|
|
6
7
|
import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
|
|
7
8
|
import { createDefaultSessionFingerprint } from './fingerprint.js';
|
|
8
9
|
import { Session } from './session.js';
|
|
9
10
|
const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
|
|
11
|
+
// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
|
|
12
|
+
// copy), so class instances like loggers keep their prototype.
|
|
13
|
+
const sessionPoolOptionsSchema = z.strictObject({
|
|
14
|
+
id: z.union([schemas.anyNumber, z.string()]).optional(),
|
|
15
|
+
maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
|
|
16
|
+
persistStateKeyValueStoreId: z.string().optional(),
|
|
17
|
+
persistStateKey: z.string().optional(),
|
|
18
|
+
createSessionFunction: schemas.anyFunction.optional(),
|
|
19
|
+
sessionOptions: schemas.anyObject.default(() => ({})),
|
|
20
|
+
log: validators.logger.default(() => serviceLocator.getLogger()),
|
|
21
|
+
persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
|
|
22
|
+
sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
|
|
23
|
+
});
|
|
24
|
+
const createSessionOptionsSchema = z.strictObject({
|
|
25
|
+
sessionOptions: schemas.anyObject.default(() => ({})),
|
|
26
|
+
});
|
|
10
27
|
/**
|
|
11
28
|
* Handles the rotation, creation and persistence of user-like sessions.
|
|
12
29
|
* Creates a pool of {@link Session} instances, that are randomly rotated.
|
|
@@ -81,20 +98,7 @@ export class SessionPool {
|
|
|
81
98
|
#queue = new AsyncQueue();
|
|
82
99
|
#roundRobinIndex = 0;
|
|
83
100
|
constructor(options = {}) {
|
|
84
|
-
|
|
85
|
-
id: ow.optional.any(ow.number, ow.string),
|
|
86
|
-
maxPoolSize: ow.optional.number,
|
|
87
|
-
persistStateKeyValueStoreId: ow.optional.string,
|
|
88
|
-
persistStateKey: ow.optional.string,
|
|
89
|
-
createSessionFunction: ow.optional.function,
|
|
90
|
-
sessionOptions: ow.optional.object,
|
|
91
|
-
log: ow.optional.object,
|
|
92
|
-
persistenceOptions: ow.optional.object,
|
|
93
|
-
sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
|
|
94
|
-
}));
|
|
95
|
-
const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
|
|
96
|
-
enable: true,
|
|
97
|
-
}, sessionReuseStrategy = 'random', } = options;
|
|
101
|
+
const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
|
|
98
102
|
this.id = id != null ? String(id) : String(SessionPool.#nextId++);
|
|
99
103
|
this.#sessionReuseStrategy = sessionReuseStrategy;
|
|
100
104
|
this.#events = serviceLocator.getEventManager();
|
|
@@ -303,8 +307,7 @@ export class SessionPool {
|
|
|
303
307
|
* @returns New session.
|
|
304
308
|
*/
|
|
305
309
|
async defaultCreateSessionFunction(options = {}) {
|
|
306
|
-
|
|
307
|
-
const { sessionOptions = {} } = options;
|
|
310
|
+
const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
|
|
308
311
|
return new Session(sessionOptions);
|
|
309
312
|
}
|
|
310
313
|
/**
|
package/storages/dataset.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { z } from 'zod';
|
|
2
2
|
import { tryCancel } from '@apify/timeout';
|
|
3
3
|
import { Configuration } from '../configuration.js';
|
|
4
4
|
import { serviceLocator } from '../service_locator.js';
|
|
5
|
+
import { parseArgument, schemas, validators } from '../validators.js';
|
|
5
6
|
import { activeStorageTransaction, rejectOperationInTransaction, snapshotValue } from './transaction.js';
|
|
6
7
|
import { KeyValueStore } from './key_value_store.js';
|
|
7
8
|
import { StorageStatsTracker } from './storage_stats.js';
|
|
8
9
|
import { resolveStorageIdentifier } from './storage_instance_manager.js';
|
|
9
10
|
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
11
|
+
const openOptionsSchema = z.strictObject({
|
|
12
|
+
configuration: z.instanceof(Configuration).optional(),
|
|
13
|
+
storageBackend: validators.storageBackend.optional(),
|
|
14
|
+
});
|
|
10
15
|
/** @internal */
|
|
11
16
|
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
12
17
|
/**
|
|
@@ -122,7 +127,7 @@ export class Dataset {
|
|
|
122
127
|
*/
|
|
123
128
|
async pushData(data) {
|
|
124
129
|
const transaction = activeStorageTransaction();
|
|
125
|
-
|
|
130
|
+
parseArgument(data, schemas.anyObject);
|
|
126
131
|
// Normalize to array and validate each item
|
|
127
132
|
const items = Array.isArray(data) ? data : [data];
|
|
128
133
|
for (let i = 0; i < items.length; i++) {
|
|
@@ -553,13 +558,10 @@ export class Dataset {
|
|
|
553
558
|
*/
|
|
554
559
|
static async open(identifier, options = {}) {
|
|
555
560
|
tryCancel();
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
})
|
|
560
|
-
options.configuration ??= Configuration.getGlobalConfiguration();
|
|
561
|
-
const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
|
|
562
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
|
|
561
|
+
const parsedOptions = parseArgument(options, openOptionsSchema);
|
|
562
|
+
const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
|
|
563
|
+
const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
|
|
564
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
|
|
563
565
|
const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'Dataset');
|
|
564
566
|
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
565
567
|
...resolved,
|