@crawlee/core 3.0.0-beta.12 → 3.0.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -10
- package/autoscaling/autoscaled_pool.js +1 -1
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +13 -15
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +10 -12
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/events/local_event_manager.d.ts +1 -2
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +1 -2
- package/events/local_event_manager.js.map +1 -1
- package/package.json +5 -5
- package/storages/dataset.d.ts +28 -3
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +31 -3
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +105 -13
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +112 -12
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +5 -5
- package/storages/request_list.js +2 -2
- package/storages/request_queue.d.ts +2 -2
- package/storages/request_queue.js +2 -2
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
1
2
|
import { StorageClient } from '@crawlee/types';
|
|
2
3
|
import { Configuration } from '../configuration';
|
|
3
|
-
import { Awaitable } from '../typedefs';
|
|
4
|
+
import { Awaitable, Dictionary } from '../typedefs';
|
|
4
5
|
import { StorageManagerOptions } from './storage_manager';
|
|
5
6
|
/**
|
|
6
7
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
@@ -14,19 +15,19 @@ export declare const maybeStringify: <T>(value: T, options: {
|
|
|
14
15
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
15
16
|
* for saving and reading data records or files. Each data record is
|
|
16
17
|
* represented by a unique key and associated with a MIME content type. Key-value stores are ideal
|
|
17
|
-
* for saving screenshots,
|
|
18
|
+
* for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
|
|
18
19
|
*
|
|
19
20
|
* Do not instantiate this class directly, use the
|
|
20
21
|
* {@link KeyValueStore.open} function instead.
|
|
21
22
|
*
|
|
22
|
-
* Each
|
|
23
|
-
* for the run. By convention, the
|
|
23
|
+
* Each crawler run is associated with a default key-value store, which is created exclusively
|
|
24
|
+
* for the run. By convention, the crawler input and output are stored into the
|
|
24
25
|
* default key-value store under the `INPUT` and `OUTPUT` key, respectively.
|
|
25
26
|
* Typically, input and output are JSON files, although it can be any other format.
|
|
26
27
|
* To access the default key-value store directly, you can use the
|
|
27
|
-
* {@link
|
|
28
|
+
* {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
|
|
28
29
|
*
|
|
29
|
-
* To access the input, you can also use the {@link
|
|
30
|
+
* To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
|
|
30
31
|
*
|
|
31
32
|
* `KeyValueStore` stores its data either on local disk or in the Apify cloud,
|
|
32
33
|
* depending on whether the [`APIFY_IS_AT_HOME`](/docs/guides/environment-variables#apify_is_at_home)
|
|
@@ -51,13 +52,13 @@ export declare const maybeStringify: <T>(value: T, options: {
|
|
|
51
52
|
* **Example usage:**
|
|
52
53
|
*
|
|
53
54
|
* ```javascript
|
|
54
|
-
* // Get
|
|
55
|
-
* const input = await
|
|
55
|
+
* // Get crawler input from the default key-value store.
|
|
56
|
+
* const input = await KeyValueStore.getInput();
|
|
56
57
|
* // Get some value from the default key-value store.
|
|
57
|
-
* const otherValue = await
|
|
58
|
+
* const otherValue = await KeyValueStore.getValue('my-key');
|
|
58
59
|
*
|
|
59
|
-
* // Write
|
|
60
|
-
* await
|
|
60
|
+
* // Write crawler output to the default key-value store.
|
|
61
|
+
* await KeyValueStore.setValue('OUTPUT', { myResult: 123 });
|
|
61
62
|
*
|
|
62
63
|
* // Open a named key-value store
|
|
63
64
|
* const store = await KeyValueStore.open('some-name');
|
|
@@ -145,7 +146,7 @@ export declare class KeyValueStore {
|
|
|
145
146
|
* {@link KeyValueStore.getValue} function.
|
|
146
147
|
*
|
|
147
148
|
* **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
|
|
148
|
-
* otherwise the
|
|
149
|
+
* otherwise the crawler process might finish before the value is stored!
|
|
149
150
|
*
|
|
150
151
|
* @param key
|
|
151
152
|
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
@@ -203,10 +204,101 @@ export declare class KeyValueStore {
|
|
|
203
204
|
*
|
|
204
205
|
* @param [storeIdOrName]
|
|
205
206
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
206
|
-
* the function returns the default key-value store associated with the
|
|
207
|
+
* the function returns the default key-value store associated with the crawler run.
|
|
207
208
|
* @param [options] Storage manager options.
|
|
208
209
|
*/
|
|
209
210
|
static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
|
|
211
|
+
/**
|
|
212
|
+
* Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
213
|
+
*
|
|
214
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getValue}.
|
|
215
|
+
* For example, calling the following code:
|
|
216
|
+
* ```javascript
|
|
217
|
+
* const value = await KeyValueStore.getValue('my-key');
|
|
218
|
+
* ```
|
|
219
|
+
*
|
|
220
|
+
* is equivalent to:
|
|
221
|
+
* ```javascript
|
|
222
|
+
* const store = await KeyValueStore.open();
|
|
223
|
+
* const value = await store.getValue('my-key');
|
|
224
|
+
* ```
|
|
225
|
+
*
|
|
226
|
+
* To store the value to the default key-value store, you can use the {@link KeyValueStore.setValue} function.
|
|
227
|
+
*
|
|
228
|
+
* For more information, see {@link KeyValueStore.open}
|
|
229
|
+
* and {@link KeyValueStore.getValue}.
|
|
230
|
+
*
|
|
231
|
+
* @param key Unique record key.
|
|
232
|
+
* @returns
|
|
233
|
+
* Returns a promise that resolves to an object, string
|
|
234
|
+
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
235
|
+
* on the MIME content type of the record, or `null`
|
|
236
|
+
* if the record is missing.
|
|
237
|
+
* @ignore
|
|
238
|
+
*/
|
|
239
|
+
static getValue<T = unknown>(key: string): Promise<T | null>;
|
|
240
|
+
/**
|
|
241
|
+
* Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
|
|
242
|
+
*
|
|
243
|
+
* This is just a convenient shortcut for {@link KeyValueStore.setValue}.
|
|
244
|
+
* For example, calling the following code:
|
|
245
|
+
* ```javascript
|
|
246
|
+
* await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
|
|
247
|
+
* ```
|
|
248
|
+
*
|
|
249
|
+
* is equivalent to:
|
|
250
|
+
* ```javascript
|
|
251
|
+
* const store = await KeyValueStore.open();
|
|
252
|
+
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
253
|
+
* ```
|
|
254
|
+
*
|
|
255
|
+
* To get a value from the default key-value store, you can use the {@link KeyValueStore.getValue} function.
|
|
256
|
+
*
|
|
257
|
+
* For more information, see {@link KeyValueStore.open}
|
|
258
|
+
* and {@link KeyValueStore.getValue}.
|
|
259
|
+
*
|
|
260
|
+
* @param key
|
|
261
|
+
* Unique record key.
|
|
262
|
+
* @param value
|
|
263
|
+
* Record data, which can be one of the following values:
|
|
264
|
+
* - If `null`, the record in the key-value store is deleted.
|
|
265
|
+
* - If no `options.contentType` is specified, `value` can be any JavaScript object, and it will be stringified to JSON.
|
|
266
|
+
* - If `options.contentType` is set, `value` is taken as is, and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
|
|
267
|
+
* For any other value an error will be thrown.
|
|
268
|
+
* @param [options]
|
|
269
|
+
* @ignore
|
|
270
|
+
*/
|
|
271
|
+
static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
|
|
272
|
+
/**
|
|
273
|
+
* Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
274
|
+
*
|
|
275
|
+
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
276
|
+
* For example, calling the following code:
|
|
277
|
+
* ```javascript
|
|
278
|
+
* const input = await KeyValueStore.getInput();
|
|
279
|
+
* ```
|
|
280
|
+
*
|
|
281
|
+
* is equivalent to:
|
|
282
|
+
* ```javascript
|
|
283
|
+
* const store = await KeyValueStore.open();
|
|
284
|
+
* await store.getValue('INPUT');
|
|
285
|
+
* ```
|
|
286
|
+
*
|
|
287
|
+
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
288
|
+
* If you need to use the input multiple times in your crawler,
|
|
289
|
+
* it is far more efficient to read it once and store it locally.
|
|
290
|
+
*
|
|
291
|
+
* For more information, see {@link KeyValueStore.open}
|
|
292
|
+
* and {@link KeyValueStore.getValue}.
|
|
293
|
+
*
|
|
294
|
+
* @returns
|
|
295
|
+
* Returns a promise that resolves to an object, string
|
|
296
|
+
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
297
|
+
* on the MIME content type of the record, or `null`
|
|
298
|
+
* if the record is missing.
|
|
299
|
+
* @ignore
|
|
300
|
+
*/
|
|
301
|
+
static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
|
|
210
302
|
}
|
|
211
303
|
/**
|
|
212
304
|
* User-function used in the {@link KeyValueStore.forEachKey} method.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAuB,MAAM,gBAAgB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,aAAa,EAAuB,MAAM,gBAAgB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,aAAa;IAQqB,QAAQ,CAAC,MAAM;IAP1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,MAAM,CAAsB;IAEpC;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAO3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAIpF,WAAW;IAiBzB;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAU7G;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAKlE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAI/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -40,19 +40,19 @@ exports.maybeStringify = maybeStringify;
|
|
|
40
40
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
41
41
|
* for saving and reading data records or files. Each data record is
|
|
42
42
|
* represented by a unique key and associated with a MIME content type. Key-value stores are ideal
|
|
43
|
-
* for saving screenshots,
|
|
43
|
+
* for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
|
|
44
44
|
*
|
|
45
45
|
* Do not instantiate this class directly, use the
|
|
46
46
|
* {@link KeyValueStore.open} function instead.
|
|
47
47
|
*
|
|
48
|
-
* Each
|
|
49
|
-
* for the run. By convention, the
|
|
48
|
+
* Each crawler run is associated with a default key-value store, which is created exclusively
|
|
49
|
+
* for the run. By convention, the crawler input and output are stored into the
|
|
50
50
|
* default key-value store under the `INPUT` and `OUTPUT` key, respectively.
|
|
51
51
|
* Typically, input and output are JSON files, although it can be any other format.
|
|
52
52
|
* To access the default key-value store directly, you can use the
|
|
53
|
-
* {@link
|
|
53
|
+
* {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
|
|
54
54
|
*
|
|
55
|
-
* To access the input, you can also use the {@link
|
|
55
|
+
* To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
|
|
56
56
|
*
|
|
57
57
|
* `KeyValueStore` stores its data either on local disk or in the Apify cloud,
|
|
58
58
|
* depending on whether the [`APIFY_IS_AT_HOME`](/docs/guides/environment-variables#apify_is_at_home)
|
|
@@ -77,13 +77,13 @@ exports.maybeStringify = maybeStringify;
|
|
|
77
77
|
* **Example usage:**
|
|
78
78
|
*
|
|
79
79
|
* ```javascript
|
|
80
|
-
* // Get
|
|
81
|
-
* const input = await
|
|
80
|
+
* // Get crawler input from the default key-value store.
|
|
81
|
+
* const input = await KeyValueStore.getInput();
|
|
82
82
|
* // Get some value from the default key-value store.
|
|
83
|
-
* const otherValue = await
|
|
83
|
+
* const otherValue = await KeyValueStore.getValue('my-key');
|
|
84
84
|
*
|
|
85
|
-
* // Write
|
|
86
|
-
* await
|
|
85
|
+
* // Write crawler output to the default key-value store.
|
|
86
|
+
* await KeyValueStore.setValue('OUTPUT', { myResult: 123 });
|
|
87
87
|
*
|
|
88
88
|
* // Open a named key-value store
|
|
89
89
|
* const store = await KeyValueStore.open('some-name');
|
|
@@ -199,7 +199,7 @@ class KeyValueStore {
|
|
|
199
199
|
* {@link KeyValueStore.getValue} function.
|
|
200
200
|
*
|
|
201
201
|
* **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
|
|
202
|
-
* otherwise the
|
|
202
|
+
* otherwise the crawler process might finish before the value is stored!
|
|
203
203
|
*
|
|
204
204
|
* @param key
|
|
205
205
|
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
@@ -304,7 +304,7 @@ class KeyValueStore {
|
|
|
304
304
|
*
|
|
305
305
|
* @param [storeIdOrName]
|
|
306
306
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
307
|
-
* the function returns the default key-value store associated with the
|
|
307
|
+
* the function returns the default key-value store associated with the crawler run.
|
|
308
308
|
* @param [options] Storage manager options.
|
|
309
309
|
*/
|
|
310
310
|
static async open(storeIdOrName, options = {}) {
|
|
@@ -315,6 +315,106 @@ class KeyValueStore {
|
|
|
315
315
|
const manager = new storage_manager_1.StorageManager(KeyValueStore, options.config);
|
|
316
316
|
return manager.openStorage(storeIdOrName);
|
|
317
317
|
}
|
|
318
|
+
/**
|
|
319
|
+
* Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
320
|
+
*
|
|
321
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getValue}.
|
|
322
|
+
* For example, calling the following code:
|
|
323
|
+
* ```javascript
|
|
324
|
+
* const value = await KeyValueStore.getValue('my-key');
|
|
325
|
+
* ```
|
|
326
|
+
*
|
|
327
|
+
* is equivalent to:
|
|
328
|
+
* ```javascript
|
|
329
|
+
* const store = await KeyValueStore.open();
|
|
330
|
+
* const value = await store.getValue('my-key');
|
|
331
|
+
* ```
|
|
332
|
+
*
|
|
333
|
+
* To store the value to the default key-value store, you can use the {@link KeyValueStore.setValue} function.
|
|
334
|
+
*
|
|
335
|
+
* For more information, see {@link KeyValueStore.open}
|
|
336
|
+
* and {@link KeyValueStore.getValue}.
|
|
337
|
+
*
|
|
338
|
+
* @param key Unique record key.
|
|
339
|
+
* @returns
|
|
340
|
+
* Returns a promise that resolves to an object, string
|
|
341
|
+
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
342
|
+
* on the MIME content type of the record, or `null`
|
|
343
|
+
* if the record is missing.
|
|
344
|
+
* @ignore
|
|
345
|
+
*/
|
|
346
|
+
static async getValue(key) {
|
|
347
|
+
const store = await this.open();
|
|
348
|
+
return store.getValue(key);
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
|
|
352
|
+
*
|
|
353
|
+
* This is just a convenient shortcut for {@link KeyValueStore.setValue}.
|
|
354
|
+
* For example, calling the following code:
|
|
355
|
+
* ```javascript
|
|
356
|
+
* await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
|
|
357
|
+
* ```
|
|
358
|
+
*
|
|
359
|
+
* is equivalent to:
|
|
360
|
+
* ```javascript
|
|
361
|
+
* const store = await KeyValueStore.open();
|
|
362
|
+
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
363
|
+
* ```
|
|
364
|
+
*
|
|
365
|
+
* To get a value from the default key-value store, you can use the {@link KeyValueStore.getValue} function.
|
|
366
|
+
*
|
|
367
|
+
* For more information, see {@link KeyValueStore.open}
|
|
368
|
+
* and {@link KeyValueStore.getValue}.
|
|
369
|
+
*
|
|
370
|
+
* @param key
|
|
371
|
+
* Unique record key.
|
|
372
|
+
* @param value
|
|
373
|
+
* Record data, which can be one of the following values:
|
|
374
|
+
* - If `null`, the record in the key-value store is deleted.
|
|
375
|
+
* - If no `options.contentType` is specified, `value` can be any JavaScript object, and it will be stringified to JSON.
|
|
376
|
+
* - If `options.contentType` is set, `value` is taken as is, and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
|
|
377
|
+
* For any other value an error will be thrown.
|
|
378
|
+
* @param [options]
|
|
379
|
+
* @ignore
|
|
380
|
+
*/
|
|
381
|
+
static async setValue(key, value, options = {}) {
|
|
382
|
+
const store = await this.open();
|
|
383
|
+
return store.setValue(key, value, options);
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
387
|
+
*
|
|
388
|
+
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
389
|
+
* For example, calling the following code:
|
|
390
|
+
* ```javascript
|
|
391
|
+
* const input = await KeyValueStore.getInput();
|
|
392
|
+
* ```
|
|
393
|
+
*
|
|
394
|
+
* is equivalent to:
|
|
395
|
+
* ```javascript
|
|
396
|
+
* const store = await KeyValueStore.open();
|
|
397
|
+
* await store.getValue('INPUT');
|
|
398
|
+
* ```
|
|
399
|
+
*
|
|
400
|
+
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
401
|
+
* If you need to use the input multiple times in your crawler,
|
|
402
|
+
* it is far more efficient to read it once and store it locally.
|
|
403
|
+
*
|
|
404
|
+
* For more information, see {@link KeyValueStore.open}
|
|
405
|
+
* and {@link KeyValueStore.getValue}.
|
|
406
|
+
*
|
|
407
|
+
* @returns
|
|
408
|
+
* Returns a promise that resolves to an object, string
|
|
409
|
+
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
410
|
+
* on the MIME content type of the record, or `null`
|
|
411
|
+
* if the record is missing.
|
|
412
|
+
* @ignore
|
|
413
|
+
*/
|
|
414
|
+
static async getInput() {
|
|
415
|
+
const store = await this.open();
|
|
416
|
+
return store.getValue(store.config.get('inputKey'));
|
|
417
|
+
}
|
|
318
418
|
}
|
|
319
419
|
exports.KeyValueStore = KeyValueStore;
|
|
320
420
|
//# sourceMappingURL=key_value_store.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAEjD,uDAA0E;AAE1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAKtB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAPpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAAoC;QAMhC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,IAAI,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,GAAW;QACpB,uGAAuG;QACvG,OAAO,6CAA6C,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IACjF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAEjD,uDAA0E;AAE1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAKtB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAPpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAAoC;QAMhC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,IAAI,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,GAAW;QACpB,uGAAuG;QACvG,OAAO,6CAA6C,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IACjF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,CAAC,CAAC;IAClC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,CAAC;CACJ;AAtTD,sCAsTC"}
|
|
@@ -49,7 +49,7 @@ export interface RequestListOptions {
|
|
|
49
49
|
* This is very useful in a scenario when getting the sources is a resource intensive or time consuming
|
|
50
50
|
* task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
|
|
51
51
|
* `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
|
|
52
|
-
* fetch and parse those URLs only once, saving valuable time when your
|
|
52
|
+
* fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
|
|
53
53
|
*
|
|
54
54
|
* If both {@link RequestListOptions.sources} and {@link RequestListOptions.sourcesFunction} are provided,
|
|
55
55
|
* the sources returned by the function will be added after the `sources`.
|
|
@@ -67,7 +67,7 @@ export interface RequestListOptions {
|
|
|
67
67
|
*
|
|
68
68
|
* // Sitemaps can change in real-time, so it's important to persist
|
|
69
69
|
* // the URLs we collected. Otherwise we might lose our scraping
|
|
70
|
-
* // state in case of an
|
|
70
|
+
* // state in case of an crawler migration / failure / time-out.
|
|
71
71
|
* const requestList = new RequestList({
|
|
72
72
|
* sourcesFunction,
|
|
73
73
|
* persistStateKey: 'state-key',
|
|
@@ -91,7 +91,7 @@ export interface RequestListOptions {
|
|
|
91
91
|
/**
|
|
92
92
|
* Identifies the key in the default key-value store under which `RequestList` periodically stores its
|
|
93
93
|
* state (i.e. which URLs were crawled and which not).
|
|
94
|
-
* If the
|
|
94
|
+
* If the crawler is restarted, `RequestList` will read the state
|
|
95
95
|
* and continue where it left off.
|
|
96
96
|
*
|
|
97
97
|
* If `persistStateKey` is not set, `RequestList` will always start from the beginning,
|
|
@@ -174,9 +174,9 @@ export interface RequestListOptions {
|
|
|
174
174
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
175
175
|
* {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
176
176
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
177
|
-
* event that is periodically emitted by {@link
|
|
177
|
+
* event that is periodically emitted by {@link EventManager}.
|
|
178
178
|
*
|
|
179
|
-
* The internal state is closely tied to the provided sources (URLs). If the sources change on
|
|
179
|
+
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
180
180
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
181
181
|
* In such case, use the `persistRequestsKey` option in conjunction with `persistStateKey`,
|
|
182
182
|
* to make the `RequestList` store the initial sources to the default key-value store and load them after restart,
|
package/storages/request_list.js
CHANGED
|
@@ -40,9 +40,9 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
|
|
|
40
40
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
41
41
|
* {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
42
42
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
43
|
-
* event that is periodically emitted by {@link
|
|
43
|
+
* event that is periodically emitted by {@link EventManager}.
|
|
44
44
|
*
|
|
45
|
-
* The internal state is closely tied to the provided sources (URLs). If the sources change on
|
|
45
|
+
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
46
46
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
47
47
|
* In such case, use the `persistRequestsKey` option in conjunction with `persistStateKey`,
|
|
48
48
|
* to make the `RequestList` store the initial sources to the default key-value store and load them after restart,
|
|
@@ -92,7 +92,7 @@ export interface RequestQueueOperationOptions {
|
|
|
92
92
|
* **Example usage:**
|
|
93
93
|
*
|
|
94
94
|
* ```javascript
|
|
95
|
-
* // Open the default request queue associated with the
|
|
95
|
+
* // Open the default request queue associated with the crawler run
|
|
96
96
|
* const queue = await RequestQueue.open();
|
|
97
97
|
*
|
|
98
98
|
* // Open a named request queue
|
|
@@ -298,7 +298,7 @@ export declare class RequestQueue {
|
|
|
298
298
|
*
|
|
299
299
|
* @param [queueIdOrName]
|
|
300
300
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
301
|
-
* the function returns the default request queue associated with the
|
|
301
|
+
* the function returns the default request queue associated with the crawler run.
|
|
302
302
|
* @param [options] Open Request Queue options.
|
|
303
303
|
*/
|
|
304
304
|
static open(queueIdOrName?: string | null): Promise<RequestQueue>;
|
|
@@ -93,7 +93,7 @@ exports.getRequestId = getRequestId;
|
|
|
93
93
|
* **Example usage:**
|
|
94
94
|
*
|
|
95
95
|
* ```javascript
|
|
96
|
-
* // Open the default request queue associated with the
|
|
96
|
+
* // Open the default request queue associated with the crawler run
|
|
97
97
|
* const queue = await RequestQueue.open();
|
|
98
98
|
*
|
|
99
99
|
* // Open a named request queue
|
|
@@ -717,7 +717,7 @@ class RequestQueue {
|
|
|
717
717
|
*
|
|
718
718
|
* @param [queueIdOrName]
|
|
719
719
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
720
|
-
* the function returns the default request queue associated with the
|
|
720
|
+
* the function returns the default request queue associated with the crawler run.
|
|
721
721
|
* @param [options] Open Request Queue options.
|
|
722
722
|
*/
|
|
723
723
|
static async open(queueIdOrName) {
|