@crawlee/core 3.0.3-beta.0 → 3.0.3-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +16 -16
- package/autoscaling/autoscaled_pool.js +13 -13
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/system_status.d.ts +12 -12
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +131 -7
- package/configuration.d.ts.map +1 -1
- package/configuration.js +27 -7
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -13
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +6 -5
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +2 -2
- package/package.json +4 -4
- package/proxy_configuration.d.ts +11 -11
- package/proxy_configuration.js +8 -8
- package/request.d.ts +3 -3
- package/request.js +2 -2
- package/session_pool/session.d.ts +1 -1
- package/session_pool/session_pool.d.ts +12 -12
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +10 -10
- package/storages/dataset.d.ts +14 -14
- package/storages/dataset.js +8 -8
- package/storages/key_value_store.d.ts +28 -37
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +24 -33
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +33 -33
- package/storages/request_list.js +23 -23
- package/storages/request_queue.d.ts +25 -26
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +23 -24
- package/storages/request_queue.js.map +1 -1
- package/storages/utils.d.ts +1 -1
- package/storages/utils.js +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
package/storages/dataset.js
CHANGED
|
@@ -87,7 +87,7 @@ exports.chunkBySize = chunkBySize;
|
|
|
87
87
|
* Typically it is used to store crawling results.
|
|
88
88
|
*
|
|
89
89
|
* Do not instantiate this class directly, use the
|
|
90
|
-
* {@
|
|
90
|
+
* {@apilink Dataset.open} function instead.
|
|
91
91
|
*
|
|
92
92
|
* `Dataset` stores its data either on local disk or in the Apify cloud,
|
|
93
93
|
* depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.
|
|
@@ -104,7 +104,7 @@ exports.chunkBySize = chunkBySize;
|
|
|
104
104
|
* If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` not, the data is stored in the
|
|
105
105
|
* [Apify Dataset](https://docs.apify.com/storage/dataset)
|
|
106
106
|
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
107
|
-
* option to {@
|
|
107
|
+
* option to {@apilink Dataset.open} function,
|
|
108
108
|
* even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
|
|
109
109
|
*
|
|
110
110
|
* **Example usage:**
|
|
@@ -208,7 +208,7 @@ class Dataset {
|
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
210
|
/**
|
|
211
|
-
* Returns {@
|
|
211
|
+
* Returns {@apilink DatasetContent} object holding the items in the dataset based on the provided parameters.
|
|
212
212
|
*/
|
|
213
213
|
async getData(options = {}) {
|
|
214
214
|
try {
|
|
@@ -343,13 +343,13 @@ class Dataset {
|
|
|
343
343
|
manager.closeStorage(this);
|
|
344
344
|
}
|
|
345
345
|
/**
|
|
346
|
-
* Opens a dataset and returns a promise resolving to an instance of the {@
|
|
346
|
+
* Opens a dataset and returns a promise resolving to an instance of the {@apilink Dataset} class.
|
|
347
347
|
*
|
|
348
348
|
* Datasets are used to store structured data where each object stored has the same attributes,
|
|
349
349
|
* such as online store products or real estate offers.
|
|
350
350
|
* The actual data is stored either on the local filesystem or in the cloud.
|
|
351
351
|
*
|
|
352
|
-
* For more details and code examples, see the {@
|
|
352
|
+
* For more details and code examples, see the {@apilink Dataset} class.
|
|
353
353
|
*
|
|
354
354
|
* @param [datasetIdOrName]
|
|
355
355
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
@@ -367,9 +367,9 @@ class Dataset {
|
|
|
367
367
|
return manager.openStorage(datasetIdOrName, options.config.getStorageClient());
|
|
368
368
|
}
|
|
369
369
|
/**
|
|
370
|
-
* Stores an object or an array of objects to the default {@
|
|
370
|
+
* Stores an object or an array of objects to the default {@apilink Dataset} of the current crawler run.
|
|
371
371
|
*
|
|
372
|
-
* This is just a convenient shortcut for {@
|
|
372
|
+
* This is just a convenient shortcut for {@apilink Dataset.pushData}.
|
|
373
373
|
* For example, calling the following code:
|
|
374
374
|
* ```javascript
|
|
375
375
|
* await Dataset.pushData({ myValue: 123 });
|
|
@@ -381,7 +381,7 @@ class Dataset {
|
|
|
381
381
|
* await dataset.pushData({ myValue: 123 });
|
|
382
382
|
* ```
|
|
383
383
|
*
|
|
384
|
-
* For more information, see {@
|
|
384
|
+
* For more information, see {@apilink Dataset.open} and {@apilink Dataset.pushData}
|
|
385
385
|
*
|
|
386
386
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
387
387
|
* otherwise the crawler process might finish before the data are stored!
|
|
@@ -18,37 +18,28 @@ export declare const maybeStringify: <T>(value: T, options: {
|
|
|
18
18
|
* for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
|
|
19
19
|
*
|
|
20
20
|
* Do not instantiate this class directly, use the
|
|
21
|
-
* {@
|
|
21
|
+
* {@apilink KeyValueStore.open} function instead.
|
|
22
22
|
*
|
|
23
23
|
* Each crawler run is associated with a default key-value store, which is created exclusively
|
|
24
24
|
* for the run. By convention, the crawler input and output are stored into the
|
|
25
25
|
* default key-value store under the `INPUT` and `OUTPUT` key, respectively.
|
|
26
26
|
* Typically, input and output are JSON files, although it can be any other format.
|
|
27
27
|
* To access the default key-value store directly, you can use the
|
|
28
|
-
* {@
|
|
28
|
+
* {@apilink KeyValueStore.getValue} and {@apilink KeyValueStore.setValue} convenience functions.
|
|
29
29
|
*
|
|
30
|
-
* To access the input, you can also use the {@
|
|
30
|
+
* To access the input, you can also use the {@apilink KeyValueStore.getInput} convenience function.
|
|
31
31
|
*
|
|
32
|
-
* `KeyValueStore` stores its data
|
|
33
|
-
* depending on whether the [`APIFY_IS_AT_HOME`](/docs/guides/environment-variables#apify_is_at_home)
|
|
34
|
-
* environment variable is set.
|
|
32
|
+
* `KeyValueStore` stores its data on a local disk.
|
|
35
33
|
*
|
|
36
|
-
* If the `
|
|
34
|
+
* If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
|
|
37
35
|
* the local directory in the following files:
|
|
38
36
|
* ```
|
|
39
|
-
* {
|
|
37
|
+
* {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
|
|
40
38
|
* ```
|
|
41
39
|
* Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
|
|
42
|
-
* unless you override it by setting the `
|
|
40
|
+
* unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
|
|
43
41
|
* The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
|
|
44
42
|
*
|
|
45
|
-
* If the [`APIFY_TOKEN`](/docs/guides/environment-variables#apify_token) environment variable is set but
|
|
46
|
-
* [`APIFY_LOCAL_STORAGE_DIR`](/docs/guides/environment-variables#apify_local_storage_dir) not,
|
|
47
|
-
* the data is stored in the [Apify Key-value store](https://docs.apify.com/storage/key-value-store)
|
|
48
|
-
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
49
|
-
* option to {@link KeyValueStore.open} function, even if the
|
|
50
|
-
* [`APIFY_LOCAL_STORAGE_DIR`](/docs/guides/environment-variables#apify_local_storage_dir) variable is set.
|
|
51
|
-
*
|
|
52
43
|
* **Example usage:**
|
|
53
44
|
*
|
|
54
45
|
* ```javascript
|
|
@@ -102,7 +93,7 @@ export declare class KeyValueStore {
|
|
|
102
93
|
* If the record does not exist, the function resolves to `null`.
|
|
103
94
|
*
|
|
104
95
|
* To save or delete a value in the key-value store, use the
|
|
105
|
-
* {@
|
|
96
|
+
* {@apilink KeyValueStore.setValue} function.
|
|
106
97
|
*
|
|
107
98
|
* **Example usage:**
|
|
108
99
|
*
|
|
@@ -150,7 +141,7 @@ export declare class KeyValueStore {
|
|
|
150
141
|
* regardless whether the record existed or not.
|
|
151
142
|
*
|
|
152
143
|
* To retrieve a value from the key-value store, use the
|
|
153
|
-
* {@
|
|
144
|
+
* {@apilink KeyValueStore.getValue} function.
|
|
154
145
|
*
|
|
155
146
|
* **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
|
|
156
147
|
* otherwise the crawler process might finish before the value is stored!
|
|
@@ -198,13 +189,13 @@ export declare class KeyValueStore {
|
|
|
198
189
|
forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
|
|
199
190
|
private _forEachKey;
|
|
200
191
|
/**
|
|
201
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
192
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
|
|
202
193
|
*
|
|
203
194
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
204
195
|
* The records are stored and retrieved using a unique key.
|
|
205
196
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
206
197
|
*
|
|
207
|
-
* For more details and code examples, see the {@
|
|
198
|
+
* For more details and code examples, see the {@apilink KeyValueStore} class.
|
|
208
199
|
*
|
|
209
200
|
* @param [storeIdOrName]
|
|
210
201
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -213,9 +204,9 @@ export declare class KeyValueStore {
|
|
|
213
204
|
*/
|
|
214
205
|
static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
|
|
215
206
|
/**
|
|
216
|
-
* Gets a value from the default {@
|
|
207
|
+
* Gets a value from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
217
208
|
*
|
|
218
|
-
* This is just a convenient shortcut for {@
|
|
209
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
|
|
219
210
|
* For example, calling the following code:
|
|
220
211
|
* ```javascript
|
|
221
212
|
* const value = await KeyValueStore.getValue('my-key');
|
|
@@ -227,10 +218,10 @@ export declare class KeyValueStore {
|
|
|
227
218
|
* const value = await store.getValue('my-key');
|
|
228
219
|
* ```
|
|
229
220
|
*
|
|
230
|
-
* To store the value to the default key-value store, you can use the {@
|
|
221
|
+
* To store the value to the default key-value store, you can use the {@apilink KeyValueStore.setValue} function.
|
|
231
222
|
*
|
|
232
|
-
* For more information, see {@
|
|
233
|
-
* and {@
|
|
223
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
224
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
234
225
|
*
|
|
235
226
|
* @param key Unique record key.
|
|
236
227
|
* @param defaultValue Fallback that will be returned if no value if present in the storage.
|
|
@@ -244,9 +235,9 @@ export declare class KeyValueStore {
|
|
|
244
235
|
static getValue<T = unknown>(key: string, defaultValue?: T): Promise<T | null>;
|
|
245
236
|
static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
|
|
246
237
|
/**
|
|
247
|
-
* Stores or deletes a value in the default {@
|
|
238
|
+
* Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
248
239
|
*
|
|
249
|
-
* This is just a convenient shortcut for {@
|
|
240
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.setValue}.
|
|
250
241
|
* For example, calling the following code:
|
|
251
242
|
* ```javascript
|
|
252
243
|
* await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -258,10 +249,10 @@ export declare class KeyValueStore {
|
|
|
258
249
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
259
250
|
* ```
|
|
260
251
|
*
|
|
261
|
-
* To get a value from the default key-value store, you can use the {@
|
|
252
|
+
* To get a value from the default key-value store, you can use the {@apilink KeyValueStore.getValue} function.
|
|
262
253
|
*
|
|
263
|
-
* For more information, see {@
|
|
264
|
-
* and {@
|
|
254
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
255
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
265
256
|
*
|
|
266
257
|
* @param key
|
|
267
258
|
* Unique record key.
|
|
@@ -276,7 +267,7 @@ export declare class KeyValueStore {
|
|
|
276
267
|
*/
|
|
277
268
|
static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
|
|
278
269
|
/**
|
|
279
|
-
* Gets the crawler input value from the default {@
|
|
270
|
+
* Gets the crawler input value from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
280
271
|
*
|
|
281
272
|
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
282
273
|
* For example, calling the following code:
|
|
@@ -294,8 +285,8 @@ export declare class KeyValueStore {
|
|
|
294
285
|
* If you need to use the input multiple times in your crawler,
|
|
295
286
|
* it is far more efficient to read it once and store it locally.
|
|
296
287
|
*
|
|
297
|
-
* For more information, see {@
|
|
298
|
-
* and {@
|
|
288
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
289
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
299
290
|
*
|
|
300
291
|
* @returns
|
|
301
292
|
* Returns a promise that resolves to an object, string
|
|
@@ -307,13 +298,13 @@ export declare class KeyValueStore {
|
|
|
307
298
|
static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
|
|
308
299
|
}
|
|
309
300
|
/**
|
|
310
|
-
* User-function used in the {@
|
|
301
|
+
* User-function used in the {@apilink KeyValueStore.forEachKey} method.
|
|
311
302
|
*/
|
|
312
303
|
export interface KeyConsumer {
|
|
313
304
|
/**
|
|
314
|
-
* @param key Current {@
|
|
315
|
-
* @param index Position of the current key in {@
|
|
316
|
-
* @param info Information about the current {@
|
|
305
|
+
* @param key Current {@apilink KeyValueStore} key being processed.
|
|
306
|
+
* @param index Position of the current key in {@apilink KeyValueStore}.
|
|
307
|
+
* @param info Information about the current {@apilink KeyValueStore} entry.
|
|
317
308
|
* @param info.size Size of the value associated with the current key in bytes.
|
|
318
309
|
*/
|
|
319
310
|
(key: string, index: number, info: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAYqB,QAAQ,CAAC,MAAM;IAX1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IAEpD;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAOvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAY3G,OAAO,CAAC,uBAAuB;IAc/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAIlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAIpF,WAAW;IAiBzB;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAW7G;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;WAKvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAI/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -44,37 +44,28 @@ exports.maybeStringify = maybeStringify;
|
|
|
44
44
|
* for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
|
|
45
45
|
*
|
|
46
46
|
* Do not instantiate this class directly, use the
|
|
47
|
-
* {@
|
|
47
|
+
* {@apilink KeyValueStore.open} function instead.
|
|
48
48
|
*
|
|
49
49
|
* Each crawler run is associated with a default key-value store, which is created exclusively
|
|
50
50
|
* for the run. By convention, the crawler input and output are stored into the
|
|
51
51
|
* default key-value store under the `INPUT` and `OUTPUT` key, respectively.
|
|
52
52
|
* Typically, input and output are JSON files, although it can be any other format.
|
|
53
53
|
* To access the default key-value store directly, you can use the
|
|
54
|
-
* {@
|
|
54
|
+
* {@apilink KeyValueStore.getValue} and {@apilink KeyValueStore.setValue} convenience functions.
|
|
55
55
|
*
|
|
56
|
-
* To access the input, you can also use the {@
|
|
56
|
+
* To access the input, you can also use the {@apilink KeyValueStore.getInput} convenience function.
|
|
57
57
|
*
|
|
58
|
-
* `KeyValueStore` stores its data
|
|
59
|
-
* depending on whether the [`APIFY_IS_AT_HOME`](/docs/guides/environment-variables#apify_is_at_home)
|
|
60
|
-
* environment variable is set.
|
|
58
|
+
* `KeyValueStore` stores its data on a local disk.
|
|
61
59
|
*
|
|
62
|
-
* If the `
|
|
60
|
+
* If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
|
|
63
61
|
* the local directory in the following files:
|
|
64
62
|
* ```
|
|
65
|
-
* {
|
|
63
|
+
* {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
|
|
66
64
|
* ```
|
|
67
65
|
* Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
|
|
68
|
-
* unless you override it by setting the `
|
|
66
|
+
* unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
|
|
69
67
|
* The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
|
|
70
68
|
*
|
|
71
|
-
* If the [`APIFY_TOKEN`](/docs/guides/environment-variables#apify_token) environment variable is set but
|
|
72
|
-
* [`APIFY_LOCAL_STORAGE_DIR`](/docs/guides/environment-variables#apify_local_storage_dir) not,
|
|
73
|
-
* the data is stored in the [Apify Key-value store](https://docs.apify.com/storage/key-value-store)
|
|
74
|
-
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
75
|
-
* option to {@link KeyValueStore.open} function, even if the
|
|
76
|
-
* [`APIFY_LOCAL_STORAGE_DIR`](/docs/guides/environment-variables#apify_local_storage_dir) variable is set.
|
|
77
|
-
*
|
|
78
69
|
* **Example usage:**
|
|
79
70
|
*
|
|
80
71
|
* ```javascript
|
|
@@ -162,7 +153,7 @@ class KeyValueStore {
|
|
|
162
153
|
* If the record does not exist, the function resolves to `null`.
|
|
163
154
|
*
|
|
164
155
|
* To save or delete a value in the key-value store, use the
|
|
165
|
-
* {@
|
|
156
|
+
* {@apilink KeyValueStore.setValue} function.
|
|
166
157
|
*
|
|
167
158
|
* **Example usage:**
|
|
168
159
|
*
|
|
@@ -232,7 +223,7 @@ class KeyValueStore {
|
|
|
232
223
|
* regardless whether the record existed or not.
|
|
233
224
|
*
|
|
234
225
|
* To retrieve a value from the key-value store, use the
|
|
235
|
-
* {@
|
|
226
|
+
* {@apilink KeyValueStore.getValue} function.
|
|
236
227
|
*
|
|
237
228
|
* **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
|
|
238
229
|
* otherwise the crawler process might finish before the value is stored!
|
|
@@ -342,13 +333,13 @@ class KeyValueStore {
|
|
|
342
333
|
: undefined; // [].forEach() returns undefined.
|
|
343
334
|
}
|
|
344
335
|
/**
|
|
345
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
336
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
|
|
346
337
|
*
|
|
347
338
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
348
339
|
* The records are stored and retrieved using a unique key.
|
|
349
340
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
350
341
|
*
|
|
351
|
-
* For more details and code examples, see the {@
|
|
342
|
+
* For more details and code examples, see the {@apilink KeyValueStore} class.
|
|
352
343
|
*
|
|
353
344
|
* @param [storeIdOrName]
|
|
354
345
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -365,9 +356,9 @@ class KeyValueStore {
|
|
|
365
356
|
return manager.openStorage(storeIdOrName);
|
|
366
357
|
}
|
|
367
358
|
/**
|
|
368
|
-
* Gets a value from the default {@
|
|
359
|
+
* Gets a value from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
369
360
|
*
|
|
370
|
-
* This is just a convenient shortcut for {@
|
|
361
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
|
|
371
362
|
* For example, calling the following code:
|
|
372
363
|
* ```javascript
|
|
373
364
|
* const value = await KeyValueStore.getValue('my-key');
|
|
@@ -379,10 +370,10 @@ class KeyValueStore {
|
|
|
379
370
|
* const value = await store.getValue('my-key');
|
|
380
371
|
* ```
|
|
381
372
|
*
|
|
382
|
-
* To store the value to the default key-value store, you can use the {@
|
|
373
|
+
* To store the value to the default key-value store, you can use the {@apilink KeyValueStore.setValue} function.
|
|
383
374
|
*
|
|
384
|
-
* For more information, see {@
|
|
385
|
-
* and {@
|
|
375
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
376
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
386
377
|
*
|
|
387
378
|
* @param key Unique record key.
|
|
388
379
|
* @param defaultValue Fallback that will be returned if no value if present in the storage.
|
|
@@ -402,9 +393,9 @@ class KeyValueStore {
|
|
|
402
393
|
return store.getAutoSavedValue(key, defaultValue);
|
|
403
394
|
}
|
|
404
395
|
/**
|
|
405
|
-
* Stores or deletes a value in the default {@
|
|
396
|
+
* Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
406
397
|
*
|
|
407
|
-
* This is just a convenient shortcut for {@
|
|
398
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.setValue}.
|
|
408
399
|
* For example, calling the following code:
|
|
409
400
|
* ```javascript
|
|
410
401
|
* await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -416,10 +407,10 @@ class KeyValueStore {
|
|
|
416
407
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
417
408
|
* ```
|
|
418
409
|
*
|
|
419
|
-
* To get a value from the default key-value store, you can use the {@
|
|
410
|
+
* To get a value from the default key-value store, you can use the {@apilink KeyValueStore.getValue} function.
|
|
420
411
|
*
|
|
421
|
-
* For more information, see {@
|
|
422
|
-
* and {@
|
|
412
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
413
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
423
414
|
*
|
|
424
415
|
* @param key
|
|
425
416
|
* Unique record key.
|
|
@@ -437,7 +428,7 @@ class KeyValueStore {
|
|
|
437
428
|
return store.setValue(key, value, options);
|
|
438
429
|
}
|
|
439
430
|
/**
|
|
440
|
-
* Gets the crawler input value from the default {@
|
|
431
|
+
* Gets the crawler input value from the default {@apilink KeyValueStore} associated with the current crawler run.
|
|
441
432
|
*
|
|
442
433
|
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
443
434
|
* For example, calling the following code:
|
|
@@ -455,8 +446,8 @@ class KeyValueStore {
|
|
|
455
446
|
* If you need to use the input multiple times in your crawler,
|
|
456
447
|
* it is far more efficient to read it once and store it locally.
|
|
457
448
|
*
|
|
458
|
-
* For more information, see {@
|
|
459
|
-
* and {@
|
|
449
|
+
* For more information, see {@apilink KeyValueStore.open}
|
|
450
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
460
451
|
*
|
|
461
452
|
* @returns
|
|
462
453
|
* Returns a promise that resolves to an object, string
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAmB;WAAC;QAMhD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QAE7C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,KAAK,KAAK,EAAE;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAM,CAAC,CAAC;qBAC7B,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;aACrC;SACJ;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,MAAM,IAAA,4BAAoB,GAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAEzE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,CAAC;CACJ;AA1WD,sCA0WC"}
|
|
@@ -9,8 +9,8 @@ export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
|
|
|
9
9
|
export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
|
|
10
10
|
export interface RequestListOptions {
|
|
11
11
|
/**
|
|
12
|
-
* An array of sources of URLs for the {@
|
|
13
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
12
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
13
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
14
14
|
*
|
|
15
15
|
* **IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes.
|
|
16
16
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
@@ -44,15 +44,15 @@ export interface RequestListOptions {
|
|
|
44
44
|
sources?: Source[];
|
|
45
45
|
/**
|
|
46
46
|
* A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
|
|
47
|
-
* was not able to fetch their persisted version (see {@
|
|
48
|
-
* It must return an `Array` of {@
|
|
47
|
+
* was not able to fetch their persisted version (see {@apilink RequestListOptions.persistRequestsKey}).
|
|
48
|
+
* It must return an `Array` of {@apilink Request} or {@apilink RequestOptions}.
|
|
49
49
|
*
|
|
50
50
|
* This is very useful in a scenario when getting the sources is a resource intensive or time consuming
|
|
51
51
|
* task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
|
|
52
52
|
* `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
|
|
53
53
|
* fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
|
|
54
54
|
*
|
|
55
|
-
* If both {@
|
|
55
|
+
* If both {@apilink RequestListOptions.sources} and {@apilink RequestListOptions.sourcesFunction} are provided,
|
|
56
56
|
* the sources returned by the function will be added after the `sources`.
|
|
57
57
|
*
|
|
58
58
|
* **Example:**
|
|
@@ -99,12 +99,12 @@ export interface RequestListOptions {
|
|
|
99
99
|
persistStateKey?: string;
|
|
100
100
|
/**
|
|
101
101
|
* Identifies the key in the default key-value store under which the `RequestList` persists its
|
|
102
|
-
* Requests during the {@
|
|
102
|
+
* Requests during the {@apilink RequestList.initialize} call.
|
|
103
103
|
* This is necessary if `persistStateKey` is set and the source URLs might potentially change,
|
|
104
104
|
* to ensure consistency of the source URLs and state object. However, it comes with some
|
|
105
105
|
* storage and performance overheads.
|
|
106
106
|
*
|
|
107
|
-
* If `persistRequestsKey` is not set, {@
|
|
107
|
+
* If `persistRequestsKey` is not set, {@apilink RequestList.initialize} will always fetch the sources
|
|
108
108
|
* from their origin, check that they are consistent with the restored state (if any)
|
|
109
109
|
* and throw an error if they are not.
|
|
110
110
|
*/
|
|
@@ -130,7 +130,7 @@ export interface RequestListOptions {
|
|
|
130
130
|
state?: RequestListState;
|
|
131
131
|
/**
|
|
132
132
|
* By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
|
|
133
|
-
* on the `uniqueKey` property of passed source {@
|
|
133
|
+
* on the `uniqueKey` property of passed source {@apilink Request} objects.
|
|
134
134
|
*
|
|
135
135
|
* If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
|
|
136
136
|
* In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
|
|
@@ -150,19 +150,19 @@ export interface RequestListOptions {
|
|
|
150
150
|
/**
|
|
151
151
|
* Represents a static list of URLs to crawl.
|
|
152
152
|
* The URLs can be provided either in code or parsed from a text file hosted on the web.
|
|
153
|
-
* `RequestList` is used by {@
|
|
154
|
-
* and {@
|
|
153
|
+
* `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
|
|
154
|
+
* and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
|
|
155
155
|
*
|
|
156
|
-
* Each URL is represented using an instance of the {@
|
|
156
|
+
* Each URL is represented using an instance of the {@apilink Request} class.
|
|
157
157
|
* The list can only contain unique URLs. More precisely, it can only contain `Request` instances
|
|
158
158
|
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
159
|
-
* To add a single URL to the list multiple times, corresponding {@
|
|
159
|
+
* To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
|
|
160
160
|
* `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
|
|
161
161
|
* `RequestList` from sources.
|
|
162
162
|
*
|
|
163
|
-
* Once you create an instance of `RequestList`, you need to call the {@
|
|
163
|
+
* Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
|
|
164
164
|
* before the instance can be used. After that, no more URLs can be added to the list.
|
|
165
|
-
* Unlike {@
|
|
165
|
+
* Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
|
|
166
166
|
* > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
|
|
167
167
|
* > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
|
|
168
168
|
* > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
|
|
@@ -171,9 +171,9 @@ export interface RequestListOptions {
|
|
|
171
171
|
*
|
|
172
172
|
* `RequestList` has an internal state where it stores information about which requests were already handled,
|
|
173
173
|
* which are in progress and which were reclaimed. The state may be automatically persisted to the default
|
|
174
|
-
* {@
|
|
174
|
+
* {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
|
|
175
175
|
* the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
|
|
176
|
-
* event that is periodically emitted by {@
|
|
176
|
+
* event that is periodically emitted by {@apilink EventManager}.
|
|
177
177
|
*
|
|
178
178
|
* The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
|
|
179
179
|
* `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
|
|
@@ -279,7 +279,7 @@ export declare class RequestList {
|
|
|
279
279
|
*/
|
|
280
280
|
protected _addRequestsFromSources(): Promise<void>;
|
|
281
281
|
/**
|
|
282
|
-
* Persists the current state of the `RequestList` into the default {@
|
|
282
|
+
* Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
|
|
283
283
|
* The state is persisted automatically in regular intervals, but calling this method manually
|
|
284
284
|
* is useful in cases where you want to have the most current state available after you pause
|
|
285
285
|
* or stop fetching its requests. For example after you pause or abort a crawl. Or just before
|
|
@@ -307,7 +307,7 @@ export declare class RequestList {
|
|
|
307
307
|
*/
|
|
308
308
|
getState(): RequestListState;
|
|
309
309
|
/**
|
|
310
|
-
* Resolves to `true` if the next call to {@
|
|
310
|
+
* Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
|
|
311
311
|
* would return `null`, otherwise it resolves to `false`.
|
|
312
312
|
* Note that even if the list is empty, there might be some pending requests currently being processed.
|
|
313
313
|
*/
|
|
@@ -317,8 +317,8 @@ export declare class RequestList {
|
|
|
317
317
|
*/
|
|
318
318
|
isFinished(): Promise<boolean>;
|
|
319
319
|
/**
|
|
320
|
-
* Gets the next {@
|
|
321
|
-
* using the {@
|
|
320
|
+
* Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
|
|
321
|
+
* using the {@apilink RequestList.reclaimRequest} function, if there is any.
|
|
322
322
|
* Otherwise it gets the next request from sources.
|
|
323
323
|
*
|
|
324
324
|
* The function's `Promise` resolves to `null` if there are no more
|
|
@@ -372,14 +372,14 @@ export declare class RequestList {
|
|
|
372
372
|
handledCount(): number;
|
|
373
373
|
/**
|
|
374
374
|
* Opens a request list and returns a promise resolving to an instance
|
|
375
|
-
* of the {@
|
|
375
|
+
* of the {@apilink RequestList} class that is already initialized.
|
|
376
376
|
*
|
|
377
|
-
* {@
|
|
377
|
+
* {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
|
|
378
378
|
* To enable picking up where left off after a process restart, the request list sources
|
|
379
379
|
* are persisted to the key-value store at initialization of the list. Then, while crawling,
|
|
380
380
|
* a small state object is regularly persisted to keep track of the crawling status.
|
|
381
381
|
*
|
|
382
|
-
* For more details and code examples, see the {@
|
|
382
|
+
* For more details and code examples, see the {@apilink RequestList} class.
|
|
383
383
|
*
|
|
384
384
|
* **Example usage:**
|
|
385
385
|
*
|
|
@@ -405,23 +405,23 @@ export declare class RequestList {
|
|
|
405
405
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
406
406
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
407
407
|
* @param [sources]
|
|
408
|
-
* An array of sources of URLs for the {@
|
|
409
|
-
* plain objects that define at least the `url` property, or an array of {@
|
|
408
|
+
* An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
|
|
409
|
+
* plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
|
|
410
410
|
*
|
|
411
|
-
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@
|
|
411
|
+
* **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
|
|
412
412
|
* This is a measure to prevent memory leaks in situations when millions of sources are
|
|
413
413
|
* added.
|
|
414
414
|
*
|
|
415
415
|
* Additionally, the `requestsFromUrl` property may be used instead of `url`,
|
|
416
|
-
* which will instruct {@
|
|
416
|
+
* which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
|
|
417
417
|
* The URLs will be parsed from the received response. In this case you can limit the URLs
|
|
418
418
|
* using `regex` parameter containing regular expression pattern for URLs to be included.
|
|
419
419
|
*
|
|
420
|
-
* For details, see the {@
|
|
420
|
+
* For details, see the {@apilink RequestListOptions.sources}
|
|
421
421
|
* @param [options]
|
|
422
|
-
* The {@
|
|
423
|
-
* the {@
|
|
424
|
-
* options and the `sources` parameter supersedes the {@
|
|
422
|
+
* The {@apilink RequestList} options. Note that the `listName` parameter supersedes
|
|
423
|
+
* the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
|
|
424
|
+
* options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
|
|
425
425
|
*/
|
|
426
426
|
static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
|
|
427
427
|
/**
|
|
@@ -430,8 +430,8 @@ export declare class RequestList {
|
|
|
430
430
|
private _downloadListOfUrls;
|
|
431
431
|
}
|
|
432
432
|
/**
|
|
433
|
-
* Represents state of a {@
|
|
434
|
-
* You can obtain the state by calling {@
|
|
433
|
+
* Represents state of a {@apilink RequestList}. It can be used to resume a {@apilink RequestList} which has been previously processed.
|
|
434
|
+
* You can obtain the state by calling {@apilink RequestList.getState} and receive an object with
|
|
435
435
|
* the following structure:
|
|
436
436
|
*
|
|
437
437
|
* ```
|