@crawlee/core 3.0.3-beta.9 → 3.0.3

package/storages/key_value_store.d.ts

@@ -18,16 +18,16 @@ export declare const maybeStringify: <T>(value: T, options: {
  * for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
  *
  * Do not instantiate this class directly, use the
- * {@link KeyValueStore.open} function instead.
+ * {@apilink KeyValueStore.open} function instead.
  *
  * Each crawler run is associated with a default key-value store, which is created exclusively
  * for the run. By convention, the crawler input and output are stored into the
  * default key-value store under the `INPUT` and `OUTPUT` key, respectively.
  * Typically, input and output are JSON files, although it can be any other format.
  * To access the default key-value store directly, you can use the
- * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
+ * {@apilink KeyValueStore.getValue} and {@apilink KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
+ * To access the input, you can also use the {@apilink KeyValueStore.getInput} convenience function.
  *
  * `KeyValueStore` stores its data on a local disk.
  *
@@ -93,7 +93,7 @@ export declare class KeyValueStore {
      * If the record does not exist, the function resolves to `null`.
      *
      * To save or delete a value in the key-value store, use the
-     * {@link KeyValueStore.setValue} function.
+     * {@apilink KeyValueStore.setValue} function.
      *
      * **Example usage:**
      *
@@ -141,7 +141,7 @@ export declare class KeyValueStore {
      * regardless whether the record existed or not.
      *
      * To retrieve a value from the key-value store, use the
-     * {@link KeyValueStore.getValue} function.
+     * {@apilink KeyValueStore.getValue} function.
      *
      * **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
      * otherwise the crawler process might finish before the value is stored!
@@ -189,13 +189,13 @@ export declare class KeyValueStore {
     forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
     private _forEachKey;
     /**
-     * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
+     * Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
      *
      * Key-value stores are used to store records or files, along with their MIME content type.
      * The records are stored and retrieved using a unique key.
      * The actual data is stored either on a local filesystem or in the Apify cloud.
      *
-     * For more details and code examples, see the {@link KeyValueStore} class.
+     * For more details and code examples, see the {@apilink KeyValueStore} class.
      *
      * @param [storeIdOrName]
      *   ID or name of the key-value store to be opened. If `null` or `undefined`,
@@ -204,9 +204,9 @@ export declare class KeyValueStore {
      */
     static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
     /**
-     * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
+     * Gets a value from the default {@apilink KeyValueStore} associated with the current crawler run.
      *
-     * This is just a convenient shortcut for {@link KeyValueStore.getValue}.
+     * This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
      * For example, calling the following code:
      * ```javascript
      * const value = await KeyValueStore.getValue('my-key');
@@ -218,10 +218,10 @@ export declare class KeyValueStore {
      * const value = await store.getValue('my-key');
      * ```
      *
-     * To store the value to the default key-value store, you can use the {@link KeyValueStore.setValue} function.
+     * To store the value to the default key-value store, you can use the {@apilink KeyValueStore.setValue} function.
      *
-     * For more information, see  {@link KeyValueStore.open}
-     * and  {@link KeyValueStore.getValue}.
+     * For more information, see  {@apilink KeyValueStore.open}
+     * and  {@apilink KeyValueStore.getValue}.
      *
      * @param key Unique record key.
      * @param defaultValue Fallback that will be returned if no value if present in the storage.
@@ -235,9 +235,9 @@ export declare class KeyValueStore {
     static getValue<T = unknown>(key: string, defaultValue?: T): Promise<T | null>;
     static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
     /**
-     * Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
+     * Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current crawler run.
      *
-     * This is just a convenient shortcut for  {@link KeyValueStore.setValue}.
+     * This is just a convenient shortcut for  {@apilink KeyValueStore.setValue}.
      * For example, calling the following code:
      * ```javascript
      * await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
@@ -249,10 +249,10 @@ export declare class KeyValueStore {
      * await store.setValue('OUTPUT', { foo: "bar" });
      * ```
      *
-     * To get a value from the default key-value store, you can use the  {@link KeyValueStore.getValue} function.
+     * To get a value from the default key-value store, you can use the  {@apilink KeyValueStore.getValue} function.
      *
-     * For more information, see  {@link KeyValueStore.open}
-     * and  {@link KeyValueStore.getValue}.
+     * For more information, see  {@apilink KeyValueStore.open}
+     * and  {@apilink KeyValueStore.getValue}.
      *
      * @param key
      *   Unique record key.
@@ -267,26 +267,16 @@ export declare class KeyValueStore {
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
     /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
-     * For example, calling the following code:
-     * ```javascript
-     * const input = await KeyValueStore.getInput();
-     * ```
-     *
-     * is equivalent to:
-     * ```javascript
-     * const store = await KeyValueStore.open();
-     * await store.getValue('INPUT');
-     * ```
+     * Gets the crawler input value from the default {@apilink KeyValueStore} associated with the current crawler run.
+     * By default, it will try to find root input files (either extension-less, `.json` or `.txt`),
+     * or alternatively read the input from the default {@apilink KeyValueStore}.
      *
      * Note that the `getInput()` function does not cache the value read from the key-value store.
      * If you need to use the input multiple times in your crawler,
      * it is far more efficient to read it once and store it locally.
      *
-     * For more information, see  {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
+     * For more information, see {@apilink KeyValueStore.open}
+     * and {@apilink KeyValueStore.getValue}.
      *
      * @returns
      *   Returns a promise that resolves to an object, string
@@ -298,13 +288,13 @@ export declare class KeyValueStore {
     static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
 }
 /**
- * User-function used in the  {@link KeyValueStore.forEachKey} method.
+ * User-function used in the  {@apilink KeyValueStore.forEachKey} method.
  */
 export interface KeyConsumer {
     /**
-     * @param key Current {@link KeyValueStore} key being processed.
-     * @param index Position of the current key in {@link KeyValueStore}.
-     * @param info Information about the current {@link KeyValueStore} entry.
+     * @param key Current {@apilink KeyValueStore} key being processed.
+     * @param index Position of the current key in {@apilink KeyValueStore}.
+     * @param info Information about the current {@apilink KeyValueStore} entry.
      * @param info.size Size of the value associated with the current key in bytes.
      */
     (key: string, index: number, info: {

package/storages/key_value_store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAYqB,QAAQ,CAAC,MAAM;IAX1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IAEpD;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAOvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAY3G,OAAO,CAAC,uBAAuB;IAc/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAIlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAIpF,WAAW;IAiBzB;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAW7G;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;WAKvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAI/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
+{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGrF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAYqB,QAAQ,CAAC,MAAM;IAX1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IAEpD;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAOvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAY3G,OAAO,CAAC,uBAAuB;IAc/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAIlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAIpF,WAAW;IAiBzB;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAW7G;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;WAKvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}

package/storages/key_value_store.js

@@ -5,6 +5,8 @@ const tslib_1 = require("tslib");
 const consts_1 = require("@apify/consts");
 const utilities_1 = require("@apify/utilities");
 const ow_1 = tslib_1.__importStar(require("ow"));
+const node_path_1 = require("node:path");
+const promises_1 = require("node:fs/promises");
 const configuration_1 = require("../configuration");
 const storage_manager_1 = require("./storage_manager");
 const utils_1 = require("./utils");
@@ -44,16 +46,16 @@ exports.maybeStringify = maybeStringify;
  * for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
  *
  * Do not instantiate this class directly, use the
- * {@link KeyValueStore.open} function instead.
+ * {@apilink KeyValueStore.open} function instead.
  *
  * Each crawler run is associated with a default key-value store, which is created exclusively
  * for the run. By convention, the crawler input and output are stored into the
  * default key-value store under the `INPUT` and `OUTPUT` key, respectively.
  * Typically, input and output are JSON files, although it can be any other format.
  * To access the default key-value store directly, you can use the
- * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
+ * {@apilink KeyValueStore.getValue} and {@apilink KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
+ * To access the input, you can also use the {@apilink KeyValueStore.getInput} convenience function.
  *
  * `KeyValueStore` stores its data on a local disk.
  *
@@ -153,7 +155,7 @@ class KeyValueStore {
      * If the record does not exist, the function resolves to `null`.
      *
      * To save or delete a value in the key-value store, use the
-     * {@link KeyValueStore.setValue} function.
+     * {@apilink KeyValueStore.setValue} function.
      *
      * **Example usage:**
      *
@@ -223,7 +225,7 @@ class KeyValueStore {
      * regardless whether the record existed or not.
      *
      * To retrieve a value from the key-value store, use the
-     * {@link KeyValueStore.getValue} function.
+     * {@apilink KeyValueStore.getValue} function.
      *
      * **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
      * otherwise the crawler process might finish before the value is stored!
@@ -333,13 +335,13 @@ class KeyValueStore {
             : undefined; // [].forEach() returns undefined.
     }
     /**
-     * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
+     * Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
      *
      * Key-value stores are used to store records or files, along with their MIME content type.
      * The records are stored and retrieved using a unique key.
      * The actual data is stored either on a local filesystem or in the Apify cloud.
      *
-     * For more details and code examples, see the {@link KeyValueStore} class.
+     * For more details and code examples, see the {@apilink KeyValueStore} class.
      *
      * @param [storeIdOrName]
      *   ID or name of the key-value store to be opened. If `null` or `undefined`,
@@ -356,9 +358,9 @@ class KeyValueStore {
         return manager.openStorage(storeIdOrName);
     }
     /**
-     * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
+     * Gets a value from the default {@apilink KeyValueStore} associated with the current crawler run.
      *
-     * This is just a convenient shortcut for {@link KeyValueStore.getValue}.
+     * This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
      * For example, calling the following code:
      * ```javascript
      * const value = await KeyValueStore.getValue('my-key');
@@ -370,10 +372,10 @@ class KeyValueStore {
      * const value = await store.getValue('my-key');
      * ```
      *
-     * To store the value to the default key-value store, you can use the {@link KeyValueStore.setValue} function.
+     * To store the value to the default key-value store, you can use the {@apilink KeyValueStore.setValue} function.
      *
-     * For more information, see  {@link KeyValueStore.open}
-     * and  {@link KeyValueStore.getValue}.
+     * For more information, see  {@apilink KeyValueStore.open}
+     * and  {@apilink KeyValueStore.getValue}.
      *
      * @param key Unique record key.
      * @param defaultValue Fallback that will be returned if no value if present in the storage.
@@ -393,9 +395,9 @@ class KeyValueStore {
         return store.getAutoSavedValue(key, defaultValue);
     }
     /**
-     * Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
+     * Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current crawler run.
      *
-     * This is just a convenient shortcut for  {@link KeyValueStore.setValue}.
+     * This is just a convenient shortcut for  {@apilink KeyValueStore.setValue}.
      * For example, calling the following code:
      * ```javascript
      * await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
@@ -407,10 +409,10 @@ class KeyValueStore {
      * await store.setValue('OUTPUT', { foo: "bar" });
      * ```
      *
-     * To get a value from the default key-value store, you can use the  {@link KeyValueStore.getValue} function.
+     * To get a value from the default key-value store, you can use the  {@apilink KeyValueStore.getValue} function.
      *
-     * For more information, see  {@link KeyValueStore.open}
-     * and  {@link KeyValueStore.getValue}.
+     * For more information, see  {@apilink KeyValueStore.open}
+     * and  {@apilink KeyValueStore.getValue}.
      *
      * @param key
      *   Unique record key.
@@ -428,26 +430,16 @@ class KeyValueStore {
         return store.setValue(key, value, options);
     }
     /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
-     * For example, calling the following code:
-     * ```javascript
-     * const input = await KeyValueStore.getInput();
-     * ```
-     *
-     * is equivalent to:
-     * ```javascript
-     * const store = await KeyValueStore.open();
-     * await store.getValue('INPUT');
-     * ```
+     * Gets the crawler input value from the default {@apilink KeyValueStore} associated with the current crawler run.
+     * By default, it will try to find root input files (either extension-less, `.json` or `.txt`),
+     * or alternatively read the input from the default {@apilink KeyValueStore}.
      *
      * Note that the `getInput()` function does not cache the value read from the key-value store.
      * If you need to use the input multiple times in your crawler,
      * it is far more efficient to read it once and store it locally.
      *
-     * For more information, see  {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
+     * For more information, see {@apilink KeyValueStore.open}
+     * and {@apilink KeyValueStore.getValue}.
      *
      * @returns
      *   Returns a promise that resolves to an object, string
@@ -458,7 +450,29 @@ class KeyValueStore {
      */
     static async getInput() {
         const store = await this.open();
-        return store.getValue(store.config.get('inputKey'));
+        const inputKey = store.config.get('inputKey');
+        const cwd = process.cwd();
+        const possibleExtensions = ['', '.json', '.txt'];
+        // Attempt to read input from root file instead of key-value store
+        for (const extension of possibleExtensions) {
+            const inputFile = (0, node_path_1.join)(cwd, `${inputKey}${extension}`);
+            let input;
+            // Try getting the file from the file system
+            try {
+                input = await (0, promises_1.readFile)(inputFile);
+            }
+            catch {
+                continue;
+            }
+            // Attempt to parse as JSON, or return the input as is otherwise
+            try {
+                return JSON.parse(input.toString());
+            }
+            catch {
+                return input;
+            }
+        }
+        return store.getValue(inputKey);
     }
 }
 exports.KeyValueStore = KeyValueStore;

package/storages/key_value_store.js.map

@@ -1 +1 @@
-{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAmB;WAAC;QAMhD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QAE7C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,KAAK,KAAK,EAAE;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAM,CAAC,CAAC;qBAC7B,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;aACrC;SACJ;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,MAAM,IAAA,4BAAoB,GAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAEzE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,CAAC;CACJ;AA1WD,sCA0WC"}
+{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,yCAAiC;AACjC,+CAA4C;AAC5C,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAmB;WAAC;QAMhD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QAE7C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,KAAK,KAAK,EAAE;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAM,CAAC,CAAC;qBAC7B,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;aACrC;SACJ;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,MAAM,IAAA,4BAAoB,GAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAEzE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;QAE/C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,kBAAkB,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEjD,kEAAkE;QAClE,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE;YACxC,MAAM,SAAS,GAAG,IAAA,gBAAI,EAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,SAAS,EAAE,CAAC,CAAC;YACvD,IAAI,KAAa,CAAC;YAElB,4CAA4C;YAC5C,IAAI;gBACA,KAAK,GAAG,MAAM,IAAA,mBAAQ,EAAC,SAAS,CAAC,CAAC;aACrC;YAAC,MAAM;gBACJ,SAAS;aACZ;YAED,gEAAgE;YAChE,IAAI;gBACA,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAM,CAAC;aAC5C;YAAC,MAAM;gBACJ,OAAO,KAAqB,CAAC;aAChC;SACJ;QAED,OAAO,KAAK,CAAC,QAAQ,CAAI,QAAQ,CAAC,CAAC;IACvC,CAAC;CACJ;AAzXD,sCAyXC"}

package/storages/request_list.d.ts

@@ -9,8 +9,8 @@ export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
 export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
 export interface RequestListOptions {
     /**
-     * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
-     * plain objects that define at least the `url` property, or an array of {@link Request} instances.
+     * An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
+     * plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
      *
      * **IMPORTANT:** The `sources` array will be consumed (left empty) after `RequestList` initializes.
      * This is a measure to prevent memory leaks in situations when millions of sources are
@@ -44,15 +44,15 @@ export interface RequestListOptions {
     sources?: Source[];
     /**
      * A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
-     * was not able to fetch their persisted version (see {@link RequestListOptions.persistRequestsKey}).
-     * It must return an `Array` of {@link Request} or {@link RequestOptions}.
+     * was not able to fetch their persisted version (see {@apilink RequestListOptions.persistRequestsKey}).
+     * It must return an `Array` of {@apilink Request} or {@apilink RequestOptions}.
      *
      * This is very useful in a scenario when getting the sources is a resource intensive or time consuming
      * task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
      * `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
      * fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
      *
-     * If both {@link RequestListOptions.sources} and {@link RequestListOptions.sourcesFunction} are provided,
+     * If both {@apilink RequestListOptions.sources} and {@apilink RequestListOptions.sourcesFunction} are provided,
      * the sources returned by the function will be added after the `sources`.
      *
      * **Example:**
@@ -99,12 +99,12 @@ export interface RequestListOptions {
     persistStateKey?: string;
     /**
      * Identifies the key in the default key-value store under which the `RequestList` persists its
-     * Requests during the {@link RequestList.initialize} call.
+     * Requests during the {@apilink RequestList.initialize} call.
      * This is necessary if `persistStateKey` is set and the source URLs might potentially change,
      * to ensure consistency of the source URLs and state object. However, it comes with some
      * storage and performance overheads.
      *
-     * If `persistRequestsKey` is not set, {@link RequestList.initialize} will always fetch the sources
+     * If `persistRequestsKey` is not set, {@apilink RequestList.initialize} will always fetch the sources
      * from their origin, check that they are consistent with the restored state (if any)
      * and throw an error if they are not.
      */
@@ -130,7 +130,7 @@ export interface RequestListOptions {
     state?: RequestListState;
     /**
      * By default, `RequestList` will deduplicate the provided URLs. Default deduplication is based
-     * on the `uniqueKey` property of passed source {@link Request} objects.
+     * on the `uniqueKey` property of passed source {@apilink Request} objects.
      *
      * If the property is not present, it is generated by normalizing the URL. If present, it is kept intact.
      * In any case, only one request per `uniqueKey` is added to the `RequestList` resulting in removal
@@ -150,19 +150,19 @@ export interface RequestListOptions {
 /**
  * Represents a static list of URLs to crawl.
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
- * `RequestList` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
+ * `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler}
+ * and {@apilink PlaywrightCrawler} as a source of URLs to crawl.
  *
- * Each URL is represented using an instance of the {@link Request} class.
+ * Each URL is represented using an instance of the {@apilink Request} class.
  * The list can only contain unique URLs. More precisely, it can only contain `Request` instances
  * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL to the list multiple times, corresponding {@link Request} objects will need to have different
+ * To add a single URL to the list multiple times, corresponding {@apilink Request} objects will need to have different
  * `uniqueKey` properties. You can use the `keepDuplicateUrls` option to do this for you when initializing the
  * `RequestList` from sources.
  *
- * Once you create an instance of `RequestList`, you need to call the {@link RequestList.initialize} function
+ * Once you create an instance of `RequestList`, you need to call the {@apilink RequestList.initialize} function
  * before the instance can be used. After that, no more URLs can be added to the list.
- * Unlike {@link RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
+ * Unlike {@apilink RequestQueue}, `RequestList` is static but it can contain even millions of URLs.
  * > Note that `RequestList` can be used together with `RequestQueue` by the same crawler.
  * > In such cases, each request from `RequestList` is enqueued into `RequestQueue` first and then consumed from the latter.
  * > This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue).
@@ -171,9 +171,9 @@ export interface RequestListOptions {
  *
  * `RequestList` has an internal state where it stores information about which requests were already handled,
  * which are in progress and which were reclaimed. The state may be automatically persisted to the default
- * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
+ * {@apilink KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
- * event that is periodically emitted by {@link EventManager}.
+ * event that is periodically emitted by {@apilink EventManager}.
  *
  * The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
  * `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
@@ -279,7 +279,7 @@ export declare class RequestList {
      */
     protected _addRequestsFromSources(): Promise<void>;
     /**
-     * Persists the current state of the `RequestList` into the default {@link KeyValueStore}.
+     * Persists the current state of the `RequestList` into the default {@apilink KeyValueStore}.
      * The state is persisted automatically in regular intervals, but calling this method manually
      * is useful in cases where you want to have the most current state available after you pause
      * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
@@ -307,7 +307,7 @@ export declare class RequestList {
      */
     getState(): RequestListState;
     /**
-     * Resolves to `true` if the next call to {@link RequestList.fetchNextRequest} function
+     * Resolves to `true` if the next call to {@apilink RequestList.fetchNextRequest} function
      * would return `null`, otherwise it resolves to `false`.
      * Note that even if the list is empty, there might be some pending requests currently being processed.
      */
@@ -317,8 +317,8 @@ export declare class RequestList {
      */
     isFinished(): Promise<boolean>;
     /**
-     * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
-     * using the {@link RequestList.reclaimRequest} function, if there is any.
+     * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
+     * using the {@apilink RequestList.reclaimRequest} function, if there is any.
      * Otherwise it gets the next request from sources.
      *
      * The function's `Promise` resolves to `null` if there are no more
@@ -372,14 +372,14 @@ export declare class RequestList {
     handledCount(): number;
     /**
      * Opens a request list and returns a promise resolving to an instance
-     * of the {@link RequestList} class that is already initialized.
+     * of the {@apilink RequestList} class that is already initialized.
      *
-     * {@link RequestList} represents a list of URLs to crawl, which is always stored in memory.
+     * {@apilink RequestList} represents a list of URLs to crawl, which is always stored in memory.
      * To enable picking up where left off after a process restart, the request list sources
      * are persisted to the key-value store at initialization of the list. Then, while crawling,
      * a small state object is regularly persisted to keep track of the crawling status.
      *
-     * For more details and code examples, see the {@link RequestList} class.
+     * For more details and code examples, see the {@apilink RequestList} class.
      *
      * **Example usage:**
      *
@@ -405,23 +405,23 @@ export declare class RequestList {
      *   If `null`, the list will not be persisted and will only be stored in memory. Process restart
      *   will then cause the list to be crawled again from the beginning. We suggest always using a name.
      * @param [sources]
-     *  An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
-     *  plain objects that define at least the `url` property, or an array of {@link Request} instances.
+     *  An array of sources of URLs for the {@apilink RequestList}. It can be either an array of strings,
+     *  plain objects that define at least the `url` property, or an array of {@apilink Request} instances.
      *
-     *  **IMPORTANT:** The `sources` array will be consumed (left empty) after {@link RequestList} initializes.
+     *  **IMPORTANT:** The `sources` array will be consumed (left empty) after {@apilink RequestList} initializes.
      *  This is a measure to prevent memory leaks in situations when millions of sources are
      *  added.
      *
      *  Additionally, the `requestsFromUrl` property may be used instead of `url`,
-     *  which will instruct {@link RequestList} to download the source URLs from a given remote location.
+     *  which will instruct {@apilink RequestList} to download the source URLs from a given remote location.
      *  The URLs will be parsed from the received response. In this case you can limit the URLs
      *  using `regex` parameter containing regular expression pattern for URLs to be included.
      *
-     *  For details, see the {@link RequestListOptions.sources}
+     *  For details, see the {@apilink RequestListOptions.sources}
      * @param [options]
-     *   The {@link RequestList} options. Note that the `listName` parameter supersedes
-     *   the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
-     *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
+     *   The {@apilink RequestList} options. Note that the `listName` parameter supersedes
+     *   the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}
+     *   options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.
      */
     static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
     /**
@@ -430,8 +430,8 @@ export declare class RequestList {
     private _downloadListOfUrls;
 }
 /**
- * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.
- * You can obtain the state by calling {@link RequestList.getState} and receive an object with
+ * Represents state of a {@apilink RequestList}. It can be used to resume a {@apilink RequestList} which has been previously processed.
+ * You can obtain the state by calling {@apilink RequestList.getState} and receive an object with
  * the following structure:
  *
  * ```