@crawlee/core 3.0.0-beta.9 → 3.0.0

package/storages/key_value_store.js

@@ -7,6 +7,7 @@ const utilities_1 = require("@apify/utilities");
 const ow_1 = tslib_1.__importStar(require("ow"));
 const configuration_1 = require("../configuration");
 const storage_manager_1 = require("./storage_manager");
+const utils_1 = require("./utils");
 /**
  * Helper function to possibly stringify value if options.contentType is not set.
  *
@@ -40,19 +41,19 @@ exports.maybeStringify = maybeStringify;
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
  * for saving and reading data records or files. Each data record is
  * represented by a unique key and associated with a MIME content type. Key-value stores are ideal
- * for saving screenshots, actor inputs and outputs, web pages, PDFs or to persist the state of crawlers.
+ * for saving screenshots, crawler inputs and outputs, web pages, PDFs or to persist the state of crawlers.
  *
  * Do not instantiate this class directly, use the
  * {@link KeyValueStore.open} function instead.
  *
- * Each actor run is associated with a default key-value store, which is created exclusively
- * for the run. By convention, the actor input and output are stored into the
+ * Each crawler run is associated with a default key-value store, which is created exclusively
+ * for the run. By convention, the crawler input and output are stored into the
  * default key-value store under the `INPUT` and `OUTPUT` key, respectively.
  * Typically, input and output are JSON files, although it can be any other format.
  * To access the default key-value store directly, you can use the
- * {@link Actor.getValue} and {@link Actor.setValue} convenience functions.
+ * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link Actor.getInput} convenience function.
+ * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
  *
  * `KeyValueStore` stores its data either on local disk or in the Apify cloud,
  * depending on whether the [`APIFY_IS_AT_HOME`](/docs/guides/environment-variables#apify_is_at_home)
@@ -77,13 +78,13 @@ exports.maybeStringify = maybeStringify;
  * **Example usage:**
  *
  * ```javascript
- * // Get actor input from the default key-value store.
- * const input = await Actor.getInput();
+ * // Get crawler input from the default key-value store.
+ * const input = await KeyValueStore.getInput();
  * // Get some value from the default key-value store.
- * const otherValue = await Actor.getValue('my-key');
+ * const otherValue = await KeyValueStore.getValue('my-key');
  *
- * // Write actor output to the default key-value store.
- * await Actor.setValue('OUTPUT', { myResult: 123 });
+ * // Write crawler output to the default key-value store.
+ * await KeyValueStore.setValue('OUTPUT', { myResult: 123 });
  *
  * // Open a named key-value store
  * const store = await KeyValueStore.open('some-name');
@@ -130,6 +131,19 @@ class KeyValueStore {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "persistStateEventStarted", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
+        Object.defineProperty(this, "cache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
         this.id = options.id;
         this.name = options.name;
         this.client = options.client.keyValueStore(this.id);
@@ -159,15 +173,37 @@ class KeyValueStore {
      * @param key
      *   Unique key of the record. It can be at most 256 characters long and only consist
      *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     * @param defaultValue
+     *   Fallback that will be returned if no value if present in the storage.
      * @returns
      *   Returns a promise that resolves to an object, string
      *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
      *   on the MIME content type of the record.
      */
-    async getValue(key) {
+    async getValue(key, defaultValue) {
         (0, ow_1.default)(key, ow_1.default.string.nonEmpty);
         const record = await this.client.getRecord(key);
-        return record?.value ?? null;
+        return record?.value ?? defaultValue ?? null;
+    }
+    async getAutoSavedValue(key, defaultValue = {}) {
+        if (this.cache.has(key)) {
+            return this.cache.get(key);
+        }
+        const value = await this.getValue(key, defaultValue);
+        this.cache.set(key, value);
+        this.ensurePersistStateEvent();
+        return value;
+    }
+    ensurePersistStateEvent() {
+        if (this.persistStateEventStarted) {
+            return;
+        }
+        this.config.getEventManager().on('persistState', async () => {
+            for (const [key, value] of this.cache) {
+                await this.setValue(key, value);
+            }
+        });
+        this.persistStateEventStarted = true;
     }
     /**
      * Saves or deletes a record in the key-value store.
@@ -199,7 +235,7 @@ class KeyValueStore {
      * {@link KeyValueStore.getValue} function.
      *
      * **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
-     * otherwise the actor process might finish before the value is stored!
+     * otherwise the crawler process might finish before the value is stored!
      *
      * @param key
      *   Unique key of the record. It can be at most 256 characters long and only consist
@@ -227,6 +263,22 @@ class KeyValueStore {
         }));
         // Make copy of options, don't update what user passed.
         const optionsCopy = { ...options };
+        // If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
+        const cachedValue = this.cache.get(key);
+        if (this.cache.has(key) && cachedValue !== value) {
+            if (value === null) {
+                // Cached state can be only object, so a propagation of `null` means removing all its properties.
+                Object.keys(cachedValue).forEach((k) => this.cache.delete(k));
+            }
+            else if (typeof value === 'object') {
+                // We need to remove the keys that are no longer present in the new value.
+                Object.keys(cachedValue)
+                    .filter((k) => !(k in value))
+                    .forEach((k) => this.cache.delete(k));
+                // And update the existing ones + add new ones.
+                Object.assign(cachedValue, value);
+            }
+        }
         // In this case delete the record.
         if (value === null)
             return this.client.deleteRecord(key);
@@ -243,16 +295,12 @@ class KeyValueStore {
      */
     async drop() {
         await this.client.delete();
-        const manager = new storage_manager_1.StorageManager(KeyValueStore, this.config);
+        const manager = storage_manager_1.StorageManager.getManager(KeyValueStore, this.config);
         manager.closeStorage(this);
     }
-    /**
-     * Returns a URL for the given key that may be used to publicly
-     * access the value in the remote key-value store.
-     */
-    getPublicUrl(key) {
-        // FIXME how should this work? should we remove this method or provide a way to configure the base url?
-        return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`;
+    /** @internal */
+    clearCache() {
+        this.cache.clear();
     }
     /**
      * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
@@ -304,17 +352,123 @@ class KeyValueStore {
      *
      * @param [storeIdOrName]
      *   ID or name of the key-value store to be opened. If `null` or `undefined`,
-     *   the function returns the default key-value store associated with the actor run.
+     *   the function returns the default key-value store associated with the crawler run.
      * @param [options] Storage manager options.
      */
     static async open(storeIdOrName, options = {}) {
-        (0, ow_1.default)(storeIdOrName, ow_1.default.optional.string);
+        (0, ow_1.default)(storeIdOrName, ow_1.default.optional.any(ow_1.default.string, ow_1.default.null));
         (0, ow_1.default)(options, ow_1.default.object.exactShape({
             config: ow_1.default.optional.object.instanceOf(configuration_1.Configuration),
         }));
-        const manager = new storage_manager_1.StorageManager(KeyValueStore, options.config);
+        await (0, utils_1.purgeDefaultStorages)();
+        const manager = storage_manager_1.StorageManager.getManager(KeyValueStore, options.config);
         return manager.openStorage(storeIdOrName);
     }
+    /**
+     * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
+     *
+     * This is just a convenient shortcut for {@link KeyValueStore.getValue}.
+     * For example, calling the following code:
+     * ```javascript
+     * const value = await KeyValueStore.getValue('my-key');
+     * ```
+     *
+     * is equivalent to:
+     * ```javascript
+     * const store = await KeyValueStore.open();
+     * const value = await store.getValue('my-key');
+     * ```
+     *
+     * To store the value to the default key-value store, you can use the {@link KeyValueStore.setValue} function.
+     *
+     * For more information, see  {@link KeyValueStore.open}
+     * and  {@link KeyValueStore.getValue}.
+     *
+     * @param key Unique record key.
+     * @param defaultValue Fallback that will be returned if no value if present in the storage.
+     * @returns
+     *   Returns a promise that resolves to an object, string
+     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
+     *   on the MIME content type of the record, or `null`
+     *   if the record is missing.
+     * @ignore
+     */
+    static async getValue(key, defaultValue) {
+        const store = await this.open();
+        return store.getValue(key, defaultValue);
+    }
+    static async getAutoSavedValue(key, defaultValue = {}) {
+        const store = await this.open();
+        return store.getAutoSavedValue(key, defaultValue);
+    }
+    /**
+     * Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
+     *
+     * This is just a convenient shortcut for  {@link KeyValueStore.setValue}.
+     * For example, calling the following code:
+     * ```javascript
+     * await KeyValueStore.setValue('OUTPUT', { foo: "bar" });
+     * ```
+     *
+     * is equivalent to:
+     * ```javascript
+     * const store = await KeyValueStore.open();
+     * await store.setValue('OUTPUT', { foo: "bar" });
+     * ```
+     *
+     * To get a value from the default key-value store, you can use the  {@link KeyValueStore.getValue} function.
+     *
+     * For more information, see  {@link KeyValueStore.open}
+     * and  {@link KeyValueStore.getValue}.
+     *
+     * @param key
+     *   Unique record key.
+     * @param value
+     *   Record data, which can be one of the following values:
+     *    - If `null`, the record in the key-value store is deleted.
+     *    - If no `options.contentType` is specified, `value` can be any JavaScript object, and it will be stringified to JSON.
+     *    - If `options.contentType` is set, `value` is taken as is, and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
+     *   For any other value an error will be thrown.
+     * @param [options]
+     * @ignore
+     */
+    static async setValue(key, value, options = {}) {
+        const store = await this.open();
+        return store.setValue(key, value, options);
+    }
+    /**
+     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
+     *
+     * This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
+     * For example, calling the following code:
+     * ```javascript
+     * const input = await KeyValueStore.getInput();
+     * ```
+     *
+     * is equivalent to:
+     * ```javascript
+     * const store = await KeyValueStore.open();
+     * await store.getValue('INPUT');
+     * ```
+     *
+     * Note that the `getInput()` function does not cache the value read from the key-value store.
+     * If you need to use the input multiple times in your crawler,
+     * it is far more efficient to read it once and store it locally.
+     *
+     * For more information, see  {@link KeyValueStore.open}
+     * and {@link KeyValueStore.getValue}.
+     *
+     * @returns
+     *   Returns a promise that resolves to an object, string
+     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
+     *   on the MIME content type of the record, or `null`
+     *   if the record is missing.
+     * @ignore
+     */
+    static async getInput() {
+        const store = await this.open();
+        return store.getValue(store.config.get('inputKey'));
+    }
 }
 exports.KeyValueStore = KeyValueStore;
 //# sourceMappingURL=key_value_store.js.map

package/storages/key_value_store.js.map

@@ -1 +1 @@
-{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAEjD,uDAA0E;AAE1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAKtB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAPpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAAoC;QAMhC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,IAAI,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,GAAW;QACpB,uGAAuG;QACvG,OAAO,6CAA6C,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IACjF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;CACJ;AA/MD,sCA+MC"}
+{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAmB;WAAC;QAMhD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QAE7C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,KAAK,KAAK,EAAE;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAM,CAAC,CAAC;qBAC7B,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;aACrC;SACJ;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,MAAM,IAAA,4BAAoB,GAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAEzE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,CAAC;CACJ;AA1WD,sCA0WC"}

package/storages/request_list.d.ts

@@ -1,7 +1,8 @@
 /// <reference types="node" />
 import { Configuration } from '../configuration';
-import { Request, RequestOptions } from '../request';
-import { ProxyConfiguration } from '../proxy_configuration';
+import type { RequestOptions } from '../request';
+import { Request } from '../request';
+import type { ProxyConfiguration } from '../proxy_configuration';
 /** @internal */
 export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
 /** @internal */
@@ -49,7 +50,7 @@ export interface RequestListOptions {
      * This is very useful in a scenario when getting the sources is a resource intensive or time consuming
      * task, such as fetching URLs from multiple sitemaps or parsing URLs from large datasets. Using the
      * `sourcesFunction` in combination with `persistStateKey` and `persistRequestsKey` will allow you to
-     * fetch and parse those URLs only once, saving valuable time when your actor migrates or restarts.
+     * fetch and parse those URLs only once, saving valuable time when your crawler migrates or restarts.
      *
      * If both {@link RequestListOptions.sources} and {@link RequestListOptions.sourcesFunction} are provided,
      * the sources returned by the function will be added after the `sources`.
@@ -63,22 +64,20 @@ export interface RequestListOptions {
      *     // and big websites typically have multiple sitemaps.
      *     const sitemaps = await downloadHugeSitemaps();
      *     return parseUrlsFromSitemaps(sitemaps);
-     * }
+     * };
      *
      * // Sitemaps can change in real-time, so it's important to persist
      * // the URLs we collected. Otherwise we might lose our scraping
-     * // state in case of an actor migration / failure / time-out.
-     * const requestList = new RequestList({
+     * // state in case of an crawler migration / failure / time-out.
+     * const requestList = await RequestList.open(null, [], {
+     *     // The sourcesFunction is called now and the Requests are persisted.
+     *     // If something goes wrong and we need to start again, RequestList
+     *     // will load the persisted Requests from storage and will NOT
+     *     // call the sourcesFunction again, saving time and resources.
      *     sourcesFunction,
      *     persistStateKey: 'state-key',
      *     persistRequestsKey: 'requests-key',
      * })
-     *
-     * // The sourcesFunction is called now and the Requests are persisted.
-     * // If something goes wrong and we need to start again, RequestList
-     * // will load the persisted Requests from storage and will NOT
-     * // call the sourcesFunction again, saving time and resources.
-     * await requestList.initialize();
      * ```
      */
     sourcesFunction?: RequestListSourcesFunction;
@@ -91,7 +90,7 @@ export interface RequestListOptions {
     /**
      * Identifies the key in the default key-value store under which `RequestList` periodically stores its
      * state (i.e. which URLs were crawled and which not).
-     * If the actor is restarted, `RequestList` will read the state
+     * If the crawler is restarted, `RequestList` will read the state
      * and continue where it left off.
      *
      * If `persistStateKey` is not set, `RequestList` will always start from the beginning,
@@ -174,9 +173,9 @@ export interface RequestListOptions {
  * which are in progress and which were reclaimed. The state may be automatically persisted to the default
  * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
- * event that is periodically emitted by {@link events|Actor.events}.
+ * event that is periodically emitted by {@link EventManager}.
  *
- * The internal state is closely tied to the provided sources (URLs). If the sources change on actor restart, the state will become corrupted and
+ * The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
  * `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
  * In such case, use the `persistRequestsKey` option in conjunction with `persistStateKey`,
  * to make the `RequestList` store the initial sources to the default key-value store and load them after restart,
@@ -196,23 +195,19 @@ export interface RequestListOptions {
  * **Advanced usage:**
  * ```javascript
  * // Use the constructor to get more control over the initialization.
- * const requestList = new RequestList({
- *     sources: [
- *         // Separate requests
- *         { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
- *         { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
- *
- *         // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
- *         // Note that all URLs must start with http:// or https://
- *         { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
- *     ],
+ * const requestList = await RequestList.open(null, [
+ *     // Separate requests
+ *     { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
+ *     { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
  *
+ *     // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
+ *     // Note that all URLs must start with http:// or https://
+ *     { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ * ], {
  *     // Persist the state to avoid re-crawling which can lead to data duplications.
  *     // Keep in mind that the sources have to be immutable or this will throw an error.
  *     persistStateKey: 'my-state',
  * });
- *
- * await requestList.initialize();
  * ```
  * @category Sources
  */
@@ -260,14 +255,16 @@ export declare class RequestList {
     private proxyConfiguration?;
     private events;
     /**
+     * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
+     * @internal
      */
-    constructor(options?: RequestListOptions);
+    private constructor();
     /**
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
      * This function must be called before you can start using the instance in a meaningful way.
      */
-    initialize(): Promise<void>;
+    private initialize;
     /**
      * Adds previously persisted Requests, as retrieved from the key-value store.
      * This needs to be done in a memory efficient way. We should update the input
@@ -396,10 +393,10 @@ export declare class RequestList {
      * const requestList = await RequestList.open('my-name', sources);
      * ```
      *
-     * @param listName
-     *   Name of the request list to be opened. Setting a name enables the `RequestList`'s state to be persisted
-     *   in the key-value store. This is useful in case of a restart or migration. Since `RequestList` is only
-     *   stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
+     * @param listNameOrOptions
+     *   Name of the request list to be opened, or the options object. Setting a name enables the `RequestList`'s
+     *   state to be persisted in the key-value store. This is useful in case of a restart or migration. Since `RequestList`
+     *   is only stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
      *   state to survive those situations and continue where it left off.
      *
      *   The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE`
@@ -407,7 +404,7 @@ export declare class RequestList {
      *
      *   If `null`, the list will not be persisted and will only be stored in memory. Process restart
      *   will then cause the list to be crawled again from the beginning. We suggest always using a name.
-     * @param sources
+     * @param [sources]
      *  An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
      *  plain objects that define at least the `url` property, or an array of {@link Request} instances.
      *
@@ -426,7 +423,7 @@ export declare class RequestList {
      *   the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
      *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
      */
-    static open(listName: string | null, sources: Source[], options?: RequestListOptions): Promise<RequestList>;
+    static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
     /**
      * @internal wraps public utility for mocking purposes
      */

package/storages/request_list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;MAIE;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6C5C;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBjC;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IAgBrH;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,oBAAY,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AACjH,aAAK,cAAc,GAAG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAClE,oBAAY,0BAA0B,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}
+{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;MAIE;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,MAAM,CAAe;IAE7B;;;;OAIG;IACH,OAAO;IA6CP;;;OAGG;YACW,UAAU;IA4BxB;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,GAAG,kBAAkB,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0BpJ;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,oBAAY,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AACjH,aAAK,cAAc,GAAG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAClE,oBAAY,0BAA0B,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}

package/storages/request_list.js

@@ -8,7 +8,8 @@ const configuration_1 = require("../configuration");
 const log_1 = require("../log");
 const request_1 = require("../request");
 const serialization_1 = require("../serialization");
-const key_value_store_1 = require("../storages/key_value_store");
+const key_value_store_1 = require("./key_value_store");
+const utils_2 = require("./utils");
 /** @internal */
 exports.STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
 /** @internal */
@@ -40,9 +41,9 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * which are in progress and which were reclaimed. The state may be automatically persisted to the default
  * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
- * event that is periodically emitted by {@link events|Actor.events}.
+ * event that is periodically emitted by {@link EventManager}.
  *
- * The internal state is closely tied to the provided sources (URLs). If the sources change on actor restart, the state will become corrupted and
+ * The internal state is closely tied to the provided sources (URLs). If the sources change on crawler restart, the state will become corrupted and
  * `RequestList` will raise an exception. This typically happens when the sources is a list of URLs downloaded from the web.
  * In such case, use the `persistRequestsKey` option in conjunction with `persistStateKey`,
  * to make the `RequestList` store the initial sources to the default key-value store and load them after restart,
@@ -62,29 +63,27 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * **Advanced usage:**
  * ```javascript
  * // Use the constructor to get more control over the initialization.
- * const requestList = new RequestList({
- *     sources: [
- *         // Separate requests
- *         { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
- *         { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
- *
- *         // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
- *         // Note that all URLs must start with http:// or https://
- *         { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
- *     ],
+ * const requestList = await RequestList.open(null, [
+ *     // Separate requests
+ *     { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
+ *     { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
  *
+ *     // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
+ *     // Note that all URLs must start with http:// or https://
+ *     { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ * ], {
  *     // Persist the state to avoid re-crawling which can lead to data duplications.
  *     // Keep in mind that the sources have to be immutable or this will throw an error.
  *     persistStateKey: 'my-state',
  * });
- *
- * await requestList.initialize();
  * ```
  * @category Sources
  */
 class RequestList {
     /**
+     * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
+     * @internal
      */
     constructor(options = {}) {
         Object.defineProperty(this, "log", {
@@ -248,7 +247,7 @@ class RequestList {
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
         this.keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
-        this.sources = sources || [];
+        this.sources = sources ? [...sources] : [];
         this.sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrls` requests.
         this.proxyConfiguration = proxyConfiguration;
@@ -262,6 +261,7 @@ class RequestList {
             throw new Error('RequestList sources are already loading or were loaded.');
         }
         this.isLoading = true;
+        await (0, utils_2.purgeDefaultStorages)();
         const [state, persistedRequests] = await this._loadStateAndPersistedRequests();
         // Add persisted requests / new sources in a memory efficient way because with very
         // large lists, we were running out of memory.
@@ -278,6 +278,7 @@ class RequestList {
         if (this.persistStateKey) {
             this.events.on("persistState" /* EventType.PERSIST_STATE */, this.persistState.bind(this));
         }
+        return this;
     }
     /**
      * Adds previously persisted Requests, as retrieved from the key-value store.
@@ -382,7 +383,7 @@ class RequestList {
             throw new Error('The state object is invalid: nextIndex must be a non-negative number.');
         }
         if (state.nextIndex > this.requests.length) {
-            throw new Error('The state object is not consistent with RequestList too few requests loaded.');
+            throw new Error('The state object is not consistent with RequestList, too few requests loaded.');
         }
         if (state.nextIndex < this.requests.length
             && this.requests[state.nextIndex].uniqueKey !== state.nextUniqueKey) {
@@ -670,10 +671,10 @@ class RequestList {
      * const requestList = await RequestList.open('my-name', sources);
      * ```
      *
-     * @param listName
-     *   Name of the request list to be opened. Setting a name enables the `RequestList`'s state to be persisted
-     *   in the key-value store. This is useful in case of a restart or migration. Since `RequestList` is only
-     *   stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
+     * @param listNameOrOptions
+     *   Name of the request list to be opened, or the options object. Setting a name enables the `RequestList`'s
+     *   state to be persisted in the key-value store. This is useful in case of a restart or migration. Since `RequestList`
+     *   is only stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
      *   state to survive those situations and continue where it left off.
      *
      *   The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE`
@@ -681,7 +682,7 @@ class RequestList {
      *
      *   If `null`, the list will not be persisted and will only be stored in memory. Process restart
      *   will then cause the list to be crawled again from the beginning. We suggest always using a name.
-     * @param sources
+     * @param [sources]
      *  An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
      *  plain objects that define at least the `url` property, or an array of {@link Request} instances.
      *
@@ -700,8 +701,15 @@ class RequestList {
      *   the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
      *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
      */
-    static async open(listName, sources, options = {}) {
-        (0, ow_1.default)(listName, ow_1.default.any(ow_1.default.string, ow_1.default.null));
+    static async open(listNameOrOptions, sources, options = {}) {
+        if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
+            options = { ...listNameOrOptions, ...options };
+            const rl = new RequestList(options);
+            await rl.initialize();
+            return rl;
+        }
+        const listName = listNameOrOptions;
+        (0, ow_1.default)(listName, ow_1.default.optional.any(ow_1.default.string, ow_1.default.null));
         (0, ow_1.default)(sources, ow_1.default.array);
         (0, ow_1.default)(options, ow_1.default.object.is((v) => !Array.isArray(v)));
         const rl = new RequestList({