@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.71

package/storages/storage_instance_manager.js

@@ -0,0 +1,261 @@
+import { AsyncQueue } from '@sapphire/async-queue';
+/** Reserved alias for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/**
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
+ *
+ * Each tier maps `[storageClass][key][backendCacheKey] → instance`:
+ *   - `byId`    — keyed by the backend-assigned storage id
+ *   - `byName`  — keyed by the persistent storage name
+ *   - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
+ */
+class StorageCache {
+    byId = new Map();
+    byName = new Map();
+    byAlias = new Map();
+    get(cls, { id, name, alias, backendCacheKey, }) {
+        for (const [tier, key] of [
+            [this.byId, id],
+            [this.byName, name],
+            [this.byAlias, alias],
+        ]) {
+            if (key === undefined)
+                continue;
+            const cached = tier.get(cls)?.get(key)?.get(backendCacheKey);
+            if (cached) {
+                if (cached instanceof cls) {
+                    return cached;
+                }
+                throw new Error('Cached storage instance type mismatch.');
+            }
+        }
+        return undefined;
+    }
+    /** Write a single entry into a given tier. */
+    setInMap(tier, cls, key, instance, backendCacheKey) {
+        if (!tier.has(cls))
+            tier.set(cls, new Map());
+        const keyMap = tier.get(cls);
+        if (!keyMap.has(key))
+            keyMap.set(key, new Map());
+        keyMap.get(key).set(backendCacheKey, instance);
+    }
+    /**
+     * Cache an instance under its actual id, name, and an optional alias.
+     */
+    set(cls, instance, backendCacheKey, alias) {
+        // Always cache by id.
+        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        // Cache by name — only for named storages.
+        if (instance.name) {
+            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+        }
+        // Cache by alias — only for unnamed storages opened via alias.
+        if (alias !== undefined) {
+            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+        }
+    }
+    removeFromCache(instance) {
+        const storageType = instance.constructor;
+        for (const tier of [this.byId, this.byName, this.byAlias]) {
+            const classMap = tier.get(storageType);
+            if (!classMap)
+                continue;
+            for (const keyMap of classMap.values()) {
+                for (const [cacheKey, cached] of keyMap) {
+                    if (cached === instance) {
+                        keyMap.delete(cacheKey);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Ensure that the same string is not used as both a name and an alias for the same
+     * storage class + backend combination. Mirrors crawlee-python's `_check_name_alias_conflict`.
+     */
+    checkNameAliasConflict(cls, { name, alias, backendCacheKey }) {
+        if (alias) {
+            const existingByName = this.byName.get(cls)?.get(alias)?.get(backendCacheKey);
+            if (existingByName) {
+                throw new Error(`Cannot open storage with alias "${alias}" because a named storage with the same identifier already exists.`);
+            }
+        }
+        if (name) {
+            const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(backendCacheKey);
+            if (existingByAlias) {
+                throw new Error(`Cannot open storage with name "${name}" because an alias storage with the same identifier already exists.` +
+                    ` If you meant to open the alias storage, use { alias: "${name}" } instead.`);
+            }
+        }
+    }
+    /** Iterate all cached instances across all storage types. */
+    *allValues() {
+        const seen = new Set();
+        for (const classMap of this.byId.values()) {
+            for (const keyMap of classMap.values()) {
+                for (const instance of keyMap.values()) {
+                    if (!seen.has(instance)) {
+                        seen.add(instance);
+                        yield instance;
+                    }
+                }
+            }
+        }
+    }
+    clear() {
+        this.byId.clear();
+        this.byName.clear();
+        this.byAlias.clear();
+    }
+}
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ *
+ * @ignore
+ */
+export class StorageInstanceManager {
+    cache = new StorageCache();
+    openerLocks = new Map();
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param backendOpener      A **lazy** factory that creates the sub-backend.
+     *                          Only called on a cache miss.
+     * @param backendCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    async openStorage(cls, { id, name, alias, backendOpener, backendCacheKey, }) {
+        // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
+        if (!id && !name && !alias) {
+            alias = DEFAULT_STORAGE_ALIAS;
+        }
+        // Fast-path cache check (no lock).
+        if (alias !== undefined) {
+            const cached = this.cache.get(cls, { alias, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (id) {
+            const cached = this.cache.get(cls, { id, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (name) {
+            const cached = this.cache.get(cls, { name, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
+        const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
+        if (!this.openerLocks.has(lockKey)) {
+            this.openerLocks.set(lockKey, new AsyncQueue());
+        }
+        const queue = this.openerLocks.get(lockKey);
+        await queue.wait();
+        try {
+            // Double-check cache under lock (another caller may have filled it while we waited).
+            if (alias !== undefined) {
+                const cached = this.cache.get(cls, { alias, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (id) {
+                const cached = this.cache.get(cls, { id, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (name) {
+                const cached = this.cache.get(cls, { name, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            // Prevent the same string from being used as both a name and an alias.
+            this.cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
+            // Cache miss — create the sub-backend and storage instance.
+            const subBackend = await backendOpener();
+            const storageInfo = await subBackend.getMetadata();
+            const instance = new cls({
+                id: storageInfo.id,
+                name: storageInfo.name,
+                backend: subBackend,
+            });
+            // Atomic cache writes (no awaits between these).
+            this.cache.set(cls, instance, backendCacheKey, alias);
+            return instance;
+        }
+        finally {
+            queue.shift();
+            // Clean up idle locks so the map doesn't grow unboundedly
+            // (mirrors crawlee-python's WeakValueDictionary behaviour).
+            if (queue.remaining === 0) {
+                this.openerLocks.delete(lockKey);
+            }
+        }
+    }
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance) {
+        this.cache.removeFromCache(instance);
+    }
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache() {
+        for (const instance of this.cache.allValues()) {
+            if ('clearCache' in instance && typeof instance.clearCache === 'function') {
+                instance.clearCache();
+            }
+        }
+        this.cache.clear();
+    }
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
+ */
+export async function resolveStorageIdentifier(identifier, storageBackend, storageType) {
+    if (identifier === null || identifier === undefined) {
+        return { alias: DEFAULT_STORAGE_ALIAS };
+    }
+    if (typeof identifier === 'string') {
+        if (storageBackend.storageExists && (await storageBackend.storageExists(identifier, storageType))) {
+            return { id: identifier };
+        }
+        return { name: identifier };
+    }
+    if (identifier.id) {
+        return { id: identifier.id };
+    }
+    if (identifier.name) {
+        return { name: identifier.name };
+    }
+    if ('alias' in identifier && identifier.alias) {
+        return { alias: identifier.alias };
+    }
+    // Empty object — treated as default storage.
+    return { alias: DEFAULT_STORAGE_ALIAS };
+}
+//# sourceMappingURL=storage_instance_manager.js.map

package/storages/storage_instance_manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_instance_manager.js","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBnD,wDAAwD;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,YAAY;IACL,IAAI,GAAc,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAC9B,OAAO,GAAc,IAAI,GAAG,EAAE,CAAC;IAExC,GAAG,CACC,GAAmB,EACnB,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,eAAe,GAKc;QAEjC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI;YACtB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;YACnB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACa,EAAE,CAAC;YACrC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC7D,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,MAAM,YAAa,GAAuD,EAAE,CAAC;oBAC7E,OAAO,MAAW,CAAC;gBACvB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,8CAA8C;IACtC,QAAQ,CACZ,IAAe,EACf,GAAmB,EACnB,GAAW,EACX,QAAW,EACX,eAAyB;QAEzB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,GAAG,CAAqB,GAAmB,EAAE,QAAW,EAAE,eAAyB,EAAE,KAAc;QAC/F,sBAAsB;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QAEtE,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QAC9E,CAAC;QAED,+DAA+D;QAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAED,eAAe,CAAC,QAAkB;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAoC,CAAC;QAElE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;oBACtC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC5B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,sBAAsB,CAClB,GAAmB,EACnB,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAgE;QAE9F,IAAI,KAAK,EAAE,CAAC;YACR,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC9E,IAAI,cAAc,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CACX,mCAAmC,KAAK,oEAAoE,CAC/G,CAAC;YACN,CAAC;QACL,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC/E,IAAI,eAAe,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACX,kCAAkC,IAAI,qEAAqE;oBACvG,0DAA0D,IAAI,cAAc,CACnF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,CAAC,SAAS;QACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAY,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACnB,MAAM,QAAQ,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,sBAAsB;IACd,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,WAAW,CACb,GAA0B,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,aAAa,EACb,eAAe,GAIlB;QAED,0FAA0F;QAC1F,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACzB,KAAK,GAAG,qBAAqB,CAAC;QAClC,CAAC;QAED,mCAAmC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC/D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC;YAC5D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;QAElE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAE7C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC;YACD,qFAAqF;YACrF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC/D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,EAAE,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC5D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC9D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;YAED,uEAAuE;YACvE,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAEzE,4DAA4D;YAC5D,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;YACzC,MAAM,WAAW,GAAG,MAChB,UACH,CAAC,WAAW,EAAE,CAAC;YAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;gBACrB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,OAAO,EAAE,UAAU;aACtB,CAAa,CAAC;YAEf,iDAAiD;YACjD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;YAEtD,OAAO,QAAQ,CAAC;QACpB,CAAC;gBAAS,CAAC;YACP,KAAK,CAAC,KAAK,EAAE,CAAC;YAEd,0DAA0D;YAC1D,4DAA4D;YAC5D,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,QAAkB;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,UAAU;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;YAC5C,IAAI,YAAY,IAAI,QAAQ,IAAI,OAAQ,QAAgB,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAChF,QAAgB,CAAC,UAAU,EAAE,CAAC;YACnC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACJ;AAoBD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,UAAyD,EACzD,cAA8B,EAC9B,WAAyD;IAEzD,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACjC,IAAI,cAAc,CAAC,aAAa,IAAI,CAAC,MAAM,cAAc,CAAC,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChG,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,IAAI,OAAO,IAAI,UAAU,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QAC5C,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,6CAA6C;IAC7C,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;AAC5C,CAAC"}

package/storages/storage_stats.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/** Usage counters for a {@link Dataset}. */
+export interface DatasetStats {
+    /** Number of read operations issued to the dataset client (e.g. `getData`). */
+    readCount: number;
+    /** Number of write operations issued to the dataset client (e.g. `pushData`). */
+    writeCount: number;
+}
+/** Usage counters for a {@link KeyValueStore}. */
+export interface KeyValueStoreStats {
+    /** Number of read operations issued to the key-value store client (e.g. `getValue`). */
+    readCount: number;
+    /** Number of write operations issued to the key-value store client (e.g. `setValue`). */
+    writeCount: number;
+    /** Number of delete operations issued to the key-value store client (e.g. `deleteValue`). */
+    deleteCount: number;
+    /** Number of listing operations issued to the key-value store client (e.g. `listKeys`). */
+    listCount: number;
+}
+/** Usage counters for a {@link RequestQueue}. */
+export interface RequestQueueStats {
+    /** Number of write operations issued to the request queue client (add / handle / reclaim). */
+    writeCount: number;
+    /** Number of queue-head reads issued to the request queue client (`fetchNextRequest`). */
+    headItemReadCount: number;
+}
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
+    private readonly counters;
+    constructor(initial: T);
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key: keyof T, by?: number): void;
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current(): T;
+}
+//# sourceMappingURL=storage_stats.d.ts.map

package/storages/storage_stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_stats.d.ts","sourceRoot":"","sources":["../../src/storages/storage_stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,+CAA+C;AAC/C,MAAM,WAAW,YAAY;IACzB,+EAA+E;IAC/E,SAAS,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,qDAAqD;AACrD,MAAM,WAAW,kBAAkB;IAC/B,wFAAwF;IACxF,SAAS,EAAE,MAAM,CAAC;IAClB,yFAAyF;IACzF,UAAU,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;IACpB,2FAA2F;IAC3F,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAC9B,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED;;;;GAIG;AACH,qBAAa,mBAAmB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAC9D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAI;gBAEjB,OAAO,EAAE,CAAC;IAItB,wDAAwD;IACxD,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,EAAE,SAAI,GAAG,IAAI;IAI/B,iGAAiG;IACjG,IAAI,OAAO,IAAI,CAAC,CAEf;CACJ"}

package/storages/storage_stats.js

@@ -0,0 +1,30 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export class StorageStatsTracker {
+    counters;
+    constructor(initial) {
+        this.counters = { ...initial };
+    }
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key, by = 1) {
+        this.counters[key] += by;
+    }
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current() {
+        return { ...this.counters };
+    }
+}
+//# sourceMappingURL=storage_stats.js.map

package/storages/storage_stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_stats.js","sourceRoot":"","sources":["../../src/storages/storage_stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA8BH;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACX,QAAQ,CAAI;IAE7B,YAAY,OAAU;QAClB,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACnC,CAAC;IAED,wDAAwD;IACxD,GAAG,CAAC,GAAY,EAAE,EAAE,GAAG,CAAC;QACnB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAY,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,IAAI,OAAO;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;IAChC,CAAC;CACJ"}

package/storages/utils.d.ts

@@ -1,5 +1,6 @@
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
+import type { ProxyConfiguration } from '../proxy_configuration.js';
 /**
  * Options for purging default storage.
  */
@@ -9,7 +10,7 @@ interface PurgeDefaultStorageOptions {
      */
     onlyPurgeOnce?: boolean;
     config?: Configuration;
-    client?: StorageClient;
+    storageBackend?: StorageBackend;
 }
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
@@ -19,7 +20,7 @@ interface PurgeDefaultStorageOptions {
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
  * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
  *
- * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
+ * This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
  * it will call the `purge` method of the underlying storage implementation we are currently using. You can
  * make sure the storage is purged only once for a given execution context if you set `onlyPurgeOnce` to `true` in
  * the `options` object
@@ -33,10 +34,10 @@ export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOption
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
  * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
  *
- * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
+ * This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
  * it will call the `purge` method of the underlying storage implementation we are currently using.
  */
-export declare function purgeDefaultStorages(config?: Configuration, client?: StorageClient): Promise<void>;
+export declare function purgeDefaultStorages(config?: Configuration, storageBackend?: StorageBackend): Promise<void>;
 export interface UseStateOptions {
     config?: Configuration;
     /**
@@ -89,5 +90,50 @@ export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
  * @internal
  */
 export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
+/** @internal */
+export interface DualIterableOptions<TItem, TRawPage> {
+    /** Factory that returns an async generator yielding pages. */
+    createPages: () => AsyncGenerator<TRawPage>;
+    /** Extracts individual items from a page (for iteration). */
+    extractItems: (page: TRawPage) => TItem[];
+}
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export declare function createDualIterable<TItem, TRawPage>(options: DualIterableOptions<TItem, TRawPage>): AsyncIterable<TItem> & Promise<TItem[]>;
+/**
+ * Options for the static `open()` method on storage classes ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ */
+export interface StorageOpenOptions {
+    /**
+     * SDK configuration instance, defaults to the static register.
+     */
+    config?: Configuration;
+    /**
+     * Optional storage backend that should be used to open storages.
+     */
+    storageBackend?: StorageBackend;
+    /**
+     * Used to pass the proxy configuration for the `requestsFromUrl` objects.
+     * Takes advantage of the internal address rotation and authentication process.
+     * If undefined, the `requestsFromUrl` requests will be made without proxy.
+     */
+    proxyConfiguration?: ProxyConfiguration;
+    /**
+     * HTTP client to be used to download the list of URLs in `RequestQueue`.
+     */
+    httpClient?: BaseHttpClient;
+}
 export {};
 //# sourceMappingURL=utils.d.ts.map

package/storages/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEjF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,cAAc,CAAC,EAAE,cAAc,CAAC;CACnC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwBnH,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAE7C,gBAAgB;AAChB,MAAM,WAAW,mBAAmB,CAAC,KAAK,EAAE,QAAQ;IAChD,8DAA8D;IAC9D,WAAW,EAAE,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,6DAA6D;IAC7D,YAAY,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,KAAK,EAAE,CAAC;CAC7C;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAC9C,OAAO,EAAE,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,GAC9C,aAAa,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CA6CzC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B"}

package/storages/utils.js

@@ -1,18 +1,19 @@
 import crypto from 'node:crypto';
 import { Configuration } from '../configuration.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from './key_value_store.js';
-export async function purgeDefaultStorages(configOrOptions, client) {
+export async function purgeDefaultStorages(configOrOptions, storageBackend) {
     const options = configOrOptions instanceof Configuration
         ? {
-            client,
+            storageBackend,
             config: configOrOptions,
         }
         : (configOrOptions ?? {});
-    const { config = Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
-    ({ client = config.getStorageClient() } = options);
-    const casted = client;
+    const { config = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
+    ({ storageBackend = serviceLocator.getStorageBackend() } = options);
+    const casted = storageBackend;
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
-    if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
+    if (!onlyPurgeOnce || (config.purgeOnStart && !casted.__purged)) {
         casted.__purged = true;
         await casted.purge?.();
     }
@@ -27,8 +28,8 @@ export async function purgeDefaultStorages(configOrOptions, client) {
  * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
  */
 export async function useState(name, defaultValue = {}, options) {
-    const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
-        config: options?.config || Configuration.getGlobalConfig(),
+    const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
+        config: options?.config || serviceLocator.getConfiguration(),
     });
     return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
 }
@@ -69,4 +70,55 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
  * @internal
  */
 export const MAX_QUERIES_FOR_CONSISTENCY = 6;
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export function createDualIterable(options) {
+    const { createPages, extractItems } = options;
+    let cached = null;
+    function getOrCreate() {
+        if (!cached) {
+            cached = (async () => {
+                const items = [];
+                for await (const page of createPages()) {
+                    items.push(...extractItems(page));
+                }
+                return items;
+            })();
+        }
+        return cached;
+    }
+    async function* iterateAll() {
+        for await (const page of createPages()) {
+            yield* extractItems(page);
+        }
+    }
+    const result = {
+        [Symbol.asyncIterator]() {
+            return iterateAll();
+        },
+        then(onfulfilled, onrejected) {
+            return getOrCreate().then(onfulfilled, onrejected);
+        },
+        catch(onrejected) {
+            return getOrCreate().catch(onrejected);
+        },
+        finally(onfinally) {
+            return getOrCreate().finally(onfinally);
+        },
+        [Symbol.toStringTag]: 'DualIterable',
+    };
+    return result;
+}
 //# sourceMappingURL=utils.js.map

package/storages/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,cAA+B;IAE/B,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,cAAc;YACd,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,CAAC,EAAE,cAAc,GAAG,cAAc,CAAC,iBAAiB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEpE,MAAM,MAAM,GAAG,cAAyD,CAAC;IAEzE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9G,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE;KAC/D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAU7C;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAC9B,OAA6C;IAE7C,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAC9C,IAAI,MAAM,GAA4B,IAAI,CAAC;IAE3C,SAAS,WAAW;QAChB,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,GAAG,CAAC,KAAK,IAAI,EAAE;gBACjB,MAAM,KAAK,GAAY,EAAE,CAAC;gBAC1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,WAAW,EAAE,EAAE,CAAC;oBACrC,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;gBACtC,CAAC;gBACD,OAAO,KAAK,CAAC;YACjB,CAAC,CAAC,EAAE,CAAC;QACT,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,SAAS,CAAC,CAAC,UAAU;QACtB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,WAAW,EAAE,EAAE,CAAC;YACrC,KAAK,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG;QACX,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,UAAU,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,CACA,WAA2E,EAC3E,UAAuE;YAEvE,OAAO,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACvD,CAAC;QACD,KAAK,CACD,UAAqE;YAErE,OAAO,WAAW,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,CAAC,SAA+B;YACnC,OAAO,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,cAAc;KACI,CAAC;IAE7C,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/typedefs.d.ts

@@ -6,5 +6,5 @@ export type Awaitable<T> = T | PromiseLike<T>;
 export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
 /** @ignore */
 export declare function keys<T extends {}>(obj: T): (keyof T)[];
-export declare type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH';
+export type { AllowedHttpMethods } from '@crawlee/types';
 //# sourceMappingURL=typedefs.d.ts.map

package/typedefs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,MAAM,CAAC,OAAO,MAAM,kBAAkB,GAChC,KAAK,GACL,MAAM,GACN,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,OAAO,CAAC"}
+{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}

package/validators.d.ts

@@ -17,5 +17,13 @@ export declare const validators: {
         validator: boolean;
         message: (label: string) => string;
     };
+    browserPool: (value: Dictionary) => {
+        validator: boolean;
+        message: (label: string) => string;
+    };
+    sessionPool: (value: Dictionary) => {
+        validator: boolean;
+        message: (label: string) => string;
+    };
 };
 //# sourceMappingURL=validators.d.ts.map

package/validators.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;CAE9B,CAAC"}
+{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;yBAGN,UAAU;;yBAEV,MAAM;;CAG9B,CAAC"}

package/validators.js

@@ -18,5 +18,13 @@ export const validators = {
         validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'addRequest')),
         message: (label) => `Expected argument '${label}' to be a RequestQueue, got something else.`,
     }),
+    browserPool: (value) => ({
+        validator: ow.isValid(value, ow.object.hasKeys('newPage', 'closePage', 'extractPageState', 'injectPageState')),
+        message: (label) => `Expected argument '${label}' to implement the IBrowserPool interface (missing one of 'newPage', 'closePage', 'extractPageState', 'injectPageState'), got something else.`,
+    }),
+    sessionPool: (value) => ({
+        validator: ow.isValid(value, ow.object.hasKeys('getSession')),
+        message: (label) => `Expected argument '${label}' to implement the ISessionPool interface (missing 'getSession'), got something else.`,
+    }),
 };
 //# sourceMappingURL=validators.js.map

package/validators.js.map

@@ -1 +1 @@
-{"version":3,"file":"validators.js","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gBAAgB;AAChB,MAAM,CAAC,MAAM,UAAU,GAAG;IACtB,6DAA6D;IAC7D,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,+CAA+C;KACzG,CAAC;IACF,kBAAkB,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACxC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,mDAAmD;KAC7G,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,4CAA4C;KACtG,CAAC;IACF,YAAY,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACjF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,6CAA6C;KACvG,CAAC;CACL,CAAC"}
+{"version":3,"file":"validators.js","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gBAAgB;AAChB,MAAM,CAAC,MAAM,UAAU,GAAG;IACtB,6DAA6D;IAC7D,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,+CAA+C;KACzG,CAAC;IACF,kBAAkB,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACxC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,mDAAmD;KAC7G,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,4CAA4C;KACtG,CAAC;IACF,YAAY,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACjF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,6CAA6C;KACvG,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,kBAAkB,EAAE,iBAAiB,CAAC,CAAC;QAC9G,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CACvB,sBAAsB,KAAK,+IAA+I;KACjL,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC7D,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CACvB,sBAAsB,KAAK,uFAAuF;KACzH,CAAC;CACL,CAAC"}

package/crawlers/crawler_extension.d.ts

@@ -1,12 +0,0 @@
-import { type Log } from '../log.js';
-/**
- * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
- * @category Crawlers
- * @ignore
- */
-export declare abstract class CrawlerExtension {
-    name: string;
-    log: Log;
-    getCrawlerOptions(): Record<string, unknown>;
-}
-//# sourceMappingURL=crawler_extension.d.ts.map

package/crawlers/crawler_extension.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_extension.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,GAAG,EAAqB,MAAM,WAAW,CAAC;AAExD;;;;GAIG;AACH,8BAAsB,gBAAgB;IAClC,IAAI,SAAyB;IAC7B,GAAG,EAAE,GAAG,CAA2C;IAEnD,iBAAiB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAG/C"}