@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.61

package/storages/storage_instance_manager.js

@@ -0,0 +1,236 @@
+import { AsyncQueue } from '@sapphire/async-queue';
+/** Reserved alias for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/**
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
+ *
+ * Each tier maps `[storageClass][key][clientCacheKey] → instance`:
+ *   - `byId`    — keyed by the backend-assigned storage id
+ *   - `byName`  — keyed by the persistent storage name
+ *   - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
+ */
+class StorageCache {
+    byId = new Map();
+    byName = new Map();
+    byAlias = new Map();
+    get(cls, { id, name, alias, clientCacheKey, }) {
+        for (const [tier, key] of [
+            [this.byId, id],
+            [this.byName, name],
+            [this.byAlias, alias],
+        ]) {
+            if (key === undefined)
+                continue;
+            const cached = tier.get(cls)?.get(key)?.get(clientCacheKey);
+            if (cached) {
+                if (cached instanceof cls) {
+                    return cached;
+                }
+                throw new Error('Cached storage instance type mismatch.');
+            }
+        }
+        return undefined;
+    }
+    /** Write a single entry into a given tier. */
+    setInMap(tier, cls, key, instance, clientCacheKey) {
+        if (!tier.has(cls))
+            tier.set(cls, new Map());
+        const keyMap = tier.get(cls);
+        if (!keyMap.has(key))
+            keyMap.set(key, new Map());
+        keyMap.get(key).set(clientCacheKey, instance);
+    }
+    /**
+     * Cache an instance under its actual id, name, and an optional alias.
+     */
+    set(cls, instance, clientCacheKey, alias) {
+        // Always cache by id.
+        this.setInMap(this.byId, cls, instance.id, instance, clientCacheKey);
+        // Cache by name — only for named storages.
+        if (instance.name) {
+            this.setInMap(this.byName, cls, instance.name, instance, clientCacheKey);
+        }
+        // Cache by alias — only for unnamed storages opened via alias.
+        if (alias !== undefined) {
+            this.setInMap(this.byAlias, cls, alias, instance, clientCacheKey);
+        }
+    }
+    removeFromCache(instance) {
+        const storageType = instance.constructor;
+        for (const tier of [this.byId, this.byName, this.byAlias]) {
+            const classMap = tier.get(storageType);
+            if (!classMap)
+                continue;
+            for (const keyMap of classMap.values()) {
+                for (const [cacheKey, cached] of keyMap) {
+                    if (cached === instance) {
+                        keyMap.delete(cacheKey);
+                    }
+                }
+            }
+        }
+    }
+    /** Iterate all cached instances across all storage types. */
+    *allValues() {
+        const seen = new Set();
+        for (const classMap of this.byId.values()) {
+            for (const keyMap of classMap.values()) {
+                for (const instance of keyMap.values()) {
+                    if (!seen.has(instance)) {
+                        seen.add(instance);
+                        yield instance;
+                    }
+                }
+            }
+        }
+    }
+    clear() {
+        this.byId.clear();
+        this.byName.clear();
+        this.byAlias.clear();
+    }
+}
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, clientCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `clientOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ *
+ * @ignore
+ */
+export class StorageInstanceManager {
+    cache = new StorageCache();
+    openerLocks = new Map();
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param clientOpener      A **lazy** factory that creates the sub-client.
+     *                          Only called on a cache miss.
+     * @param clientCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    async openStorage(cls, { id, name, alias, clientOpener, clientCacheKey, }) {
+        // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
+        if (!id && !name && !alias) {
+            alias = DEFAULT_STORAGE_ALIAS;
+        }
+        // Fast-path cache check (no lock).
+        if (alias !== undefined) {
+            const cached = this.cache.get(cls, { alias, clientCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (id) {
+            const cached = this.cache.get(cls, { id, clientCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (name) {
+            const cached = this.cache.get(cls, { name, clientCacheKey });
+            if (cached)
+                return cached;
+        }
+        const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
+        const lockKey = `${cls.name}:${identifierKey}:${clientCacheKey}`;
+        if (!this.openerLocks.has(lockKey)) {
+            this.openerLocks.set(lockKey, new AsyncQueue());
+        }
+        const queue = this.openerLocks.get(lockKey);
+        await queue.wait();
+        try {
+            // Double-check cache under lock (another caller may have filled it while we waited).
+            if (alias !== undefined) {
+                const cached = this.cache.get(cls, { alias, clientCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (id) {
+                const cached = this.cache.get(cls, { id, clientCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (name) {
+                const cached = this.cache.get(cls, { name, clientCacheKey });
+                if (cached)
+                    return cached;
+            }
+            // Cache miss — create the sub-client and storage instance.
+            const subClient = await clientOpener();
+            const storageInfo = await subClient.getMetadata();
+            const instance = new cls({
+                id: storageInfo.id,
+                name: storageInfo.name,
+                client: subClient,
+            });
+            // Atomic cache writes (no awaits between these).
+            this.cache.set(cls, instance, clientCacheKey, alias);
+            return instance;
+        }
+        finally {
+            queue.shift();
+            // Clean up idle locks so the map doesn't grow unboundedly
+            // (mirrors crawlee-python's WeakValueDictionary behaviour).
+            if (queue.remaining === 0) {
+                this.openerLocks.delete(lockKey);
+            }
+        }
+    }
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance) {
+        this.cache.removeFromCache(instance);
+    }
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache() {
+        for (const instance of this.cache.allValues()) {
+            if ('clearCache' in instance && typeof instance.clearCache === 'function') {
+                instance.clearCache();
+            }
+        }
+        this.cache.clear();
+    }
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageClient.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ */
+export async function resolveStorageIdentifier(identifier, client, storageType) {
+    if (identifier === null || identifier === undefined) {
+        return { alias: DEFAULT_STORAGE_ALIAS };
+    }
+    if (typeof identifier === 'string') {
+        if (client.storageExists && (await client.storageExists(identifier, storageType))) {
+            return { id: identifier };
+        }
+        return { name: identifier };
+    }
+    if (identifier.id) {
+        return { id: identifier.id };
+    }
+    if (identifier.name) {
+        return { name: identifier.name };
+    }
+    // Empty object — treated as default storage.
+    return { alias: DEFAULT_STORAGE_ALIAS };
+}
+//# sourceMappingURL=storage_instance_manager.js.map

package/storages/storage_instance_manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_instance_manager.js","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBnD,wDAAwD;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,YAAY;IACL,IAAI,GAAc,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAC9B,OAAO,GAAc,IAAI,GAAG,EAAE,CAAC;IAExC,GAAG,CACC,GAAmB,EACnB,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,cAAc,GAKc;QAEhC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI;YACtB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;YACnB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACa,EAAE,CAAC;YACrC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5D,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,MAAM,YAAa,GAAuD,EAAE,CAAC;oBAC7E,OAAO,MAAW,CAAC;gBACvB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,8CAA8C;IACtC,QAAQ,CACZ,IAAe,EACf,GAAmB,EACnB,GAAW,EACX,QAAW,EACX,cAAwB;QAExB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,GAAG,CAAqB,GAAmB,EAAE,QAAW,EAAE,cAAwB,EAAE,KAAc;QAC9F,sBAAsB;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAErE,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAC7E,CAAC;QAED,+DAA+D;QAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAED,eAAe,CAAC,QAAkB;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAoC,CAAC;QAElE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;oBACtC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC5B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,CAAC,SAAS;QACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAY,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACnB,MAAM,QAAQ,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,sBAAsB;IACd,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,WAAW,CACb,GAA0B,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,YAAY,EACZ,cAAc,GAIjB;QAED,0FAA0F;QAC1F,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACzB,KAAK,GAAG,qBAAqB,CAAC;QAClC,CAAC;QAED,mCAAmC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;YAC3D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;YAC7D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,aAAa,IAAI,cAAc,EAAE,CAAC;QAEjE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAE7C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC;YACD,qFAAqF;YACrF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC9D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,EAAE,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC3D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC7D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;YAED,2DAA2D;YAC3D,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,MAChB,SACH,CAAC,WAAW,EAAE,CAAC;YAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;gBACrB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,SAAS;aACpB,CAAa,CAAC;YAEf,iDAAiD;YACjD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;YAErD,OAAO,QAAQ,CAAC;QACpB,CAAC;gBAAS,CAAC;YACP,KAAK,CAAC,KAAK,EAAE,CAAC;YAEd,0DAA0D;YAC1D,4DAA4D;YAC5D,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,QAAkB;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,UAAU;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;YAC5C,IAAI,YAAY,IAAI,QAAQ,IAAI,OAAQ,QAAgB,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAChF,QAAgB,CAAC,UAAU,EAAE,CAAC;YACnC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACJ;AAoBD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,UAAyD,EACzD,MAAqB,EACrB,WAAyD;IAEzD,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACjC,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChF,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,6CAA6C;IAC7C,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;AAC5C,CAAC"}

package/storages/utils.d.ts

@@ -1,5 +1,6 @@
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { BaseHttpClient, Dictionary, StorageClient } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
+import type { ProxyConfiguration } from '../proxy_configuration.js';
 /**
  * Options for purging default storage.
  */
@@ -89,5 +90,50 @@ export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
  * @internal
  */
 export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
+/** @internal */
+export interface DualIterableOptions<TItem, TRawPage> {
+    /** Factory that returns an async generator yielding pages. */
+    createPages: () => AsyncGenerator<TRawPage>;
+    /** Extracts individual items from a page (for iteration). */
+    extractItems: (page: TRawPage) => TItem[];
+}
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export declare function createDualIterable<TItem, TRawPage>(options: DualIterableOptions<TItem, TRawPage>): AsyncIterable<TItem> & Promise<TItem[]>;
+/**
+ * Options for the static `open()` method on storage classes ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ */
+export interface StorageOpenOptions {
+    /**
+     * SDK configuration instance, defaults to the static register.
+     */
+    config?: Configuration;
+    /**
+     * Optional storage client that should be used to open storages.
+     */
+    storageClient?: StorageClient;
+    /**
+     * Used to pass the proxy configuration for the `requestsFromUrl` objects.
+     * Takes advantage of the internal address rotation and authentication process.
+     * If undefined, the `requestsFromUrl` requests will be made without proxy.
+     */
+    proxyConfiguration?: ProxyConfiguration;
+    /**
+     * HTTP client to be used to download the list of URLs in `RequestQueue`.
+     */
+    httpClient?: BaseHttpClient;
+}
 export {};
 //# sourceMappingURL=utils.d.ts.map

package/storages/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAE7C,gBAAgB;AAChB,MAAM,WAAW,mBAAmB,CAAC,KAAK,EAAE,QAAQ;IAChD,8DAA8D;IAC9D,WAAW,EAAE,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,6DAA6D;IAC7D,YAAY,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,KAAK,EAAE,CAAC;CAC7C;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAC9C,OAAO,EAAE,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,GAC9C,aAAa,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CA6CzC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B"}

package/storages/utils.js

@@ -1,5 +1,6 @@
 import crypto from 'node:crypto';
 import { Configuration } from '../configuration.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from './key_value_store.js';
 export async function purgeDefaultStorages(configOrOptions, client) {
     const options = configOrOptions instanceof Configuration
@@ -8,11 +9,11 @@ export async function purgeDefaultStorages(configOrOptions, client) {
             config: configOrOptions,
         }
         : (configOrOptions ?? {});
-    const { config = Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
-    ({ client = config.getStorageClient() } = options);
+    const { config = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
+    ({ client = serviceLocator.getStorageClient() } = options);
     const casted = client;
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
-    if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
+    if (!onlyPurgeOnce || (config.purgeOnStart && !casted.__purged)) {
         casted.__purged = true;
         await casted.purge?.();
     }
@@ -27,8 +28,8 @@ export async function purgeDefaultStorages(configOrOptions, client) {
  * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
  */
 export async function useState(name, defaultValue = {}, options) {
-    const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
-        config: options?.config || Configuration.getGlobalConfig(),
+    const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
+        config: options?.config || serviceLocator.getConfiguration(),
     });
     return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
 }
@@ -69,4 +70,55 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
  * @internal
  */
 export const MAX_QUERIES_FOR_CONSISTENCY = 6;
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export function createDualIterable(options) {
+    const { createPages, extractItems } = options;
+    let cached = null;
+    function getOrCreate() {
+        if (!cached) {
+            cached = (async () => {
+                const items = [];
+                for await (const page of createPages()) {
+                    items.push(...extractItems(page));
+                }
+                return items;
+            })();
+        }
+        return cached;
+    }
+    async function* iterateAll() {
+        for await (const page of createPages()) {
+            yield* extractItems(page);
+        }
+    }
+    const result = {
+        [Symbol.asyncIterator]() {
+            return iterateAll();
+        },
+        then(onfulfilled, onrejected) {
+            return getOrCreate().then(onfulfilled, onrejected);
+        },
+        catch(onrejected) {
+            return getOrCreate().catch(onrejected);
+        },
+        finally(onfinally) {
+            return getOrCreate().finally(onfinally);
+        },
+        [Symbol.toStringTag]: 'DualIterable',
+    };
+    return result;
+}
 //# sourceMappingURL=utils.js.map

package/storages/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9G,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE;KAC/D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAU7C;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAC9B,OAA6C;IAE7C,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAC9C,IAAI,MAAM,GAA4B,IAAI,CAAC;IAE3C,SAAS,WAAW;QAChB,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,GAAG,CAAC,KAAK,IAAI,EAAE;gBACjB,MAAM,KAAK,GAAY,EAAE,CAAC;gBAC1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,WAAW,EAAE,EAAE,CAAC;oBACrC,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;gBACtC,CAAC;gBACD,OAAO,KAAK,CAAC;YACjB,CAAC,CAAC,EAAE,CAAC;QACT,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,SAAS,CAAC,CAAC,UAAU;QACtB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,WAAW,EAAE,EAAE,CAAC;YACrC,KAAK,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG;QACX,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,UAAU,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,CACA,WAA2E,EAC3E,UAAuE;YAEvE,OAAO,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACvD,CAAC;QACD,KAAK,CACD,UAAqE;YAErE,OAAO,WAAW,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,CAAC,SAA+B;YACnC,OAAO,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,cAAc;KACI,CAAC;IAE7C,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/typedefs.d.ts

@@ -6,5 +6,5 @@ export type Awaitable<T> = T | PromiseLike<T>;
 export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
 /** @ignore */
 export declare function keys<T extends {}>(obj: T): (keyof T)[];
-export declare type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH';
+export type { AllowedHttpMethods } from '@crawlee/types';
 //# sourceMappingURL=typedefs.d.ts.map

package/typedefs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,MAAM,CAAC,OAAO,MAAM,kBAAkB,GAChC,KAAK,GACL,MAAM,GACN,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,OAAO,CAAC"}
+{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}

package/validators.d.ts

@@ -17,5 +17,9 @@ export declare const validators: {
         validator: boolean;
         message: (label: string) => string;
     };
+    sessionPool: (value: Dictionary) => {
+        validator: boolean;
+        message: (label: string) => string;
+    };
 };
 //# sourceMappingURL=validators.d.ts.map

package/validators.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;CAE9B,CAAC"}
+{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;CAG9B,CAAC"}

package/validators.js

@@ -18,5 +18,9 @@ export const validators = {
         validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'addRequest')),
         message: (label) => `Expected argument '${label}' to be a RequestQueue, got something else.`,
     }),
+    sessionPool: (value) => ({
+        validator: ow.isValid(value, ow.object.hasKeys('getSession')),
+        message: (label) => `Expected argument '${label}' to implement the ISessionPool interface (missing 'getSession'), got something else.`,
+    }),
 };
 //# sourceMappingURL=validators.js.map

package/validators.js.map

@@ -1 +1 @@
-{"version":3,"file":"validators.js","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gBAAgB;AAChB,MAAM,CAAC,MAAM,UAAU,GAAG;IACtB,6DAA6D;IAC7D,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,+CAA+C;KACzG,CAAC;IACF,kBAAkB,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACxC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,mDAAmD;KAC7G,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,4CAA4C;KACtG,CAAC;IACF,YAAY,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACjF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,6CAA6C;KACvG,CAAC;CACL,CAAC"}
+{"version":3,"file":"validators.js","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gBAAgB;AAChB,MAAM,CAAC,MAAM,UAAU,GAAG;IACtB,6DAA6D;IAC7D,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,+CAA+C;KACzG,CAAC;IACF,kBAAkB,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACxC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,mDAAmD;KAC7G,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,4CAA4C;KACtG,CAAC;IACF,YAAY,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACjF,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,sBAAsB,KAAK,6CAA6C;KACvG,CAAC;IACF,WAAW,EAAE,CAAC,KAAiB,EAAE,EAAE,CAAC,CAAC;QACjC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC7D,OAAO,EAAE,CAAC,KAAa,EAAE,EAAE,CACvB,sBAAsB,KAAK,uFAAuF;KACzH,CAAC;CACL,CAAC"}

package/crawlers/crawler_extension.d.ts

@@ -1,12 +0,0 @@
-import { type Log } from '../log.js';
-/**
- * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
- * @category Crawlers
- * @ignore
- */
-export declare abstract class CrawlerExtension {
-    name: string;
-    log: Log;
-    getCrawlerOptions(): Record<string, unknown>;
-}
-//# sourceMappingURL=crawler_extension.d.ts.map

package/crawlers/crawler_extension.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_extension.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,GAAG,EAAqB,MAAM,WAAW,CAAC;AAExD;;;;GAIG;AACH,8BAAsB,gBAAgB;IAClC,IAAI,SAAyB;IAC7B,GAAG,EAAE,GAAG,CAA2C;IAEnD,iBAAiB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAG/C"}

package/crawlers/crawler_extension.js

@@ -1,14 +0,0 @@
-import { log as defaultLog } from '../log.js';
-/**
- * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
- * @category Crawlers
- * @ignore
- */
-export class CrawlerExtension {
-    name = this.constructor.name;
-    log = defaultLog.child({ prefix: this.name });
-    getCrawlerOptions() {
-        throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`);
-    }
-}
-//# sourceMappingURL=crawler_extension.js.map

package/crawlers/crawler_extension.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_extension.js","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAExD;;;;GAIG;AACH,MAAM,OAAgB,gBAAgB;IAClC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;IAC7B,GAAG,GAAQ,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAEnD,iBAAiB;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,kDAAkD,CAAC,CAAC;IACpF,CAAC;CACJ"}

package/http_clients/base-http-client.d.ts

@@ -1,134 +0,0 @@
-import type { Readable } from 'node:stream';
-import { type SearchParams } from '@crawlee/utils';
-import type { FormDataLike } from './form-data-like.js';
-type Timeout = {
-    lookup: number;
-    connect: number;
-    secureConnect: number;
-    socket: number;
-    send: number;
-    response: number;
-} | {
-    request: number;
-};
-type Method = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'HEAD' | 'DELETE' | 'OPTIONS' | 'TRACE' | 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete' | 'options' | 'trace';
-/**
- * Maps permitted values of the `responseType` option on {@link HttpRequest} to the types that they produce.
- */
-export interface ResponseTypes {
-    'json': unknown;
-    'text': string;
-    'buffer': Buffer;
-}
-interface Progress {
-    percent: number;
-    transferred: number;
-    total?: number;
-}
-interface ToughCookieJar {
-    getCookieString: ((currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookies: string) => void) => string) & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string);
-    setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookie: unknown) => void) => void) & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void);
-}
-interface PromiseCookieJar {
-    getCookieString: (url: string) => Promise<string>;
-    setCookie: (rawCookie: string, url: string) => Promise<unknown>;
-}
-type SimpleHeaders = Record<string, string | string[] | undefined>;
-/**
- * HTTP Request as accepted by {@link BaseHttpClient} methods.
- */
-export interface HttpRequest<TResponseType extends keyof ResponseTypes = 'text'> {
-    [k: string]: unknown;
-    url: string | URL;
-    method?: Method;
-    headers?: SimpleHeaders;
-    body?: string | Buffer | Readable | Generator | AsyncGenerator | FormDataLike;
-    signal?: AbortSignal;
-    timeout?: Partial<Timeout>;
-    cookieJar?: ToughCookieJar | PromiseCookieJar;
-    followRedirect?: boolean | ((response: any) => boolean);
-    maxRedirects?: number;
-    encoding?: BufferEncoding;
-    responseType?: TResponseType;
-    throwHttpErrors?: boolean;
-    proxyUrl?: string;
-    headerGeneratorOptions?: Record<string, unknown>;
-    useHeaderGenerator?: boolean;
-    headerGenerator?: {
-        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
-    };
-    insecureHTTPParser?: boolean;
-    sessionToken?: object;
-}
-/**
- * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
- */
-export interface HttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'> extends HttpRequest<TResponseType> {
-    /** Search (query string) parameters to be appended to the request URL */
-    searchParams?: SearchParams;
-    /** A form to be sent in the HTTP request body (URL encoding will be used) */
-    form?: Record<string, string>;
-    /** Artbitrary object to be JSON-serialized and sent as the HTTP request body */
-    json?: unknown;
-    /** Basic HTTP Auth username */
-    username?: string;
-    /** Basic HTTP Auth password */
-    password?: string;
-}
-/**
- * HTTP response data, without a body, as returned by {@link BaseHttpClient} methods.
- */
-export interface BaseHttpResponseData {
-    redirectUrls: URL[];
-    url: string;
-    ip?: string;
-    statusCode: number;
-    statusMessage?: string;
-    headers: SimpleHeaders;
-    trailers: SimpleHeaders;
-    complete: boolean;
-}
-interface HttpResponseWithoutBody<TResponseType extends keyof ResponseTypes = keyof ResponseTypes> extends BaseHttpResponseData {
-    request: HttpRequest<TResponseType>;
-}
-/**
- * HTTP response data as returned by the {@link BaseHttpClient.sendRequest} method.
- */
-export interface HttpResponse<TResponseType extends keyof ResponseTypes = keyof ResponseTypes> extends HttpResponseWithoutBody<TResponseType> {
-    [k: string]: any;
-    body: ResponseTypes[TResponseType];
-}
-/**
- * HTTP response data as returned by the {@link BaseHttpClient.stream} method.
- */
-export interface StreamingHttpResponse extends HttpResponseWithoutBody {
-    stream: Readable;
-    readonly downloadProgress: Progress;
-    readonly uploadProgress: Progress;
-}
-/**
- * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
- */
-export type RedirectHandler = (redirectResponse: BaseHttpResponseData, updatedRequest: {
-    url?: string | URL;
-    headers: SimpleHeaders;
-}) => void;
-/**
- * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
- */
-export interface BaseHttpClient {
-    /**
-     * Perform an HTTP Request and return the complete response.
-     */
-    sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(request: HttpRequest<TResponseType>): Promise<HttpResponse<TResponseType>>;
-    /**
-     * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.
-     */
-    stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
-}
-/**
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
- */
-export declare function processHttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'>({ searchParams, form, json, username, password, ...request }: HttpRequestOptions<TResponseType>): HttpRequest<TResponseType>;
-export {};
-//# sourceMappingURL=base-http-client.d.ts.map

package/http_clients/base-http-client.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,KAAK,OAAO,GACN;IACI,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CACpB,GACD;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1B,KAAK,MAAM,GACL,KAAK,GACL,MAAM,GACN,KAAK,GACL,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,KAAK,GACL,MAAM,GACN,KAAK,GACL,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,QAAQ;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,UAAU,cAAc;IACpB,eAAe,EAAE,CAAC,CACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,KACvD,MAAM,CAAC,GACR,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,YAAY,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7F,SAAS,EAAE,CAAC,CACR,cAAc,EAAE,OAAO,EACvB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KACvD,IAAI,CAAC,GACN,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC5G;AAED,UAAU,gBAAgB;IACtB,eAAe,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACnE;AAED,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM;IAC3E,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAErB,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,cAAc,GAAG,YAAY,CAAC;IAE9E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3B,SAAS,CAAC,EAAE,cAAc,GAAG,gBAAgB,CAAC;IAC9C,cAAc,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,eAAe,CAAC,EAAE,OAAO,CAAC;IAG1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE;QACd,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5E,CAAC;IACF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,CAClF,SAAQ,WAAW,CAAC,aAAa,CAAC;IAClC,yEAAyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,gFAAgF;IAChF,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,YAAY,EAAE,GAAG,EAAE,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IAEZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,OAAO,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,aAAa,CAAC;IAExB,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,UAAU,uBAAuB,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,aAAa,CAC7F,SAAQ,oBAAoB;IAC5B,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,aAAa,CACzF,SAAQ,uBAAuB,CAAC,aAAa,CAAC;IAC9C,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAEjB,IAAI,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,uBAAuB;IAClE,MAAM,EAAE,QAAQ,CAAC;IACjB,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,CAC1B,gBAAgB,EAAE,oBAAoB,EACtC,cAAc,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAAC,OAAO,EAAE,aAAa,CAAA;CAAE,KAC7D,IAAI,CAAC;AAEV;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,EAC1D,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC;IAExC;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC9F;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,EAAE,EAC1F,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,CAAC,aAAa,CAAC,GAAG,WAAW,CAAC,aAAa,CAAC,CAoChE"}