@crawlee/core 4.0.0-beta.54 → 4.0.0-beta.55

package/storages/storage_manager.js

@@ -1,111 +0,0 @@
-import { AsyncQueue } from '@sapphire/async-queue';
-import { serviceLocator } from '../service_locator.js';
-const DEFAULT_ID_CONFIG_KEYS = {
-    Dataset: 'defaultDatasetId',
-    KeyValueStore: 'defaultKeyValueStoreId',
-    RequestQueue: 'defaultRequestQueueId',
-};
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export class StorageManager {
-    config;
-    name;
-    StorageConstructor;
-    cache = new Map();
-    storageOpenQueue = new AsyncQueue();
-    constructor(StorageConstructor, config) {
-        this.config = config;
-        this.StorageConstructor = StorageConstructor;
-        this.name = this.StorageConstructor.name;
-    }
-    static async openStorage(storageClass, identifier, client) {
-        return this.getManager(storageClass).openStorage(identifier, client);
-    }
-    static getManager(storageClass) {
-        let storageManager = serviceLocator.getStorageManager(storageClass);
-        if (storageManager === undefined) {
-            storageManager = new StorageManager(storageClass, serviceLocator.getConfiguration());
-            serviceLocator.setStorageManager(storageClass, storageManager);
-        }
-        return serviceLocator.getStorageManager(storageClass);
-    }
-    async openStorage(identifier, client) {
-        await this.storageOpenQueue.wait();
-        const resolvedIdentifier = await this._resolveIdentifier(identifier, client);
-        const cacheKey = resolvedIdentifier.id ?? resolvedIdentifier.name;
-        let storage = this.cache.get(cacheKey);
-        if (!storage) {
-            client ??= serviceLocator.getStorageClient();
-            const subClient = await this._createSubClient(resolvedIdentifier, this.name, client);
-            const storageInfo = await subClient.getMetadata();
-            storage = new this.StorageConstructor({
-                id: storageInfo.id,
-                name: storageInfo.name,
-                client: subClient,
-            }, this.config);
-            this._addStorageToCache(storage);
-        }
-        this.storageOpenQueue.shift();
-        return storage;
-    }
-    /**
-     * Resolves the user-provided identifier to an unambiguous `StorageIdentifier`.
-     *
-     * - `null`/`undefined` → uses the default storage ID from config
-     * - `StorageIdentifier` object → passed through (with default ID fallback if empty)
-     * - `string` → tries to find an existing storage with that ID first;
-     *   if none exists, treats the string as a name
-     */
-    async _resolveIdentifier(identifier, client) {
-        if (typeof identifier === 'string') {
-            client ??= serviceLocator.getStorageClient();
-            if (client.storageExists && (await client.storageExists(identifier, this.name))) {
-                return { id: identifier };
-            }
-            return { name: identifier };
-        }
-        if (identifier?.id) {
-            return { id: identifier.id };
-        }
-        if (identifier?.name) {
-            return { name: identifier.name };
-        }
-        const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
-        const defaultId = this.config[defaultIdConfigKey];
-        return { id: defaultId };
-    }
-    closeStorage(storage) {
-        const idKey = storage.id;
-        this.cache.delete(idKey);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.delete(nameKey);
-        }
-    }
-    /**
-     * Creates a sub-client for the given storage type using the new factory-based StorageClient interface.
-     */
-    async _createSubClient(identifier, storageConstructorName, apiClient) {
-        switch (storageConstructorName) {
-            case 'Dataset':
-                return apiClient.createDatasetClient(identifier);
-            case 'KeyValueStore':
-                return apiClient.createKeyValueStoreClient(identifier);
-            case 'RequestQueue':
-                return apiClient.createRequestQueueClient(identifier);
-            default:
-                throw new Error(`Unknown storage type: ${storageConstructorName}`);
-        }
-    }
-    _addStorageToCache(storage) {
-        const idKey = storage.id;
-        this.cache.set(idKey, storage);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.set(nameKey, storage);
-        }
-    }
-}
-//# sourceMappingURL=storage_manager.js.map

package/storages/storage_manager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAInD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAKvD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAM,OAAO,cAAc;IAQF;IAPJ,IAAI,CAA+C;IACnD,kBAAkB,CAAoC;IACtD,KAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7B,gBAAgB,GAAG,IAAI,UAAU,EAAE,CAAC;IAErD,YACI,kBAAkC,EACjB,MAAqB;QAArB,WAAM,GAAN,MAAM,CAAe;QAEtC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,WAAW,CACpB,YAA4B,EAC5B,UAAuC,EACvC,MAAsB;QAEtB,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,UAAU,CAAqB,YAA4B;QAC9D,IAAI,cAAc,GAAG,cAAc,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YAC/B,cAAc,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,CAAC;YACrF,cAAc,CAAC,iBAAiB,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,cAAc,CAAC,iBAAiB,CAAC,YAAY,CAAsB,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,UAA8C,EAAE,MAAsB;QACpF,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAEnC,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAE7E,MAAM,QAAQ,GAAG,kBAAkB,CAAC,EAAE,IAAI,kBAAkB,CAAC,IAAK,CAAC;QACnE,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,KAAK,cAAc,CAAC,gBAAgB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,MAChB,SACH,CAAC,WAAW,EAAE,CAAC;YAChB,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CACjC;gBACI,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,SAAS;aACpB,EACD,IAAI,CAAC,MAAM,CACd,CAAC;YACF,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,kBAAkB,CAC5B,UAA8C,EAC9C,MAAsB;QAEtB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,KAAK,cAAc,CAAC,gBAAgB,EAAE,CAAC;YAE7C,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC9E,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;YAC9B,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,UAAU,EAAE,EAAE,EAAE,CAAC;YACjB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;QACjC,CAAC;QAED,IAAI,UAAU,EAAE,IAAI,EAAE,CAAC;YACnB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAW,CAAC;QAE5D,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;IAC7B,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,gBAAgB,CAC5B,UAA6B,EAC7B,sBAA8B,EAC9B,SAAwB;QAExB,QAAQ,sBAAsB,EAAE,CAAC;YAC7B,KAAK,SAAS;gBACV,OAAO,SAAS,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YACrD,KAAK,eAAe;gBAChB,OAAO,SAAS,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;YAC3D,KAAK,cAAc;gBACf,OAAO,SAAS,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;YAC1D;gBACI,MAAM,IAAI,KAAK,CAAC,yBAAyB,sBAAsB,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;CACJ"}