@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.71

package/storages/storage_manager.js

@@ -1,105 +0,0 @@
-import { AsyncQueue } from '@sapphire/async-queue';
-import { Configuration } from '../configuration.js';
-const DEFAULT_ID_CONFIG_KEYS = {
-    Dataset: 'defaultDatasetId',
-    KeyValueStore: 'defaultKeyValueStoreId',
-    RequestQueue: 'defaultRequestQueueId',
-};
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export class StorageManager {
-    config;
-    name;
-    StorageConstructor;
-    cache = new Map();
-    storageOpenQueue = new AsyncQueue();
-    constructor(StorageConstructor, config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.StorageConstructor = StorageConstructor;
-        this.name = this.StorageConstructor.name;
-    }
-    static async openStorage(storageClass, idOrName, client, config = Configuration.getGlobalConfig()) {
-        return this.getManager(storageClass, config).openStorage(idOrName, client);
-    }
-    static getManager(storageClass, config = Configuration.getGlobalConfig()) {
-        if (!config.storageManagers.has(storageClass)) {
-            const manager = new StorageManager(storageClass, config);
-            config.storageManagers.set(storageClass, manager);
-        }
-        return config.storageManagers.get(storageClass);
-    }
-    /** @internal */
-    static clearCache(config = Configuration.getGlobalConfig()) {
-        config.storageManagers.forEach((manager) => {
-            if (manager.name === 'KeyValueStore') {
-                manager.cache.forEach((item) => {
-                    item.clearCache?.();
-                });
-            }
-        });
-        config.storageManagers.clear();
-    }
-    async openStorage(idOrName, client) {
-        await this.storageOpenQueue.wait();
-        if (!idOrName) {
-            const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
-            idOrName = this.config.get(defaultIdConfigKey);
-        }
-        const cacheKey = idOrName;
-        let storage = this.cache.get(cacheKey);
-        if (!storage) {
-            client ??= this.config.getStorageClient();
-            const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);
-            storage = new this.StorageConstructor({
-                id: storageObject.id,
-                name: storageObject.name,
-                storageObject,
-                client,
-            });
-            this._addStorageToCache(storage);
-        }
-        this.storageOpenQueue.shift();
-        return storage;
-    }
-    closeStorage(storage) {
-        const idKey = storage.id;
-        this.cache.delete(idKey);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.delete(nameKey);
-        }
-    }
-    /**
-     * Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
-     */
-    async _getOrCreateStorage(storageIdOrName, storageConstructorName, apiClient) {
-        const { createStorageClient, createStorageCollectionClient } = this._getStorageClientFactories(apiClient, storageConstructorName);
-        const storageClient = createStorageClient(storageIdOrName);
-        const existingStorage = await storageClient.get();
-        if (existingStorage)
-            return existingStorage;
-        const storageCollectionClient = createStorageCollectionClient();
-        return storageCollectionClient.getOrCreate(storageIdOrName);
-    }
-    _getStorageClientFactories(client, storageConstructorName) {
-        // Dataset => dataset
-        const clientName = (storageConstructorName[0].toLowerCase() + storageConstructorName.slice(1));
-        // dataset => datasets
-        const collectionClientName = `${clientName}s`;
-        return {
-            createStorageClient: client[clientName].bind(client),
-            createStorageCollectionClient: client[collectionClientName].bind(client),
-        };
-    }
-    _addStorageToCache(storage) {
-        const idKey = storage.id;
-        this.cache.set(idKey, storage);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.set(nameKey, storage);
-        }
-    }
-}
-//# sourceMappingURL=storage_manager.js.map

package/storages/storage_manager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAM,OAAO,cAAc;IAQF;IAPJ,IAAI,CAA+C;IACnD,kBAAkB,CAAoC;IACtD,KAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7B,gBAAgB,GAAG,IAAI,UAAU,EAAE,CAAC;IAErD,YACI,kBAAkC,EACjB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEzD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,WAAW,CACpB,YAA4B,EAC5B,QAAiB,EACjB,MAAsB,EACtB,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QAExC,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,CAAC,UAAU,CACb,YAA4B,EAC5B,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QAExC,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAsB,CAAC;IACzE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QACtD,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACvC,IAAI,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBACnC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC1B,IAAmB,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxC,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAwB,EAAE,MAAsB;QAC9D,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAW,CAAC;QAC7D,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QAC1B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClF,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClC,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa;gBACb,MAAM;aACT,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,eAAuB,EACvB,sBAA8B,EAC9B,SAAwB;QAExB,MAAM,EAAE,mBAAmB,EAAE,6BAA6B,EAAE,GAAG,IAAI,CAAC,0BAA0B,CAC1F,SAAS,EACT,sBAAsB,CACzB,CAAC;QAEF,MAAM,aAAa,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAE5C,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAES,0BAA0B,CAAC,MAAqB,EAAE,sBAA8B;QACtF,qBAAqB;QACrB,MAAM,UAAU,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAgB,CAAC;QAC9G,sBAAsB;QACtB,MAAM,oBAAoB,GAAG,GAAG,UAAU,GAA4B,CAAC;QAEvE,OAAO;YACH,mBAAmB,EAAE,MAAM,CAAC,UAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACrD,6BAA6B,EAAE,MAAM,CAAC,oBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC5E,CAAC;IACN,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;CACJ"}