@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/storages/storage_manager.js

@@ -1,105 +0,0 @@
-import { AsyncQueue } from '@sapphire/async-queue';
-import { Configuration } from '../configuration.js';
-const DEFAULT_ID_CONFIG_KEYS = {
-    Dataset: 'defaultDatasetId',
-    KeyValueStore: 'defaultKeyValueStoreId',
-    RequestQueue: 'defaultRequestQueueId',
-};
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export class StorageManager {
-    config;
-    name;
-    StorageConstructor;
-    cache = new Map();
-    storageOpenQueue = new AsyncQueue();
-    constructor(StorageConstructor, config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.StorageConstructor = StorageConstructor;
-        this.name = this.StorageConstructor.name;
-    }
-    static async openStorage(storageClass, idOrName, client, config = Configuration.getGlobalConfig()) {
-        return this.getManager(storageClass, config).openStorage(idOrName, client);
-    }
-    static getManager(storageClass, config = Configuration.getGlobalConfig()) {
-        if (!config.storageManagers.has(storageClass)) {
-            const manager = new StorageManager(storageClass, config);
-            config.storageManagers.set(storageClass, manager);
-        }
-        return config.storageManagers.get(storageClass);
-    }
-    /** @internal */
-    static clearCache(config = Configuration.getGlobalConfig()) {
-        config.storageManagers.forEach((manager) => {
-            if (manager.name === 'KeyValueStore') {
-                manager.cache.forEach((item) => {
-                    item.clearCache?.();
-                });
-            }
-        });
-        config.storageManagers.clear();
-    }
-    async openStorage(idOrName, client) {
-        await this.storageOpenQueue.wait();
-        if (!idOrName) {
-            const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
-            idOrName = this.config.get(defaultIdConfigKey);
-        }
-        const cacheKey = idOrName;
-        let storage = this.cache.get(cacheKey);
-        if (!storage) {
-            client ??= this.config.getStorageClient();
-            const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);
-            storage = new this.StorageConstructor({
-                id: storageObject.id,
-                name: storageObject.name,
-                storageObject,
-                client,
-            });
-            this._addStorageToCache(storage);
-        }
-        this.storageOpenQueue.shift();
-        return storage;
-    }
-    closeStorage(storage) {
-        const idKey = storage.id;
-        this.cache.delete(idKey);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.delete(nameKey);
-        }
-    }
-    /**
-     * Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
-     */
-    async _getOrCreateStorage(storageIdOrName, storageConstructorName, apiClient) {
-        const { createStorageClient, createStorageCollectionClient } = this._getStorageClientFactories(apiClient, storageConstructorName);
-        const storageClient = createStorageClient(storageIdOrName);
-        const existingStorage = await storageClient.get();
-        if (existingStorage)
-            return existingStorage;
-        const storageCollectionClient = createStorageCollectionClient();
-        return storageCollectionClient.getOrCreate(storageIdOrName);
-    }
-    _getStorageClientFactories(client, storageConstructorName) {
-        // Dataset => dataset
-        const clientName = (storageConstructorName[0].toLowerCase() + storageConstructorName.slice(1));
-        // dataset => datasets
-        const collectionClientName = `${clientName}s`;
-        return {
-            createStorageClient: client[clientName].bind(client),
-            createStorageCollectionClient: client[collectionClientName].bind(client),
-        };
-    }
-    _addStorageToCache(storage) {
-        const idKey = storage.id;
-        this.cache.set(idKey, storage);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.set(nameKey, storage);
-        }
-    }
-}
-//# sourceMappingURL=storage_manager.js.map

package/storages/storage_manager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAM,OAAO,cAAc;IAQF;IAPJ,IAAI,CAA+C;IACnD,kBAAkB,CAAoC;IACtD,KAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7B,gBAAgB,GAAG,IAAI,UAAU,EAAE,CAAC;IAErD,YACI,kBAAkC,EACjB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEzD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,WAAW,CACpB,YAA4B,EAC5B,QAAiB,EACjB,MAAsB,EACtB,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QAExC,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,CAAC,UAAU,CACb,YAA4B,EAC5B,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QAExC,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAsB,CAAC;IACzE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QACtD,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACvC,IAAI,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBACnC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC1B,IAAmB,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxC,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAwB,EAAE,MAAsB;QAC9D,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAW,CAAC;QAC7D,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QAC1B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClF,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClC,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,aAAa;gBACb,MAAM;aACT,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,eAAuB,EACvB,sBAA8B,EAC9B,SAAwB;QAExB,MAAM,EAAE,mBAAmB,EAAE,6BAA6B,EAAE,GAAG,IAAI,CAAC,0BAA0B,CAC1F,SAAS,EACT,sBAAsB,CACzB,CAAC;QAEF,MAAM,aAAa,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAE5C,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAES,0BAA0B,CAAC,MAAqB,EAAE,sBAA8B;QACtF,qBAAqB;QACrB,MAAM,UAAU,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAgB,CAAC;QAC9G,sBAAsB;QACtB,MAAM,oBAAoB,GAAG,GAAG,UAAU,GAA4B,CAAC;QAEvE,OAAO;YACH,mBAAmB,EAAE,MAAM,CAAC,UAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACrD,6BAA6B,EAAE,MAAM,CAAC,oBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC5E,CAAC;IACN,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;CACJ"}

package/storages/utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC"}

package/storages/utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}