@crawlee/core 4.0.0-beta.54 → 4.0.0-beta.56

package/storages/storage_manager.d.ts

@@ -1,63 +0,0 @@
-import type { BaseHttpClient, DatasetClient, KeyValueStoreClient, RequestQueueClient, StorageClient, StorageIdentifier } from '@crawlee/types';
-import type { Configuration } from '../configuration.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import type { Constructor } from '../typedefs.js';
-export type { StorageIdentifier } from '@crawlee/types';
-export interface IStorage {
-    id: string;
-    name?: string;
-}
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export declare class StorageManager<T extends IStorage = IStorage> {
-    private readonly config;
-    private readonly name;
-    private readonly StorageConstructor;
-    private readonly cache;
-    private readonly storageOpenQueue;
-    constructor(StorageConstructor: Constructor<T>, config: Configuration);
-    static openStorage<T extends IStorage>(storageClass: Constructor<T>, identifier?: string | StorageIdentifier, client?: StorageClient): Promise<T>;
-    static getManager<T extends IStorage>(storageClass: Constructor<T>): StorageManager<T>;
-    openStorage(identifier?: string | StorageIdentifier | null, client?: StorageClient): Promise<T>;
-    /**
-     * Resolves the user-provided identifier to an unambiguous `StorageIdentifier`.
-     *
-     * - `null`/`undefined` → uses the default storage ID from config
-     * - `StorageIdentifier` object → passed through (with default ID fallback if empty)
-     * - `string` → tries to find an existing storage with that ID first;
-     *   if none exists, treats the string as a name
-     */
-    private _resolveIdentifier;
-    closeStorage(storage: {
-        id: string;
-        name?: string;
-    }): void;
-    /**
-     * Creates a sub-client for the given storage type using the new factory-based StorageClient interface.
-     */
-    protected _createSubClient(identifier: StorageIdentifier, storageConstructorName: string, apiClient: StorageClient): Promise<DatasetClient | KeyValueStoreClient | RequestQueueClient>;
-    protected _addStorageToCache(storage: T): void;
-}
-export interface StorageManagerOptions {
-    /**
-     * SDK configuration instance, defaults to the static register.
-     */
-    config?: Configuration;
-    /**
-     * Optional storage client that should be used to open storages.
-     */
-    storageClient?: StorageClient;
-    /**
-     * Used to pass the proxy configuration for the `requestsFromUrl` objects.
-     * Takes advantage of the internal address rotation and authentication process.
-     * If undefined, the `requestsFromUrl` requests will be made without proxy.
-     */
-    proxyConfiguration?: ProxyConfiguration;
-    /**
-     * HTTP client to be used to download the list of URLs in `RequestQueue`.
-     */
-    httpClient?: BaseHttpClient;
-}
-//# sourceMappingURL=storage_manager.d.ts.map

package/storages/storage_manager.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACpB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAQxD,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;gBAGjD,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,EAAE,aAAa;WAM7B,WAAW,CAAC,CAAC,SAAS,QAAQ,EACvC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,EACvC,MAAM,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,CAAC,CAAC;IAIb,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,QAAQ,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC;IAUhF,WAAW,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IA8BrG;;;;;;;OAOG;YACW,kBAAkB;IA4BhC,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,gBAAgB,CAC5B,UAAU,EAAE,iBAAiB,EAC7B,sBAAsB,EAAE,MAAM,EAC9B,SAAS,EAAE,aAAa,GACzB,OAAO,CAAC,aAAa,GAAG,mBAAmB,GAAG,kBAAkB,CAAC;IAapE,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAED,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B"}

package/storages/storage_manager.js

@@ -1,111 +0,0 @@
-import { AsyncQueue } from '@sapphire/async-queue';
-import { serviceLocator } from '../service_locator.js';
-const DEFAULT_ID_CONFIG_KEYS = {
-    Dataset: 'defaultDatasetId',
-    KeyValueStore: 'defaultKeyValueStoreId',
-    RequestQueue: 'defaultRequestQueueId',
-};
-/**
- * StorageManager takes care of opening remote or local storages.
- * @ignore
- */
-export class StorageManager {
-    config;
-    name;
-    StorageConstructor;
-    cache = new Map();
-    storageOpenQueue = new AsyncQueue();
-    constructor(StorageConstructor, config) {
-        this.config = config;
-        this.StorageConstructor = StorageConstructor;
-        this.name = this.StorageConstructor.name;
-    }
-    static async openStorage(storageClass, identifier, client) {
-        return this.getManager(storageClass).openStorage(identifier, client);
-    }
-    static getManager(storageClass) {
-        let storageManager = serviceLocator.getStorageManager(storageClass);
-        if (storageManager === undefined) {
-            storageManager = new StorageManager(storageClass, serviceLocator.getConfiguration());
-            serviceLocator.setStorageManager(storageClass, storageManager);
-        }
-        return serviceLocator.getStorageManager(storageClass);
-    }
-    async openStorage(identifier, client) {
-        await this.storageOpenQueue.wait();
-        const resolvedIdentifier = await this._resolveIdentifier(identifier, client);
-        const cacheKey = resolvedIdentifier.id ?? resolvedIdentifier.name;
-        let storage = this.cache.get(cacheKey);
-        if (!storage) {
-            client ??= serviceLocator.getStorageClient();
-            const subClient = await this._createSubClient(resolvedIdentifier, this.name, client);
-            const storageInfo = await subClient.getMetadata();
-            storage = new this.StorageConstructor({
-                id: storageInfo.id,
-                name: storageInfo.name,
-                client: subClient,
-            }, this.config);
-            this._addStorageToCache(storage);
-        }
-        this.storageOpenQueue.shift();
-        return storage;
-    }
-    /**
-     * Resolves the user-provided identifier to an unambiguous `StorageIdentifier`.
-     *
-     * - `null`/`undefined` → uses the default storage ID from config
-     * - `StorageIdentifier` object → passed through (with default ID fallback if empty)
-     * - `string` → tries to find an existing storage with that ID first;
-     *   if none exists, treats the string as a name
-     */
-    async _resolveIdentifier(identifier, client) {
-        if (typeof identifier === 'string') {
-            client ??= serviceLocator.getStorageClient();
-            if (client.storageExists && (await client.storageExists(identifier, this.name))) {
-                return { id: identifier };
-            }
-            return { name: identifier };
-        }
-        if (identifier?.id) {
-            return { id: identifier.id };
-        }
-        if (identifier?.name) {
-            return { name: identifier.name };
-        }
-        const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
-        const defaultId = this.config[defaultIdConfigKey];
-        return { id: defaultId };
-    }
-    closeStorage(storage) {
-        const idKey = storage.id;
-        this.cache.delete(idKey);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.delete(nameKey);
-        }
-    }
-    /**
-     * Creates a sub-client for the given storage type using the new factory-based StorageClient interface.
-     */
-    async _createSubClient(identifier, storageConstructorName, apiClient) {
-        switch (storageConstructorName) {
-            case 'Dataset':
-                return apiClient.createDatasetClient(identifier);
-            case 'KeyValueStore':
-                return apiClient.createKeyValueStoreClient(identifier);
-            case 'RequestQueue':
-                return apiClient.createRequestQueueClient(identifier);
-            default:
-                throw new Error(`Unknown storage type: ${storageConstructorName}`);
-        }
-    }
-    _addStorageToCache(storage) {
-        const idKey = storage.id;
-        this.cache.set(idKey, storage);
-        if (storage.name) {
-            const nameKey = storage.name;
-            this.cache.set(nameKey, storage);
-        }
-    }
-}
-//# sourceMappingURL=storage_manager.js.map

package/storages/storage_manager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAInD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAKvD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAM,OAAO,cAAc;IAQF;IAPJ,IAAI,CAA+C;IACnD,kBAAkB,CAAoC;IACtD,KAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7B,gBAAgB,GAAG,IAAI,UAAU,EAAE,CAAC;IAErD,YACI,kBAAkC,EACjB,MAAqB;QAArB,WAAM,GAAN,MAAM,CAAe;QAEtC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,WAAW,CACpB,YAA4B,EAC5B,UAAuC,EACvC,MAAsB;QAEtB,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,UAAU,CAAqB,YAA4B;QAC9D,IAAI,cAAc,GAAG,cAAc,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YAC/B,cAAc,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,CAAC;YACrF,cAAc,CAAC,iBAAiB,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,cAAc,CAAC,iBAAiB,CAAC,YAAY,CAAsB,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,UAA8C,EAAE,MAAsB;QACpF,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAEnC,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAE7E,MAAM,QAAQ,GAAG,kBAAkB,CAAC,EAAE,IAAI,kBAAkB,CAAC,IAAK,CAAC;QACnE,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,KAAK,cAAc,CAAC,gBAAgB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,MAChB,SACH,CAAC,WAAW,EAAE,CAAC;YAChB,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CACjC;gBACI,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,SAAS;aACpB,EACD,IAAI,CAAC,MAAM,CACd,CAAC;YACF,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,kBAAkB,CAC5B,UAA8C,EAC9C,MAAsB;QAEtB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,KAAK,cAAc,CAAC,gBAAgB,EAAE,CAAC;YAE7C,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC9E,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;YAC9B,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,UAAU,EAAE,EAAE,EAAE,CAAC;YACjB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;QACjC,CAAC;QAED,IAAI,UAAU,EAAE,IAAI,EAAE,CAAC;YACnB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAW,CAAC;QAE5D,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;IAC7B,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,gBAAgB,CAC5B,UAA6B,EAC7B,sBAA8B,EAC9B,SAAwB;QAExB,QAAQ,sBAAsB,EAAE,CAAC;YAC7B,KAAK,SAAS;gBACV,OAAO,SAAS,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YACrD,KAAK,eAAe;gBAChB,OAAO,SAAS,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;YAC3D,KAAK,cAAc;gBACf,OAAO,SAAS,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;YAC1D;gBACI,MAAM,IAAI,KAAK,CAAC,yBAAyB,sBAAsB,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;CACJ"}