@crawlee/memory-storage 4.0.0-beta.65 → 4.0.0-beta.67

package/consts.d.ts

@@ -2,16 +2,4 @@
  * Length of id property of a Request instance in characters.
  */
 export declare const REQUEST_ID_LENGTH = 15;
-/**
- * Types of all emulated storages (currently used for warning messages only).
- */
-export declare enum StorageTypes {
-    RequestQueue = "Request queue",
-    KeyValueStore = "Key-value store",
-    Dataset = "Dataset"
-}
-/**
- * Except in dataset items, the default limit for API results is 1000.
- */
-export declare const DEFAULT_API_PARAM_LIMIT = 1000;
 //# sourceMappingURL=consts.d.ts.map

package/consts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC;;GAEG;AACH,oBAAY,YAAY;IACpB,YAAY,kBAAkB;IAC9B,aAAa,oBAAoB;IACjC,OAAO,YAAY;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,uBAAuB,OAAO,CAAC"}
+{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC"}

package/consts.js

@@ -2,17 +2,4 @@
  * Length of id property of a Request instance in characters.
  */
 export const REQUEST_ID_LENGTH = 15;
-/**
- * Types of all emulated storages (currently used for warning messages only).
- */
-export var StorageTypes;
-(function (StorageTypes) {
-    StorageTypes["RequestQueue"] = "Request queue";
-    StorageTypes["KeyValueStore"] = "Key-value store";
-    StorageTypes["Dataset"] = "Dataset";
-})(StorageTypes || (StorageTypes = {}));
-/**
- * Except in dataset items, the default limit for API results is 1000.
- */
-export const DEFAULT_API_PARAM_LIMIT = 1000;
 //# sourceMappingURL=consts.js.map

package/consts.js.map

@@ -1 +1 @@
-{"version":3,"file":"consts.js","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEpC;;GAEG;AACH,MAAM,CAAN,IAAY,YAIX;AAJD,WAAY,YAAY;IACpB,8CAA8B,CAAA;IAC9B,iDAAiC,CAAA;IACjC,mCAAmB,CAAA;AACvB,CAAC,EAJW,YAAY,KAAZ,YAAY,QAIvB;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC"}
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC"}

package/memory-storage.d.ts

@@ -5,44 +5,24 @@ import { KeyValueStoreClient } from './resource-clients/key-value-store.js';
 import { RequestQueueClient } from './resource-clients/request-queue.js';
 export interface MemoryStorageOptions {
     /**
-     * Path to directory where the data will also be saved.
-     * @default process.env.CRAWLEE_STORAGE_DIR ?? './storage'
-     */
-    localDataDirectory?: string;
-    /**
-     * Whether to also write optional metadata files when storing to disk.
-     * @default process.env.DEBUG?.includes('*') ?? process.env.DEBUG?.includes('crawlee:memory-storage') ?? false
-     */
-    writeMetadata?: boolean;
-    /**
-     * Whether the memory storage should also write its stored content to the disk.
-     *
-     * You can also disable this by setting the `CRAWLEE_PERSIST_STORAGE` environment variable to `false`.
-     * @default true
-     */
-    persistStorage?: boolean;
-    /**
-     * Optional logger for MemoryStorage warnings.
+     * Optional logger for MemoryStorageClient warnings.
      */
     logger?: CrawleeLogger;
 }
-export declare class MemoryStorage implements storage.StorageClient {
-    readonly localDataDirectory: string;
-    readonly datasetsDirectory: string;
-    readonly keyValueStoresDirectory: string;
-    readonly requestQueuesDirectory: string;
-    readonly writeMetadata: boolean;
-    readonly persistStorage: boolean;
+export declare class MemoryStorageClient implements storage.StorageClient {
     readonly logger?: CrawleeLogger;
+    /**
+     * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageClient` partitions its
+     * cache by storage directory: two distinct `MemoryStorageClient` instances must not share cached clients.
+     */
+    private readonly instanceCacheKey;
     readonly keyValueStoreCache: KeyValueStoreClient[];
     readonly datasetClientCache: DatasetClient[];
     readonly requestQueueCache: RequestQueueClient[];
     constructor(options?: MemoryStorageOptions);
     /**
-     * Return a cache key that includes the resolved storage directory, so that
-     * two `MemoryStorage` instances pointing at different directories get separate
-     * cache partitions. Mirrors crawlee-python's `FileSystemStorageClient` which
-     * includes `configuration.storage_dir` in its cache key.
+     * Return a per-instance unique cache key so that distinct `MemoryStorageClient` instances get separate
+     * cache partitions in the storage-client cache.
      */
     getStorageClientCacheKey(): string;
     private static resolveStorageKey;
@@ -52,17 +32,17 @@ export declare class MemoryStorage implements storage.StorageClient {
     storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;
     /**
-     * Cleans up the default storage directories before the run starts:
-     *  - local directory containing the default dataset;
-     *  - all records from the default key-value store in the local directory, except for the "INPUT" key;
-     *  - local directory containing the default request queue.
+     * Cleans up the default storages before the run starts. For the in-memory storage this simply
+     * resets the in-memory state of the cached default dataset, key-value store and request queue.
+     *
+     * As with `FileSystemStorageClient`, the run's input (the `INPUT` key in the default key-value
+     * store) is preserved — only the rest of the default storages is cleared.
      */
     purge(): Promise<void>;
     /**
-     * This method should be called at the end of the process, to ensure all data is saved.
+     * This method should be called at the end of the process. The in-memory storage holds no resources
+     * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
      */
     teardown(): Promise<void>;
-    private handleDefaultKeyValueStore;
-    private batchRemoveFiles;
 }
 //# sourceMappingURL=memory-storage.d.ts.map

package/memory-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"memory-storage.d.ts","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAUpD,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAEzE,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qBAAa,aAAc,YAAW,OAAO,CAAC,aAAa;IACvD,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAEhC,QAAQ,CAAC,kBAAkB,EAAE,mBAAmB,EAAE,CAAM;IACxD,QAAQ,CAAC,kBAAkB,EAAE,aAAa,EAAE,CAAM;IAClD,QAAQ,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,CAAM;gBAE1C,OAAO,GAAE,oBAAyB;IAoC9C;;;;;OAKG;IACH,wBAAwB,IAAI,MAAM;IAIlC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAW1B,mBAAmB,CAAC,OAAO,GAAE,OAAO,CAAC,0BAA+B,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;IAqCrG,yBAAyB,CAC3B,OAAO,GAAE,OAAO,CAAC,gCAAqC,GACvD,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC;IAqCjC,wBAAwB,CAC1B,OAAO,GAAE,OAAO,CAAC,+BAAoC,GACtD,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAqChC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IA+C/F,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,uBAA4B,GAAG,OAAO,CAAC,IAAI,CAAC;IASrG;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD5B;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAM/B,OAAO,CAAC,0BAA0B;YA+CpB,gBAAgB;CA0CjC"}
+{"version":3,"file":"memory-storage.d.ts","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAEzE,MAAM,WAAW,oBAAoB;IACjC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qBAAa,mBAAoB,YAAW,OAAO,CAAC,aAAa;IAC7D,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAEhC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAyC;IAE1E,QAAQ,CAAC,kBAAkB,EAAE,mBAAmB,EAAE,CAAM;IACxD,QAAQ,CAAC,kBAAkB,EAAE,aAAa,EAAE,CAAM;IAClD,QAAQ,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,CAAM;gBAE1C,OAAO,GAAE,oBAAyB;IAI9C;;;OAGG;IACH,wBAAwB,IAAI,MAAM;IAIlC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAW1B,mBAAmB,CAAC,OAAO,GAAE,OAAO,CAAC,0BAA+B,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;IAyBrG,yBAAyB,CAC3B,OAAO,GAAE,OAAO,CAAC,gCAAqC,GACvD,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC;IAyBjC,wBAAwB,CAC1B,OAAO,GAAE,OAAO,CAAC,+BAAoC,GACtD,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAyBhC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IAqB/F,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,uBAA4B,GAAG,OAAO,CAAC,IAAI,CAAC;IASrG;;;;;;OAMG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB5B;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGlC"}

package/memory-storage.js

@@ -1,196 +1,106 @@
-/* eslint-disable import/no-duplicates */
-import { access, readdir, rm } from 'node:fs/promises';
-import { resolve } from 'node:path';
+import { randomUUID } from 'node:crypto';
 import { s } from '@sapphire/shapeshift';
-import { ensureDirSync, move, moveSync, pathExistsSync } from 'fs-extra/esm';
-import { promiseMap, scheduleBackgroundTask } from './background-handler/index.js';
-import { findOrCacheDatasetByPossibleId, findOrCacheKeyValueStoreByPossibleId, findRequestQueueByPossibleId, } from './cache-helpers.js';
 import { DatasetClient } from './resource-clients/dataset.js';
 import { KeyValueStoreClient } from './resource-clients/key-value-store.js';
 import { RequestQueueClient } from './resource-clients/request-queue.js';
-export class MemoryStorage {
-    localDataDirectory;
-    datasetsDirectory;
-    keyValueStoresDirectory;
-    requestQueuesDirectory;
-    writeMetadata;
-    persistStorage;
+export class MemoryStorageClient {
     logger;
+    /**
+     * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageClient` partitions its
+     * cache by storage directory: two distinct `MemoryStorageClient` instances must not share cached clients.
+     */
+    instanceCacheKey = `MemoryStorageClient:${randomUUID()}`;
     keyValueStoreCache = [];
     datasetClientCache = [];
     requestQueueCache = [];
     constructor(options = {}) {
-        s.object({
-            localDataDirectory: s.string().optional(),
-            writeMetadata: s.boolean().optional(),
-            persistStorage: s.boolean().optional(),
-        }).parse(options);
         this.logger = options.logger;
-        // v3.0.0 used `crawlee_storage` as the default, we changed this in v3.0.1 to just `storage`,
-        // this function handles it without making BC breaks - it respects existing `crawlee_storage`
-        // directories, and uses the `storage` only if it's not there.
-        const defaultStorageDir = () => {
-            if (pathExistsSync(resolve('./crawlee_storage'))) {
-                return './crawlee_storage';
-            }
-            return './storage';
-        };
-        this.localDataDirectory = options.localDataDirectory ?? process.env.CRAWLEE_STORAGE_DIR ?? defaultStorageDir();
-        this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
-        this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
-        this.requestQueuesDirectory = resolve(this.localDataDirectory, 'request_queues');
-        this.writeMetadata =
-            options.writeMetadata ??
-                process.env.DEBUG?.includes('*') ??
-                process.env.DEBUG?.includes('crawlee:memory-storage') ??
-                false;
-        this.persistStorage =
-            options.persistStorage ??
-                (process.env.CRAWLEE_PERSIST_STORAGE
-                    ? !['false', '0', ''].includes(process.env.CRAWLEE_PERSIST_STORAGE)
-                    : true);
     }
     /**
-     * Return a cache key that includes the resolved storage directory, so that
-     * two `MemoryStorage` instances pointing at different directories get separate
-     * cache partitions. Mirrors crawlee-python's `FileSystemStorageClient` which
-     * includes `configuration.storage_dir` in its cache key.
+     * Return a per-instance unique cache key so that distinct `MemoryStorageClient` instances get separate
+     * cache partitions in the storage-client cache.
      */
     getStorageClientCacheKey() {
-        return `MemoryStorage:${resolve(this.localDataDirectory)}`;
+        return this.instanceCacheKey;
     }
     static resolveStorageKey(options) {
         const isAlias = 'alias' in options && !!options.alias;
         const rawKey = isAlias ? options.alias : (options.name ?? options.id);
         // Normalize the internal __default__ alias to the user-facing 'default' name.
-        const directoryKey = rawKey === '__default__' ? 'default' : rawKey;
-        return { isAlias, directoryKey };
+        const cacheKey = rawKey === '__default__' ? 'default' : rawKey;
+        return { isAlias, cacheKey };
     }
     async createDatasetClient(options = {}) {
-        const { isAlias, directoryKey } = MemoryStorage.resolveStorageKey(options);
-        if (directoryKey) {
-            const found = await findOrCacheDatasetByPossibleId(this, directoryKey);
+        const { isAlias, cacheKey } = MemoryStorageClient.resolveStorageKey(options);
+        if (cacheKey) {
+            const found = this.datasetClientCache.find((store) => store.id === cacheKey ||
+                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
             if (found) {
                 return found;
             }
         }
         const newStore = new DatasetClient({
-            name: isAlias ? undefined : directoryKey,
-            directoryName: directoryKey,
-            baseStorageDirectory: this.datasetsDirectory,
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
             client: this,
         });
         this.datasetClientCache.push(newStore);
-        // Schedule the worker to write to the disk
-        const datasetInfo = newStore.toDatasetInfo();
-        scheduleBackgroundTask({
-            action: 'update-metadata',
-            entityType: 'datasets',
-            entityDirectory: newStore.datasetDirectory,
-            id: datasetInfo.name ?? datasetInfo.id,
-            data: datasetInfo,
-            writeMetadata: this.writeMetadata,
-            persistStorage: this.persistStorage,
-        }, this.logger);
         return newStore;
     }
     async createKeyValueStoreClient(options = {}) {
-        const { isAlias, directoryKey } = MemoryStorage.resolveStorageKey(options);
-        if (directoryKey) {
-            const found = await findOrCacheKeyValueStoreByPossibleId(this, directoryKey);
+        const { isAlias, cacheKey } = MemoryStorageClient.resolveStorageKey(options);
+        if (cacheKey) {
+            const found = this.keyValueStoreCache.find((store) => store.id === cacheKey ||
+                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
             if (found) {
                 return found;
             }
         }
         const newStore = new KeyValueStoreClient({
-            name: isAlias ? undefined : directoryKey,
-            directoryName: directoryKey,
-            baseStorageDirectory: this.keyValueStoresDirectory,
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
             client: this,
         });
         this.keyValueStoreCache.push(newStore);
-        // Schedule the worker to write to the disk
-        const kvStoreInfo = newStore.toKeyValueStoreInfo();
-        scheduleBackgroundTask({
-            action: 'update-metadata',
-            entityType: 'keyValueStores',
-            entityDirectory: newStore.keyValueStoreDirectory,
-            id: kvStoreInfo.name ?? kvStoreInfo.id,
-            data: kvStoreInfo,
-            writeMetadata: this.writeMetadata,
-            persistStorage: this.persistStorage,
-        }, this.logger);
         return newStore;
     }
     async createRequestQueueClient(options = {}) {
-        const { isAlias, directoryKey } = MemoryStorage.resolveStorageKey(options);
-        if (directoryKey) {
-            const found = await findRequestQueueByPossibleId(this, directoryKey);
+        const { isAlias, cacheKey } = MemoryStorageClient.resolveStorageKey(options);
+        if (cacheKey) {
+            const found = this.requestQueueCache.find((queue) => queue.id === cacheKey ||
+                queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
+                queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
             if (found) {
                 return found;
             }
         }
         const newStore = new RequestQueueClient({
-            name: isAlias ? undefined : directoryKey,
-            directoryName: directoryKey,
-            baseStorageDirectory: this.requestQueuesDirectory,
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
             client: this,
         });
         this.requestQueueCache.push(newStore);
-        // Schedule the worker to write to the disk
-        const queueInfo = newStore.toRequestQueueInfo();
-        scheduleBackgroundTask({
-            action: 'update-metadata',
-            entityType: 'requestQueues',
-            entityDirectory: newStore.requestQueueDirectory,
-            id: queueInfo.name ?? queueInfo.id,
-            data: queueInfo,
-            writeMetadata: this.writeMetadata,
-            persistStorage: this.persistStorage,
-        }, this.logger);
         return newStore;
     }
     async storageExists(id, type) {
         let clients;
-        let baseDir;
         switch (type) {
             case 'Dataset':
                 clients = this.datasetClientCache;
-                baseDir = this.datasetsDirectory;
                 break;
             case 'KeyValueStore':
                 clients = this.keyValueStoreCache;
-                baseDir = this.keyValueStoresDirectory;
                 break;
             case 'RequestQueue':
                 clients = this.requestQueueCache;
-                baseDir = this.requestQueuesDirectory;
                 break;
             default:
                 return false;
         }
-        // Check in-memory cache by actual storage ID
-        if (clients.some((store) => store.id === id)) {
-            return true;
-        }
-        // Check if a directory with that ID exists on disk.
-        // Only consider directories whose name matches the queried ID — this avoids
-        // false positives for alias-created directories (e.g. a directory named 'asdf'
-        // created via `{ alias: 'asdf' }` should not make `storageExists('asdf')` return true,
-        // since the actual storage ID is a UUID, not the alias string).
-        try {
-            await access(resolve(baseDir, id));
-            // If the directory exists but a cached client already owns this directory
-            // under a different ID, this is not a match.
-            const cachedClients = clients;
-            if (cachedClients.some((store) => store.directoryName === id && store.id !== id)) {
-                return false;
-            }
-            return true;
-        }
-        catch {
-            return false;
-        }
+        // In-memory storage only knows about clients in its cache.
+        return clients.some((store) => store.id === id);
     }
     async setStatusMessage(message, options = {}) {
         s.string().parse(message);
@@ -200,126 +110,36 @@ export class MemoryStorage {
         return Promise.resolve();
     }
     /**
-     * Cleans up the default storage directories before the run starts:
-     *  - local directory containing the default dataset;
-     *  - all records from the default key-value store in the local directory, except for the "INPUT" key;
-     *  - local directory containing the default request queue.
+     * Cleans up the default storages before the run starts. For the in-memory storage this simply
+     * resets the in-memory state of the cached default dataset, key-value store and request queue.
+     *
+     * As with `FileSystemStorageClient`, the run's input (the `INPUT` key in the default key-value
+     * store) is preserved — only the rest of the default storages is cleared.
      */
     async purge() {
-        // Key-value stores
-        const keyValueStores = await readdir(this.keyValueStoresDirectory).catch(() => []);
-        const keyValueStorePromises = [];
-        for (const keyValueStoreFolder of keyValueStores) {
-            if (keyValueStoreFolder.startsWith('__CRAWLEE_TEMPORARY') || keyValueStoreFolder.startsWith('__OLD')) {
-                keyValueStorePromises.push((await this.batchRemoveFiles(resolve(this.keyValueStoresDirectory, keyValueStoreFolder)))());
-            }
-            else if (keyValueStoreFolder === 'default' || keyValueStoreFolder === '__default__') {
-                keyValueStorePromises.push(this.handleDefaultKeyValueStore(resolve(this.keyValueStoresDirectory, keyValueStoreFolder))());
-            }
-        }
-        void Promise.allSettled(keyValueStorePromises);
-        // Datasets
-        const datasets = await readdir(this.datasetsDirectory).catch(() => []);
-        const datasetPromises = [];
-        for (const datasetFolder of datasets) {
-            if (datasetFolder === 'default' ||
-                datasetFolder === '__default__' ||
-                datasetFolder.startsWith('__CRAWLEE_TEMPORARY')) {
-                datasetPromises.push((await this.batchRemoveFiles(resolve(this.datasetsDirectory, datasetFolder)))());
-            }
-        }
-        void Promise.allSettled(datasetPromises);
-        // Request queues
-        const requestQueues = await readdir(this.requestQueuesDirectory).catch(() => []);
-        const requestQueuePromises = [];
-        for (const requestQueueFolder of requestQueues) {
-            if (requestQueueFolder === 'default' ||
-                requestQueueFolder === '__default__' ||
-                requestQueueFolder.startsWith('__CRAWLEE_TEMPORARY')) {
-                requestQueuePromises.push((await this.batchRemoveFiles(resolve(this.requestQueuesDirectory, requestQueueFolder)))());
-            }
-        }
-        void Promise.allSettled(requestQueuePromises);
+        // The run default is opened via `{ alias: '__default__' }`, which `resolveStorageKey`
+        // normalizes to `cacheKey === 'default'` (with `name === undefined`) — that is the clause
+        // that actually matches it. The `name === 'default'` clause additionally covers a store a user
+        // explicitly opened via `{ name: 'default' }`. (`'__default__'` never reaches `cacheKey`,
+        // as it is always normalized to `'default'` first, so it does not need to be checked here.)
+        const isDefault = (store) => store.name === 'default' || store.cacheKey === 'default';
+        const purgeDefaults = async (cache, purgeStore) => {
+            await Promise.all(cache.filter(isDefault).map(async (store) => purgeStore(store)));
+        };
+        await Promise.all([
+            // Preserve the run input (INPUT) when purging the default key-value store, matching
+            // `FileSystemStorageClient`.
+            purgeDefaults(this.keyValueStoreCache, async (store) => store.purgeExceptInput()),
+            purgeDefaults(this.datasetClientCache, async (store) => store.purge()),
+            purgeDefaults(this.requestQueueCache, async (store) => store.purge()),
+        ]);
     }
     /**
-     * This method should be called at the end of the process, to ensure all data is saved.
+     * This method should be called at the end of the process. The in-memory storage holds no resources
+     * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
      */
     async teardown() {
-        const promises = [...promiseMap.values()].map(async ({ promise }) => promise);
-        await Promise.all(promises);
-    }
-    handleDefaultKeyValueStore(folder) {
-        const storagePathExists = pathExistsSync(folder);
-        const temporaryPath = resolve(folder, '../__CRAWLEE_MIGRATING_KEY_VALUE_STORE__');
-        // For optimization, we want to only attempt to copy a few files from the default key-value store
-        const possibleInputKeys = ['INPUT', 'INPUT.json', 'INPUT.bin', 'INPUT.txt'];
-        if (storagePathExists) {
-            // Create temporary folder to save important files in
-            ensureDirSync(temporaryPath);
-            // Go through each file and save the ones that are important
-            for (const entity of possibleInputKeys) {
-                const originalFilePath = resolve(folder, entity);
-                const tempFilePath = resolve(temporaryPath, entity);
-                try {
-                    moveSync(originalFilePath, tempFilePath);
-                }
-                catch {
-                    // Ignore
-                }
-            }
-            // Remove the original folder and all its content
-            let counter = 0;
-            let tempPathForOldFolder = resolve(folder, `../__OLD_DEFAULT_${counter}__`);
-            let done = false;
-            while (!done) {
-                try {
-                    moveSync(folder, tempPathForOldFolder);
-                    done = true;
-                }
-                catch {
-                    tempPathForOldFolder = resolve(folder, `../__OLD_DEFAULT_${++counter}__`);
-                }
-            }
-            // Replace the temporary folder with the original folder
-            moveSync(temporaryPath, folder);
-            // Remove the old folder
-            return async () => (await this.batchRemoveFiles(tempPathForOldFolder))();
-        }
-        return async () => Promise.resolve();
-    }
-    async batchRemoveFiles(folder, counter = 0) {
-        const folderExists = pathExistsSync(folder);
-        if (folderExists) {
-            const temporaryFolder = resolve(folder, `../__CRAWLEE_TEMPORARY_${counter}__`);
-            try {
-                // Rename the old folder to the new one to allow background deletions
-                await move(folder, temporaryFolder);
-            }
-            catch {
-                // Folder exists already, try again with an incremented counter
-                return this.batchRemoveFiles(folder, ++counter);
-            }
-            return async () => {
-                // Read all files in the folder
-                const entries = await readdir(temporaryFolder);
-                let processed = 0;
-                let promises = [];
-                for (const entry of entries) {
-                    processed++;
-                    promises.push(rm(resolve(temporaryFolder, entry), { force: true }));
-                    // Every 2000 files, delete them
-                    if (processed % 2000 === 0) {
-                        await Promise.allSettled(promises);
-                        promises = [];
-                    }
-                }
-                // Ensure last promises are handled
-                await Promise.allSettled(promises);
-                // Delete the folder itself
-                await rm(temporaryFolder, { force: true, recursive: true });
-            };
-        }
-        return async () => Promise.resolve();
+        // Nothing to tear down for in-memory storage.
     }
 }
 //# sourceMappingURL=memory-storage.js.map

package/memory-storage.js.map

@@ -1 +1 @@
-{"version":3,"file":"memory-storage.js","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE7E,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACnF,OAAO,EACH,8BAA8B,EAC9B,oCAAoC,EACpC,4BAA4B,GAC/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AA6BzE,MAAM,OAAO,aAAa;IACb,kBAAkB,CAAS;IAC3B,iBAAiB,CAAS;IAC1B,uBAAuB,CAAS;IAChC,sBAAsB,CAAS;IAC/B,aAAa,CAAU;IACvB,cAAc,CAAU;IACxB,MAAM,CAAiB;IAEvB,kBAAkB,GAA0B,EAAE,CAAC;IAC/C,kBAAkB,GAAoB,EAAE,CAAC;IACzC,iBAAiB,GAAyB,EAAE,CAAC;IAEtD,YAAY,UAAgC,EAAE;QAC1C,CAAC,CAAC,MAAM,CAAC;YACL,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACzC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YACrC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SACzC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAE7B,6FAA6F;QAC7F,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAM,iBAAiB,GAAG,GAAG,EAAE;YAC3B,IAAI,cAAc,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,EAAE,CAAC;gBAC/C,OAAO,mBAAmB,CAAC;YAC/B,CAAC;YAED,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,iBAAiB,EAAE,CAAC;QAC/G,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;QACpF,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QACjF,IAAI,CAAC,aAAa;YACd,OAAO,CAAC,aAAa;gBACrB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,wBAAwB,CAAC;gBACrD,KAAK,CAAC;QACV,IAAI,CAAC,cAAc;YACf,OAAO,CAAC,cAAc;gBACtB,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB;oBAChC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAwB,CAAC;oBACpE,CAAC,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,wBAAwB;QACpB,OAAO,iBAAiB,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;IAC/D,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,OAAuD;QAIpF,MAAM,OAAO,GAAG,OAAO,IAAI,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACtE,8EAA8E;QAC9E,MAAM,YAAY,GAAG,MAAM,KAAK,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACnE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,UAA8C,EAAE;QACtE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE3E,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACvE,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,aAAa,CAAC;YAC/B,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY;YACxC,aAAa,EAAE,YAAY;YAC3B,oBAAoB,EAAE,IAAI,CAAC,iBAAiB;YAC5C,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QAE7C,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,QAAQ,CAAC,gBAAgB;YAC1C,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;YACjB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,EACD,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC3B,UAAoD,EAAE;QAEtD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE3E,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,oCAAoC,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAC7E,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;YACrC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY;YACxC,aAAa,EAAE,YAAY;YAC3B,oBAAoB,EAAE,IAAI,CAAC,uBAAuB;YAClD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QAEnD,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,QAAQ,CAAC,sBAAsB;YAChD,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;YACjB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,EACD,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC1B,UAAmD,EAAE;QAErD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE3E,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,4BAA4B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACrE,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;YACpC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY;YACxC,aAAa,EAAE,YAAY;YAC3B,oBAAoB,EAAE,IAAI,CAAC,sBAAsB;YACjD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEtC,2CAA2C;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,EAAE,CAAC;QAEhD,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,eAAe;YAC3B,eAAe,EAAE,QAAQ,CAAC,qBAAqB;YAC/C,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE;YAClC,IAAI,EAAE,SAAS;YACf,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,EACD,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAU,EAAE,IAAkD;QAC9E,IAAI,OAAyB,CAAC;QAC9B,IAAI,OAAe,CAAC;QAEpB,QAAQ,IAAI,EAAE,CAAC;YACX,KAAK,SAAS;gBACV,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBAClC,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjC,MAAM;YACV,KAAK,eAAe;gBAChB,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBAClC,OAAO,GAAG,IAAI,CAAC,uBAAuB,CAAC;gBACvC,MAAM;YACV,KAAK,cAAc;gBACf,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjC,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC;gBACtC,MAAM;YACV;gBACI,OAAO,KAAK,CAAC;QACrB,CAAC;QAED,6CAA6C;QAC7C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,oDAAoD;QACpD,4EAA4E;QAC5E,+EAA+E;QAC/E,uFAAuF;QACvF,gEAAgE;QAChE,IAAI,CAAC;YACD,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAEnC,0EAA0E;YAC1E,6CAA6C;YAC7C,MAAM,aAAa,GAAG,OAAmD,CAAC;YAC1E,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,EAAE,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAC/E,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,UAA2C,EAAE;QACjF,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,MAAM,CAAC;YACL,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAClD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,mBAAmB;QACnB,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,qBAAqB,GAAoB,EAAE,CAAC;QAElD,KAAK,MAAM,mBAAmB,IAAI,cAAc,EAAE,CAAC;YAC/C,IAAI,mBAAmB,CAAC,UAAU,CAAC,qBAAqB,CAAC,IAAI,mBAAmB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnG,qBAAqB,CAAC,IAAI,CACtB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAC9F,CAAC;YACN,CAAC;iBAAM,IAAI,mBAAmB,KAAK,SAAS,IAAI,mBAAmB,KAAK,aAAa,EAAE,CAAC;gBACpF,qBAAqB,CAAC,IAAI,CACtB,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,EAAE,CAChG,CAAC;YACN,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QAE/C,WAAW;QACX,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,eAAe,GAAoB,EAAE,CAAC;QAE5C,KAAK,MAAM,aAAa,IAAI,QAAQ,EAAE,CAAC;YACnC,IACI,aAAa,KAAK,SAAS;gBAC3B,aAAa,KAAK,aAAa;gBAC/B,aAAa,CAAC,UAAU,CAAC,qBAAqB,CAAC,EACjD,CAAC;gBACC,eAAe,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QAEzC,iBAAiB;QACjB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACjF,MAAM,oBAAoB,GAAoB,EAAE,CAAC;QAEjD,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;YAC7C,IACI,kBAAkB,KAAK,SAAS;gBAChC,kBAAkB,KAAK,aAAa;gBACpC,kBAAkB,CAAC,UAAU,CAAC,qBAAqB,CAAC,EACtD,CAAC;gBACC,oBAAoB,CAAC,IAAI,CACrB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAC5F,CAAC;YACN,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QAE9E,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAEO,0BAA0B,CAAC,MAAc;QAC7C,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,0CAA0C,CAAC,CAAC;QAElF,iGAAiG;QACjG,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAE5E,IAAI,iBAAiB,EAAE,CAAC;YACpB,qDAAqD;YACrD,aAAa,CAAC,aAAa,CAAC,CAAC;YAE7B,4DAA4D;YAC5D,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;gBACrC,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACjD,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBAEpD,IAAI,CAAC;oBACD,QAAQ,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACL,SAAS;gBACb,CAAC;YACL,CAAC;YAED,iDAAiD;YACjD,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,oBAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,oBAAoB,OAAO,IAAI,CAAC,CAAC;YAC5E,IAAI,IAAI,GAAG,KAAK,CAAC;YAEjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,CAAC;oBACD,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;oBACvC,IAAI,GAAG,IAAI,CAAC;gBAChB,CAAC;gBAAC,MAAM,CAAC;oBACL,oBAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,IAAI,CAAC,CAAC;gBAC9E,CAAC;YACL,CAAC;YAED,wDAAwD;YACxD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEhC,wBAAwB;YACxB,OAAO,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,MAAc,EAAE,OAAO,GAAG,CAAC;QACtD,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAE5C,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,0BAA0B,OAAO,IAAI,CAAC,CAAC;YAE/E,IAAI,CAAC;gBACD,qEAAqE;gBACrE,MAAM,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACL,+DAA+D;gBAC/D,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAED,OAAO,KAAK,IAAI,EAAE;gBACd,+BAA+B;gBAC/B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAC;gBAE/C,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,QAAQ,GAAoB,EAAE,CAAC;gBAEnC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC1B,SAAS,EAAE,CAAC;oBACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAEpE,gCAAgC;oBAChC,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;wBACzB,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;wBACnC,QAAQ,GAAG,EAAE,CAAC;oBAClB,CAAC;gBACL,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAEnC,2BAA2B;gBAC3B,MAAM,EAAE,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC;QACN,CAAC;QAED,OAAO,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;CACJ"}
+{"version":3,"file":"memory-storage.js","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AASzE,MAAM,OAAO,mBAAmB;IACnB,MAAM,CAAiB;IAEhC;;;OAGG;IACc,gBAAgB,GAAG,uBAAuB,UAAU,EAAE,EAAE,CAAC;IAEjE,kBAAkB,GAA0B,EAAE,CAAC;IAC/C,kBAAkB,GAAoB,EAAE,CAAC;IACzC,iBAAiB,GAAyB,EAAE,CAAC;IAEtD,YAAY,UAAgC,EAAE;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,wBAAwB;QACpB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,OAAuD;QAIpF,MAAM,OAAO,GAAG,OAAO,IAAI,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACtE,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,MAAM,KAAK,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,UAA8C,EAAE;QACtE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,mBAAmB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CACtC,CAAC,KAAK,EAAE,EAAE,CACN,KAAK,CAAC,EAAE,KAAK,QAAQ;gBACrB,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE;gBACpD,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,CAC9D,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,aAAa,CAAC;YAC/B,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ;YACpC,QAAQ;YACR,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC3B,UAAoD,EAAE;QAEtD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,mBAAmB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CACtC,CAAC,KAAK,EAAE,EAAE,CACN,KAAK,CAAC,EAAE,KAAK,QAAQ;gBACrB,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE;gBACpD,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,CAC9D,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;YACrC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ;YACpC,QAAQ;YACR,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC1B,UAAmD,EAAE;QAErD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,mBAAmB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CACrC,CAAC,KAAK,EAAE,EAAE,CACN,KAAK,CAAC,EAAE,KAAK,QAAQ;gBACrB,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE;gBACpD,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,CAC9D,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;YACpC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ;YACpC,QAAQ;YACR,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEtC,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAU,EAAE,IAAkD;QAC9E,IAAI,OAAyB,CAAC;QAE9B,QAAQ,IAAI,EAAE,CAAC;YACX,KAAK,SAAS;gBACV,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBAClC,MAAM;YACV,KAAK,eAAe;gBAChB,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBAClC,MAAM;YACV,KAAK,cAAc;gBACf,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjC,MAAM;YACV;gBACI,OAAO,KAAK,CAAC;QACrB,CAAC;QAED,2DAA2D;QAC3D,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,UAA2C,EAAE;QACjF,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,MAAM,CAAC;YACL,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAClD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK;QACP,sFAAsF;QACtF,0FAA0F;QAC1F,+FAA+F;QAC/F,0FAA0F;QAC1F,4FAA4F;QAC5F,MAAM,SAAS,GAAG,CAAC,KAA0C,EAAE,EAAE,CAC7D,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;QAE7D,MAAM,aAAa,GAAG,KAAK,EACvB,KAAU,EACV,UAAuC,EACzC,EAAE;YACA,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvF,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,oFAAoF;YACpF,6BAA6B;YAC7B,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YACjF,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACtE,aAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;SACxE,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,8CAA8C;IAClD,CAAC;CACJ"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/memory-storage",
-    "version": "4.0.0-beta.65",
+    "version": "4.0.0-beta.67",
     "description": "A simple in-memory storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -42,15 +42,12 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/types": "4.0.0-beta.65",
+        "@crawlee/types": "4.0.0-beta.67",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "content-type": "^1.0.5",
-        "fs-extra": "^11.3.0",
         "json5": "^2.2.3",
         "mime-types": "^3.0.1",
-        "p-limit": "^6.2.0",
-        "proper-lockfile": "^4.1.2",
         "tslib": "^2.8.1"
     },
     "lerna": {
@@ -60,5 +57,5 @@
             }
         }
     },
-    "gitHead": "fcee63d935043be16f7be59ab9784161cf132b11"
+    "gitHead": "a0f083cdcc46c96d5e76ed2c75addbba4942c05f"
 }

package/resource-clients/common/base-client.d.ts

@@ -1,8 +1,5 @@
-import type { StorageTypes } from '../../consts.js';
 export declare class BaseClient {
     id: string;
     constructor(id: string);
-    protected throwOnNonExisting(clientType: StorageTypes): never;
-    protected throwOnDuplicateEntry(clientType: StorageTypes, keyName: string, value: string): never;
 }
 //# sourceMappingURL=base-client.d.ts.map

package/resource-clients/common/base-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base-client.d.ts","sourceRoot":"","sources":["../../../src/resource-clients/common/base-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEpD,qBAAa,UAAU;IACnB,EAAE,EAAE,MAAM,CAAC;gBAEC,EAAE,EAAE,MAAM;IAItB,SAAS,CAAC,kBAAkB,CAAC,UAAU,EAAE,YAAY,GAAG,KAAK;IAI7D,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK;CAGnG"}
+{"version":3,"file":"base-client.d.ts","sourceRoot":"","sources":["../../../src/resource-clients/common/base-client.ts"],"names":[],"mappings":"AAAA,qBAAa,UAAU;IACnB,EAAE,EAAE,MAAM,CAAC;gBAEC,EAAE,EAAE,MAAM;CAGzB"}

package/resource-clients/common/base-client.js

@@ -3,11 +3,5 @@ export class BaseClient {
     constructor(id) {
         this.id = id;
     }
-    throwOnNonExisting(clientType) {
-        throw new Error(`${clientType} with id: ${this.id} does not exist.`);
-    }
-    throwOnDuplicateEntry(clientType, keyName, value) {
-        throw new Error(`${clientType} with ${keyName}: ${value} already exists.`);
-    }
 }
 //# sourceMappingURL=base-client.js.map

package/resource-clients/common/base-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"base-client.js","sourceRoot":"","sources":["../../../src/resource-clients/common/base-client.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,UAAU;IACnB,EAAE,CAAS;IAEX,YAAY,EAAU;QAClB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;IAES,kBAAkB,CAAC,UAAwB;QACjD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,aAAa,IAAI,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACzE,CAAC;IAES,qBAAqB,CAAC,UAAwB,EAAE,OAAe,EAAE,KAAa;QACpF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,SAAS,OAAO,KAAK,KAAK,kBAAkB,CAAC,CAAC;IAC/E,CAAC;CACJ"}
+{"version":3,"file":"base-client.js","sourceRoot":"","sources":["../../../src/resource-clients/common/base-client.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,UAAU;IACnB,EAAE,CAAS;IAEX,YAAY,EAAU;QAClB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;CACJ"}