@crawlee/core 3.18.0 → 3.18.1-beta.1

package/configuration.js

@@ -10,6 +10,7 @@ const fs_extra_1 = require("fs-extra");
 const log_1 = tslib_1.__importStar(require("@apify/log"));
 const events_1 = require("./events");
 const typedefs_1 = require("./typedefs");
+const memoryStorages = new Map();
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
  * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
@@ -200,17 +201,20 @@ class Configuration {
      * @internal
      */
     createMemoryStorage(options = {}) {
-        const cacheKey = `MemoryStorage-${JSON.stringify(options)}`;
-        if (this.services.has(cacheKey)) {
-            return this.services.get(cacheKey);
-        }
-        const storage = new memory_storage_1.MemoryStorage({
+        const storageOptions = {
             persistStorage: this.get('persistStorage'),
             // Override persistStorage if user provides it via storageClientOptions
             ...options,
-        });
-        this.services.set(cacheKey, storage);
-        return storage;
+        };
+        const cacheKey = `MemoryStorage-${JSON.stringify(storageOptions)}-${process.env.CRAWLEE_STORAGE_DIR ?? ''}`;
+        // Clients writing to the same directory are not independent, so they are shared between
+        // `Configuration` instances - otherwise they would purge each other's data mid-run.
+        // Purely in-memory clients have no such conflict and stay isolated per configuration.
+        const cache = storageOptions.persistStorage ? memoryStorages : this.services;
+        if (!cache.has(cacheKey)) {
+            cache.set(cacheKey, new memory_storage_1.MemoryStorage(storageOptions));
+        }
+        return cache.get(cacheKey);
     }
     useStorageClient(client) {
         this.options.set('storageClient', client);
@@ -249,6 +253,7 @@ class Configuration {
      */
     static resetGlobalState() {
         delete this.globalConfig;
+        memoryStorages.clear();
     }
     buildOptions(options) {
         // try to load configuration from crawlee.json as the baseline

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.18.0",
+    "version": "3.18.1-beta.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,9 +59,9 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.18.0",
-        "@crawlee/types": "3.18.0",
-        "@crawlee/utils": "3.18.0",
+        "@crawlee/memory-storage": "3.18.1-beta.1",
+        "@crawlee/types": "3.18.1-beta.1",
+        "@crawlee/utils": "3.18.1-beta.1",
         "@sapphire/async-queue": "^1.5.1",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.2.2",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "49c115e1ce3b3fbf2bef61f16bffeadfdcf5bd19"
+    "gitHead": "9940ec8a3b663ff64e9674057e2ac0fa34ba63cf"
 }

package/storages/utils.js

@@ -18,11 +18,20 @@ async function purgeDefaultStorages(configOrOptions, client) {
     const { config = configuration_1.Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
     ({ client = config.getStorageClient() } = options);
     const casted = client;
+    const runPurge = async () => {
+        try {
+            await casted.purge?.();
+        }
+        catch (e) {
+            casted.__purged = undefined;
+            throw e;
+        }
+    };
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
     if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
-        casted.__purged = true;
-        await casted.purge?.();
+        casted.__purged = runPurge();
     }
+    await casted.__purged;
 }
 /**
  * Easily create and manage state values. All state values are automatically persisted.