@crawlee/memory-storage 3.4.3-beta.9 → 3.5.0

package/workers/file-storage-worker-emulator.js

@@ -1,16 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FileStorageWorkerEmulator = void 0;
-const instance_1 = require("./instance");
-const worker_utils_1 = require("./worker-utils");
-class FileStorageWorkerEmulator {
-    async postMessage(value) {
-        await (0, worker_utils_1.handleMessage)(value);
-        instance_1.promiseMap.get(value.messageId)?.resolve();
-        instance_1.promiseMap.delete(value.messageId);
-    }
-    terminate() { }
-    unref() { }
-}
-exports.FileStorageWorkerEmulator = FileStorageWorkerEmulator;
-//# sourceMappingURL=file-storage-worker-emulator.js.map

package/workers/file-storage-worker-emulator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"file-storage-worker-emulator.js","sourceRoot":"","sources":["../../src/workers/file-storage-worker-emulator.ts"],"names":[],"mappings":";;;AAAA,yCAAwC;AACxC,iDAA+C;AAE/C,MAAa,yBAAyB;IAClC,KAAK,CAAC,WAAW,CAAC,KAAU;QACxB,MAAM,IAAA,4BAAa,EAAC,KAAK,CAAC,CAAC;QAE3B,qBAAU,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;QAC3C,qBAAU,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IAED,SAAS,KAAU,CAAC;IAEpB,KAAK,KAAU,CAAC;CACnB;AAXD,8DAWC"}

package/workers/file-storage-worker.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=file-storage-worker.d.ts.map

package/workers/file-storage-worker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"file-storage-worker.d.ts","sourceRoot":"","sources":["../../src/workers/file-storage-worker.ts"],"names":[],"mappings":""}

package/workers/file-storage-worker.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const node_worker_threads_1 = require("node:worker_threads");
-const worker_utils_1 = require("./worker-utils");
-if (node_worker_threads_1.isMainThread || !node_worker_threads_1.parentPort) {
-    throw new Error('This file should only be run in a worker thread!');
-}
-// Keep worker alive
-setInterval(() => {
-    node_worker_threads_1.parentPort.postMessage('ping');
-}, 30000);
-node_worker_threads_1.parentPort.on('message', async (message) => {
-    await (0, worker_utils_1.handleMessage)(message);
-});
-//# sourceMappingURL=file-storage-worker.js.map

package/workers/file-storage-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"file-storage-worker.js","sourceRoot":"","sources":["../../src/workers/file-storage-worker.ts"],"names":[],"mappings":";;AAAA,6DAA+D;AAE/D,iDAA+C;AAG/C,IAAI,kCAAY,IAAI,CAAC,gCAAU,EAAE;IAC7B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;CACvE;AAED,oBAAoB;AACpB,WAAW,CAAC,GAAG,EAAE;IACb,gCAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,EAAE,KAAM,CAAC,CAAC;AAEX,gCAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAsD,EAAE,EAAE;IACvF,MAAM,IAAA,4BAAa,EAAC,OAAO,CAAC,CAAC;AACjC,CAAC,CAAC,CAAC"}

package/workers/instance.d.ts

@@ -1,8 +0,0 @@
-import type { WorkerReceivedMessage } from '../utils';
-export declare const promiseMap: Map<string, {
-    promise: Promise<void>;
-    resolve: () => void;
-}>;
-export declare function sendWorkerMessage(message: WorkerReceivedMessage): void;
-export declare function initWorkerIfNeeded(): void;
-//# sourceMappingURL=instance.d.ts.map

package/workers/instance.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"instance.d.ts","sourceRoot":"","sources":["../../src/workers/instance.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAKtD,eAAO,MAAM,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE;IACjC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,EAAE,MAAM,IAAI,CAAC;CACvB,CAAa,CAAC;AAEf,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,QAiB/D;AAED,wBAAgB,kBAAkB,SAkCjC"}

package/workers/instance.js

@@ -1,58 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.initWorkerIfNeeded = exports.sendWorkerMessage = exports.promiseMap = void 0;
-const node_crypto_1 = require("node:crypto");
-const node_path_1 = require("node:path");
-const node_worker_threads_1 = require("node:worker_threads");
-const file_storage_worker_emulator_1 = require("./file-storage-worker-emulator");
-const utils_1 = require("../utils");
-let workerInstance;
-exports.promiseMap = new Map();
-function sendWorkerMessage(message) {
-    const id = (0, node_crypto_1.randomUUID)();
-    let promiseResolve;
-    const promise = new Promise((res) => {
-        promiseResolve = res;
-    });
-    exports.promiseMap.set(id, {
-        promise,
-        resolve: promiseResolve,
-    });
-    void workerInstance.postMessage({
-        ...message,
-        messageId: id,
-    });
-}
-exports.sendWorkerMessage = sendWorkerMessage;
-function initWorkerIfNeeded() {
-    if (workerInstance) {
-        return;
-    }
-    process.on('exit', () => {
-        void workerInstance.terminate();
-    });
-    const workerPath = (0, node_path_1.resolve)(__dirname, './file-storage-worker.js');
-    // vladfrangu: The worker is temporarily disabled due to node/v8 having internal bugs that sometimes cause hard crashes when the process exits.
-    // const exists = existsSync(workerPath);
-    const exists = false;
-    if (exists) {
-        workerInstance = new node_worker_threads_1.Worker(workerPath);
-        workerInstance.unref();
-        workerInstance.once('exit', (code) => {
-            utils_1.memoryStorageLog.debug(`File storage worker exited with code ${code}`);
-            initWorkerIfNeeded();
-        });
-        workerInstance.on('message', (message) => {
-            if (message.type !== 'ack') {
-                return;
-            }
-            exports.promiseMap.get(message.messageId)?.resolve();
-            exports.promiseMap.delete(message.messageId);
-        });
-    }
-    else {
-        workerInstance = new file_storage_worker_emulator_1.FileStorageWorkerEmulator();
-    }
-}
-exports.initWorkerIfNeeded = initWorkerIfNeeded;
-//# sourceMappingURL=instance.js.map

package/workers/instance.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"instance.js","sourceRoot":"","sources":["../../src/workers/instance.ts"],"names":[],"mappings":";;;AAAA,6CAAyC;AACzC,yCAAoC;AACpC,6DAA6C;AAE7C,iFAA2E;AAE3E,oCAA4C;AAE5C,IAAI,cAAkD,CAAC;AAE1C,QAAA,UAAU,GAGlB,IAAI,GAAG,EAAE,CAAC;AAEf,SAAgB,iBAAiB,CAAC,OAA8B;IAC5D,MAAM,EAAE,GAAG,IAAA,wBAAU,GAAE,CAAC;IAExB,IAAI,cAA0B,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QACtC,cAAc,GAAG,GAAG,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,kBAAU,CAAC,GAAG,CAAC,EAAE,EAAE;QACf,OAAO;QACP,OAAO,EAAE,cAAe;KAC3B,CAAC,CAAC;IAEH,KAAK,cAAc,CAAC,WAAW,CAAC;QAC5B,GAAG,OAAO;QACV,SAAS,EAAE,EAAE;KAChB,CAAC,CAAC;AACP,CAAC;AAjBD,8CAiBC;AAED,SAAgB,kBAAkB;IAC9B,IAAI,cAAc,EAAE;QAChB,OAAO;KACV;IAED,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACpB,KAAK,cAAc,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAA,mBAAO,EAAC,SAAS,EAAE,0BAA0B,CAAC,CAAC;IAClE,+IAA+I;IAC/I,yCAAyC;IACzC,MAAM,MAAM,GAAG,KAAK,CAAC;IAErB,IAAI,MAAM,EAAE;QACR,cAAc,GAAG,IAAI,4BAAM,CAAC,UAAU,CAAC,CAAC;QACxC,cAAc,CAAC,KAAK,EAAE,CAAC;QAEtB,cAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7C,wBAAgB,CAAC,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;YACvE,kBAAkB,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QAEF,cAAyB,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA2C,EAAE,EAAE;YACrF,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE;gBACxB,OAAO;aACV;YAED,kBAAU,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;YAC7C,kBAAU,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;KACN;SAAM;QACH,cAAc,GAAG,IAAI,wDAAyB,EAAE,CAAC;KACpD;AACL,CAAC;AAlCD,gDAkCC"}

package/workers/worker-utils.d.ts

@@ -1,6 +0,0 @@
-import type { WorkerReceivedMessage } from '../utils';
-export declare function handleMessage(message: WorkerReceivedMessage & {
-    messageId: string;
-}): Promise<void>;
-export declare function lockAndWrite(filePath: string, data: unknown, stringify?: boolean, retry?: number, timeout?: number): Promise<void>;
-//# sourceMappingURL=worker-utils.d.ts.map

package/workers/worker-utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"worker-utils.d.ts","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,qBAAqB,EAA+B,MAAM,UAAU,CAAC;AAInF,wBAAsB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,iBAezF;AAiBD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,UAAO,EAAE,KAAK,SAAK,EAAE,OAAO,SAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAmB7H"}

package/workers/worker-utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"worker-utils.js","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":";;;;AAAA,qCAAoC;AACpC,+CAA2D;AAC3D,yCAAoC;AACpC,mDAAkD;AAClD,6DAAiD;AAEjD,6DAA6B;AAC7B,uCAAqC;AACrC,qDAAuC;AAIvC,MAAM,SAAS,GAAG,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAExD,KAAK,UAAU,aAAa,CAAC,OAAsD;IACtF,QAAQ,OAAO,CAAC,MAAM,EAAE;QACpB,KAAK,iBAAiB;YAClB,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM;QACV;YACI,0GAA0G;YAC1G,6BAA6B;YAC7B,SAAS,CAAC,OAAO,CAAC,iCAAkC,OAAiC,CAAC,MAAM,EAAE,CAAC,CAAC;KACvG;IAED,gCAAU,EAAE,WAAW,CAAC;QACpB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;KAC/B,CAAC,CAAC;AACP,CAAC;AAfD,sCAeC;AAED,KAAK,UAAU,cAAc,CAAC,OAAoC;IAC9D,2HAA2H;IAC3H,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE;QACxB,OAAO;KACV;IAED,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,iCAAiC;IACjC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IACnD,MAAM,IAAA,oBAAU,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACzE,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAa,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE;IAC1G,IAAI;QACA,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAI,EAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,OAAO,CAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YACzC,IAAA,mBAAS,EAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAc,EAAE,CAAC,GAAG,EAAE,EAAE;gBACvF,IAAI,GAAG,EAAE;oBACL,MAAM,CAAC,GAAG,CAAC,CAAC;iBACf;qBAAM;oBACH,QAAQ,EAAE,CAAC;iBACd;YACL,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,EAAE,CAAC;KACnB;IAAC,OAAO,CAAM,EAAE;QACb,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE;YACnC,MAAM,IAAA,qBAAU,EAAC,OAAO,CAAC,CAAC;YAC1B,OAAO,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;SAC1E;KACJ;AACL,CAAC;AAnBD,oCAmBC"}