@crawlee/memory-storage 3.0.0-alpha.34

package/workers/file-storage-worker-emulator.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileStorageWorkerEmulator = void 0;
+const worker_utils_1 = require("./worker-utils");
+class FileStorageWorkerEmulator {
+    postMessage(value) {
+        void (0, worker_utils_1.handleMessage)(value);
+    }
+    terminate() { }
+}
+exports.FileStorageWorkerEmulator = FileStorageWorkerEmulator;
+//# sourceMappingURL=file-storage-worker-emulator.js.map

package/workers/file-storage-worker-emulator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-storage-worker-emulator.js","sourceRoot":"","sources":["../../src/workers/file-storage-worker-emulator.ts"],"names":[],"mappings":";;;AAAA,iDAA+C;AAE/C,MAAa,yBAAyB;IAClC,WAAW,CAAC,KAAU;QAClB,KAAK,IAAA,4BAAa,EAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IAED,SAAS,KAAU,CAAC;CACvB;AAND,8DAMC"}

package/workers/file-storage-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=file-storage-worker.d.ts.map

package/workers/file-storage-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-storage-worker.d.ts","sourceRoot":"","sources":["../../src/workers/file-storage-worker.ts"],"names":[],"mappings":""}

package/workers/file-storage-worker.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_worker_threads_1 = require("node:worker_threads");
+const worker_utils_1 = require("./worker-utils");
+if (node_worker_threads_1.isMainThread || !node_worker_threads_1.parentPort) {
+    throw new Error('This file should only be run in a worker thread!');
+}
+// Keep worker alive
+setInterval(() => {
+    node_worker_threads_1.parentPort.postMessage('ping');
+}, 30000);
+node_worker_threads_1.parentPort.on('message', async (message) => {
+    await (0, worker_utils_1.handleMessage)(message);
+});
+//# sourceMappingURL=file-storage-worker.js.map

package/workers/file-storage-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-storage-worker.js","sourceRoot":"","sources":["../../src/workers/file-storage-worker.ts"],"names":[],"mappings":";;AAAA,6DAA+D;AAE/D,iDAA+C;AAE/C,IAAI,kCAAY,IAAI,CAAC,gCAAU,EAAE;IAC7B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;CACvE;AAED,oBAAoB;AACpB,WAAW,CAAC,GAAG,EAAE;IACb,gCAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,EAAE,KAAM,CAAC,CAAC;AAEX,gCAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAA8B,EAAE,EAAE;IAC/D,MAAM,IAAA,4BAAa,EAAC,OAAO,CAAC,CAAC;AACjC,CAAC,CAAC,CAAC"}

package/workers/instance.d.ts

@@ -0,0 +1,4 @@
+import { WorkerReceivedMessage } from '../utils';
+export declare function sendWorkerMessage(message: WorkerReceivedMessage): void;
+export declare function initWorkerIfNeeded(): void;
+//# sourceMappingURL=instance.d.ts.map

package/workers/instance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instance.d.ts","sourceRoot":"","sources":["../../src/workers/instance.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoB,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAKnE,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,QAE/D;AAED,wBAAgB,kBAAkB,SAsBjC"}

package/workers/instance.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initWorkerIfNeeded = exports.sendWorkerMessage = void 0;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const node_worker_threads_1 = require("node:worker_threads");
+const utils_1 = require("../utils");
+const file_storage_worker_emulator_1 = require("./file-storage-worker-emulator");
+let workerInstance;
+function sendWorkerMessage(message) {
+    workerInstance.postMessage(message);
+}
+exports.sendWorkerMessage = sendWorkerMessage;
+function initWorkerIfNeeded() {
+    if (workerInstance) {
+        return;
+    }
+    process.on('exit', () => {
+        void workerInstance.terminate();
+    });
+    const workerPath = (0, node_path_1.resolve)(__dirname, './file-storage-worker.js');
+    const exists = (0, node_fs_1.existsSync)(workerPath);
+    if (exists) {
+        workerInstance = new node_worker_threads_1.Worker(workerPath);
+        workerInstance.once('exit', (code) => {
+            utils_1.memoryStorageLog.debug(`File storage worker exited with code ${code}`);
+            initWorkerIfNeeded();
+        });
+    }
+    else {
+        workerInstance = new file_storage_worker_emulator_1.FileStorageWorkerEmulator();
+    }
+}
+exports.initWorkerIfNeeded = initWorkerIfNeeded;
+//# sourceMappingURL=instance.js.map

package/workers/instance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"instance.js","sourceRoot":"","sources":["../../src/workers/instance.ts"],"names":[],"mappings":";;;AAAA,qCAAqC;AACrC,yCAAoC;AACpC,6DAA6C;AAC7C,oCAAmE;AACnE,iFAA2E;AAE3E,IAAI,cAAkD,CAAC;AAEvD,SAAgB,iBAAiB,CAAC,OAA8B;IAC5D,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAFD,8CAEC;AAED,SAAgB,kBAAkB;IAC9B,IAAI,cAAc,EAAE;QAChB,OAAO;KACV;IAED,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACpB,KAAK,cAAc,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAA,mBAAO,EAAC,SAAS,EAAE,0BAA0B,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,IAAA,oBAAU,EAAC,UAAU,CAAC,CAAC;IAEtC,IAAI,MAAM,EAAE;QACR,cAAc,GAAG,IAAI,4BAAM,CAAC,UAAU,CAAC,CAAC;QAEvC,cAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7C,wBAAgB,CAAC,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;YACvE,kBAAkB,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;KACN;SAAM;QACH,cAAc,GAAG,IAAI,wDAAyB,EAAE,CAAC;KACpD;AACL,CAAC;AAtBD,gDAsBC"}

package/workers/worker-utils.d.ts

@@ -0,0 +1,9 @@
+import { WorkerReceivedMessage } from '../utils';
+export interface WorkerDirectoryMap {
+    datasetsDirectory: string;
+    keyValueStoresDirectory: string;
+    requestQueuesDirectory: string;
+}
+export declare type EntityTypeToDirectoryMap = Map<'datasets' | 'keyValueStores' | 'requestQueues', string>;
+export declare function handleMessage(message: WorkerReceivedMessage): Promise<void>;
+//# sourceMappingURL=worker-utils.d.ts.map

package/workers/worker-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-utils.d.ts","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAA2D,MAAM,UAAU,CAAC;AAI1G,MAAM,WAAW,kBAAkB;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,sBAAsB,EAAE,MAAM,CAAC;CAClC;AAED,oBAAY,wBAAwB,GAAG,GAAG,CAAC,UAAU,GAAG,gBAAgB,GAAG,eAAe,EAAE,MAAM,CAAC,CAAC;AAEpG,wBAAsB,aAAa,CAAC,OAAO,EAAE,qBAAqB,iBAajE"}

package/workers/worker-utils.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleMessage = void 0;
+const tslib_1 = require("tslib");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const fs_extra_1 = require("fs-extra");
+const promises_1 = require("node:fs/promises");
+const node_path_1 = require("node:path");
+const workerLog = log_1.default.child({ prefix: 'MemoryStorageWorker' });
+async function handleMessage(message) {
+    switch (message.action) {
+        case 'update-metadata':
+            await updateMetadata(message);
+            break;
+        case 'update-entries':
+            await updateItems(message);
+            break;
+        default:
+            // @ts-expect-error We're keeping this to make eslint happy + in the event we add a new action without adding checks for it
+            // we should be aware of them
+            workerLog.warning(`Unknown worker message action ${message.action}`);
+    }
+}
+exports.handleMessage = handleMessage;
+async function updateMetadata(message) {
+    workerLog.info(`Updating metadata for ${message.entityType} with id ${message.id}`);
+    // Ensure the directory for the entity exists
+    const dir = message.entityDirectory;
+    await (0, fs_extra_1.ensureDir)(dir);
+    // Write the metadata to the file
+    const filePath = (0, node_path_1.resolve)(dir, '__metadata__.json');
+    await (0, promises_1.writeFile)(filePath, JSON.stringify(message.data, null, '\t'));
+}
+async function updateItems(message) {
+    workerLog.info(`Updating entries for ${message.entityType} with id ${message.id}`);
+    // Ensure the directory for the entity exists
+    const dir = message.entityDirectory;
+    await (0, fs_extra_1.ensureDir)(dir);
+    switch (message.entityType) {
+        case 'requestQueues': {
+            // Write the entries to the file
+            const filePath = (0, node_path_1.resolve)(dir, 'entries.json');
+            await (0, promises_1.writeFile)(filePath, JSON.stringify(message.data, null, '\t'));
+            break;
+        }
+        case 'datasets': {
+            // Save all the new items to the disk
+            for (const [idx, data] of message.data) {
+                await (0, promises_1.writeFile)((0, node_path_1.resolve)(dir, `${idx}.json`), JSON.stringify(data, null, '\t'));
+            }
+            break;
+        }
+        case 'keyValueStores': {
+            // Create files for the record
+            const { action, record } = message.data;
+            const itemPath = (0, node_path_1.resolve)(dir, `${record.key}.${record.extension}`);
+            switch (action) {
+                case 'delete':
+                    await (0, promises_1.rm)(itemPath, { force: true, recursive: true });
+                    break;
+                case 'set': {
+                    await (0, promises_1.rm)(itemPath, { force: true, recursive: true });
+                    const metadataPath = (0, node_path_1.resolve)(dir, `${record.key}.__metadata__.json`);
+                    await (0, promises_1.writeFile)(metadataPath, JSON.stringify({
+                        key: record.key,
+                        contentType: record.contentType ?? 'unknown/no content type',
+                        extension: record.extension,
+                    }, null, '\t'));
+                    await (0, promises_1.writeFile)(itemPath, record.value);
+                    break;
+                }
+                default:
+            }
+            break;
+        }
+        default:
+    }
+}
+//# sourceMappingURL=worker-utils.js.map

package/workers/worker-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-utils.js","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":";;;;AAAA,6DAAoC;AACpC,uCAAqC;AACrC,+CAAiD;AACjD,yCAAoC;AAGpC,MAAM,SAAS,GAAG,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAU/D,KAAK,UAAU,aAAa,CAAC,OAA8B;IAC9D,QAAQ,OAAO,CAAC,MAAM,EAAE;QACpB,KAAK,iBAAiB;YAClB,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM;QACV,KAAK,gBAAgB;YACjB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3B,MAAM;QACV;YACI,2HAA2H;YAC3H,6BAA6B;YAC7B,SAAS,CAAC,OAAO,CAAC,iCAAiC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;KAC5E;AACL,CAAC;AAbD,sCAaC;AAED,KAAK,UAAU,cAAc,CAAC,OAAoC;IAC9D,SAAS,CAAC,IAAI,CAAC,yBAAyB,OAAO,CAAC,UAAU,YAAY,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IAEpF,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,iCAAiC;IACjC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IACnD,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,OAAmC;IAC1D,SAAS,CAAC,IAAI,CAAC,wBAAwB,OAAO,CAAC,UAAU,YAAY,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IAEnF,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,QAAQ,OAAO,CAAC,UAAU,EAAE;QACxB,KAAK,eAAe,CAAC,CAAC;YAClB,gCAAgC;YAChC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,cAAc,CAAC,CAAC;YAC9C,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACpE,MAAM;SACT;QACD,KAAK,UAAU,CAAC,CAAC;YACb,qCAAqC;YACrC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE;gBACpC,MAAM,IAAA,oBAAS,EACX,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CACnC,CAAC;aACL;YAED,MAAM;SACT;QACD,KAAK,gBAAgB,CAAC,CAAC;YACnB,8BAA8B;YAC9B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAExC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YAEnE,QAAQ,MAAM,EAAE;gBACZ,KAAK,QAAQ;oBACT,MAAM,IAAA,aAAE,EAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACrD,MAAM;gBACV,KAAK,KAAK,CAAC,CAAC;oBACR,MAAM,IAAA,aAAE,EAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAErD,MAAM,YAAY,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,oBAAoB,CAAC,CAAC;oBAErE,MAAM,IAAA,oBAAS,EACX,YAAY,EACZ,IAAI,CAAC,SAAS,CACV;wBACI,GAAG,EAAE,MAAM,CAAC,GAAG;wBACf,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,yBAAyB;wBAC5D,SAAS,EAAE,MAAM,CAAC,SAAS;qBAC9B,EACD,IAAI,EACJ,IAAI,CACP,CACJ,CAAC;oBAEF,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;oBAExC,MAAM;iBACT;gBACD,QAAQ;aACX;YAED,MAAM;SACT;QACD,QAAQ;KACX;AACL,CAAC"}