@crawlee/memory-storage 3.1.5-beta.9 → 3.2.0

package/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;;AAAA,6DAAoC;AAEpC,qDAAyC;AACzC,6CAAyC;AACzC,qCAA6C;AAI7C;;;GAGG;AACH,SAAgB,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QAChE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC/C,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;SACtF;KACJ;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AARD,oDAQC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,0BAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,0BAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAPD,oDAOC;AAAA,CAAC;AAEF,SAAgB,QAAQ,CAAC,KAAc;IACnC,IAAI;QACA,cAAC,CAAC,KAAK,CACH,cAAC,CAAC,UAAU,EAAE,EACd,cAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EACvB,cAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CACrB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEf,OAAO,IAAI,CAAC;KACf;IAAC,MAAM;QACJ,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAZD,4BAYC;AAED,SAAgB,QAAQ,CAAC,KAAc;IACnC,IAAI;QACA,cAAC,CAAC,MAAM,CAAC;YACL,EAAE,EAAE,cAAC,CAAC,GAAG;YACT,IAAI,EAAE,cAAC,CAAC,GAAG;SACd,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE5B,OAAO,IAAI,CAAC;KACf;IAAC,MAAM;QACJ,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAXD,4BAWC;AAEY,QAAA,gBAAgB,GAAG,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;;AAAA,6DAAoC;AAEpC,qDAAyC;AACzC,6CAAyC;AACzC,qCAA6C;AAE7C;;;GAGG;AACH,SAAgB,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QAChE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC/C,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;SACtF;KACJ;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AARD,oDAQC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,0BAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,0BAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAPD,oDAOC;AAAA,CAAC;AAEF,SAAgB,QAAQ,CAAC,KAAc;IACnC,IAAI;QACA,cAAC,CAAC,KAAK,CACH,cAAC,CAAC,UAAU,EAAE,EACd,cAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EACvB,cAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CACrB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEf,OAAO,IAAI,CAAC;KACf;IAAC,MAAM;QACJ,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAZD,4BAYC;AAED,SAAgB,QAAQ,CAAC,KAAc;IACnC,IAAI;QACA,cAAC,CAAC,MAAM,CAAC;YACL,EAAE,EAAE,cAAC,CAAC,GAAG;YACT,IAAI,EAAE,cAAC,CAAC,GAAG;SACd,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE5B,OAAO,IAAI,CAAC;KACf;IAAC,MAAM;QACJ,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAXD,4BAWC;AAEY,QAAA,gBAAgB,GAAG,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC"}

package/workers/worker-utils.d.ts

@@ -2,4 +2,5 @@ import type { WorkerReceivedMessage } from '../utils';
 export declare function handleMessage(message: WorkerReceivedMessage & {
     messageId: string;
 }): Promise<void>;
+export declare function lockAndWrite(filePath: string, data: unknown, stringify?: boolean, retry?: number, timeout?: number): Promise<void>;
 //# sourceMappingURL=worker-utils.d.ts.map

package/workers/worker-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-utils.d.ts","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAA4B,qBAAqB,EAA2D,MAAM,UAAU,CAAC;AAIzI,wBAAsB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,iBAqBzF"}
+{"version":3,"file":"worker-utils.d.ts","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,qBAAqB,EAA+B,MAAM,UAAU,CAAC;AAInF,wBAAsB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,iBAezF;AAiBD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,UAAO,EAAE,KAAK,SAAK,EAAE,OAAO,SAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAW7H"}

package/workers/worker-utils.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.handleMessage = void 0;
+exports.lockAndWrite = exports.handleMessage = void 0;
 const tslib_1 = require("tslib");
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const fs_extra_1 = require("fs-extra");
@@ -15,12 +15,6 @@ async function handleMessage(message) {
         case 'update-metadata':
             await updateMetadata(message);
             break;
-        case 'update-entries':
-            await updateItems(message);
-            break;
-        case 'delete-entry':
-            await deleteEntry(message);
-            break;
         default:
             // We're keeping this to make eslint happy + in the event we add a new action without adding checks for it
             // we should be aware of them
@@ -57,71 +51,5 @@ async function lockAndWrite(filePath, data, stringify = true, retry = 10, timeou
         }
     }
 }
-async function updateItems(message) {
-    // Skip writing files to the disk if the client has the option set to false
-    if (!message.persistStorage) {
-        return;
-    }
-    // Ensure the directory for the entity exists
-    const dir = message.entityDirectory;
-    await (0, fs_extra_1.ensureDir)(dir);
-    switch (message.entityType) {
-        case 'requestQueues': {
-            // Write the entry to the file
-            const filePath = (0, node_path_1.resolve)(dir, `${message.data.id}.json`);
-            await lockAndWrite(filePath, message.data);
-            break;
-        }
-        case 'datasets': {
-            // Save all the new items to the disk
-            for (const [idx, data] of message.data) {
-                const filePath = (0, node_path_1.resolve)(dir, `${idx}.json`);
-                await lockAndWrite(filePath, data);
-            }
-            break;
-        }
-        case 'keyValueStores': {
-            // Create files for the record
-            const { action, record } = message.data;
-            const itemPath = (0, node_path_1.resolve)(dir, `${record.key}.${record.extension}`);
-            const itemMetadataPath = (0, node_path_1.resolve)(dir, `${record.key}.__metadata__.json`);
-            switch (action) {
-                case 'delete':
-                    await (0, promises_1.rm)(itemPath, { force: true });
-                    await (0, promises_1.rm)(itemMetadataPath, { force: true });
-                    break;
-                case 'set': {
-                    await (0, promises_1.rm)(itemPath, { force: true });
-                    await (0, promises_1.rm)(itemMetadataPath, { force: true });
-                    if (message.writeMetadata) {
-                        await lockAndWrite(itemMetadataPath, {
-                            key: record.key,
-                            contentType: record.contentType ?? 'unknown/no content type',
-                            extension: record.extension,
-                        });
-                    }
-                    await lockAndWrite(itemPath, record.value, false);
-                    break;
-                }
-                default:
-            }
-            break;
-        }
-        default:
-    }
-}
-async function deleteEntry(message) {
-    // Ensure the directory for the entity exists
-    const dir = message.entityDirectory;
-    await (0, fs_extra_1.ensureDir)(dir);
-    switch (message.entityType) {
-        case 'requestQueues': {
-            // Write the entry to the file
-            const filePath = (0, node_path_1.resolve)(dir, `${message.data.id}.json`);
-            await (0, promises_1.rm)(filePath, { force: true });
-            break;
-        }
-        default:
-    }
-}
+exports.lockAndWrite = lockAndWrite;
 //# sourceMappingURL=worker-utils.js.map

package/workers/worker-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-utils.js","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,uCAAqC;AACrC,+CAAiD;AACjD,mDAAkD;AAClD,yCAAoC;AACpC,6DAAiD;AACjD,qDAAuC;AAGvC,MAAM,SAAS,GAAG,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAExD,KAAK,UAAU,aAAa,CAAC,OAAsD;IACtF,QAAQ,OAAO,CAAC,MAAM,EAAE;QACpB,KAAK,iBAAiB;YAClB,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM;QACV,KAAK,gBAAgB;YACjB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3B,MAAM;QACV,KAAK,cAAc;YACf,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3B,MAAM;QACV;YACI,0GAA0G;YAC1G,6BAA6B;YAC7B,SAAS,CAAC,OAAO,CAAC,iCAAkC,OAAiC,CAAC,MAAM,EAAE,CAAC,CAAC;KACvG;IAED,gCAAU,EAAE,WAAW,CAAC;QACpB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;KAC/B,CAAC,CAAC;AACP,CAAC;AArBD,sCAqBC;AAED,KAAK,UAAU,cAAc,CAAC,OAAoC;IAC9D,2HAA2H;IAC3H,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE;QACxB,OAAO;KACV;IAED,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,iCAAiC;IACjC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IACnD,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAa,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE;IACnG,IAAI;QACA,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAI,EAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC,CAAC;QACzF,MAAM,OAAO,EAAE,CAAC;KACnB;IAAC,OAAO,CAAM,EAAE;QACb,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE;YACnC,MAAM,IAAA,qBAAU,EAAC,OAAO,CAAC,CAAC;YAC1B,OAAO,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;SAC1E;KACJ;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,OAAmC;IAC1D,2EAA2E;IAC3E,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;QACzB,OAAO;KACV;IAED,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,QAAQ,OAAO,CAAC,UAAU,EAAE;QACxB,KAAK,eAAe,CAAC,CAAC;YAClB,8BAA8B;YAC9B,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YACzD,MAAM,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM;SACT;QACD,KAAK,UAAU,CAAC,CAAC;YACb,qCAAqC;YACrC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE;gBACpC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;gBAC7C,MAAM,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;aACtC;YAED,MAAM;SACT;QACD,KAAK,gBAAgB,CAAC,CAAC;YACnB,8BAA8B;YAC9B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAExC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YACnE,MAAM,gBAAgB,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,oBAAoB,CAAC,CAAC;YAEzE,QAAQ,MAAM,EAAE;gBACZ,KAAK,QAAQ;oBACT,MAAM,IAAA,aAAE,EAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpC,MAAM,IAAA,aAAE,EAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC5C,MAAM;gBACV,KAAK,KAAK,CAAC,CAAC;oBACR,MAAM,IAAA,aAAE,EAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpC,MAAM,IAAA,aAAE,EAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAE5C,IAAI,OAAO,CAAC,aAAa,EAAE;wBACvB,MAAM,YAAY,CACd,gBAAgB,EAChB;4BACI,GAAG,EAAE,MAAM,CAAC,GAAG;4BACf,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,yBAAyB;4BAC5D,SAAS,EAAE,MAAM,CAAC,SAAS;yBAC9B,CACJ,CAAC;qBACL;oBAED,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBAElD,MAAM;iBACT;gBACD,QAAQ;aACX;YAED,MAAM;SACT;QACD,QAAQ;KACX;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,OAAiC;IACxD,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,QAAQ,OAAO,CAAC,UAAU,EAAE;QACxB,KAAK,eAAe,CAAC,CAAC;YAClB,8BAA8B;YAC9B,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAEzD,MAAM,IAAA,aAAE,EAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAEpC,MAAM;SACT;QACD,QAAQ;KACX;AACL,CAAC"}
+{"version":3,"file":"worker-utils.js","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,uCAAqC;AACrC,+CAA6C;AAC7C,mDAAkD;AAClD,yCAAoC;AACpC,6DAAiD;AACjD,qDAAuC;AAGvC,MAAM,SAAS,GAAG,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAExD,KAAK,UAAU,aAAa,CAAC,OAAsD;IACtF,QAAQ,OAAO,CAAC,MAAM,EAAE;QACpB,KAAK,iBAAiB;YAClB,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM;QACV;YACI,0GAA0G;YAC1G,6BAA6B;YAC7B,SAAS,CAAC,OAAO,CAAC,iCAAkC,OAAiC,CAAC,MAAM,EAAE,CAAC,CAAC;KACvG;IAED,gCAAU,EAAE,WAAW,CAAC;QACpB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;KAC/B,CAAC,CAAC;AACP,CAAC;AAfD,sCAeC;AAED,KAAK,UAAU,cAAc,CAAC,OAAoC;IAC9D,2HAA2H;IAC3H,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE;QACxB,OAAO;KACV;IAED,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,iCAAiC;IACjC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IACnD,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACxE,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAa,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE;IAC1G,IAAI;QACA,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAI,EAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC,CAAC;QACzF,MAAM,OAAO,EAAE,CAAC;KACnB;IAAC,OAAO,CAAM,EAAE;QACb,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE;YACnC,MAAM,IAAA,qBAAU,EAAC,OAAO,CAAC,CAAC;YAC1B,OAAO,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;SAC1E;KACJ;AACL,CAAC;AAXD,oCAWC"}