@crawlee/memory-storage 3.1.5-beta.2 → 3.1.5-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cache-helpers.d.ts.map +1 -1
- package/cache-helpers.js +51 -31
- package/cache-helpers.js.map +1 -1
- package/fs/common.d.ts +6 -0
- package/fs/common.d.ts.map +1 -0
- package/fs/common.js +3 -0
- package/fs/common.js.map +1 -0
- package/fs/dataset/fs.d.ts +10 -0
- package/fs/dataset/fs.d.ts.map +1 -0
- package/fs/dataset/fs.js +30 -0
- package/fs/dataset/fs.js.map +1 -0
- package/fs/dataset/index.d.ts +10 -0
- package/fs/dataset/index.d.ts.map +1 -0
- package/fs/dataset/index.js +13 -0
- package/fs/dataset/index.js.map +1 -0
- package/fs/dataset/memory.d.ts +8 -0
- package/fs/dataset/memory.d.ts.map +1 -0
- package/fs/dataset/memory.js +24 -0
- package/fs/dataset/memory.js.map +1 -0
- package/fs/key-value-store/fs.d.ts +15 -0
- package/fs/key-value-store/fs.d.ts.map +1 -0
- package/fs/key-value-store/fs.js +87 -0
- package/fs/key-value-store/fs.js.map +1 -0
- package/fs/key-value-store/index.d.ts +9 -0
- package/fs/key-value-store/index.d.ts.map +1 -0
- package/fs/key-value-store/index.js +13 -0
- package/fs/key-value-store/index.js.map +1 -0
- package/fs/key-value-store/memory.d.ts +9 -0
- package/fs/key-value-store/memory.d.ts.map +1 -0
- package/fs/key-value-store/memory.js +24 -0
- package/fs/key-value-store/memory.js.map +1 -0
- package/fs/request-queue/fs.d.ts +11 -0
- package/fs/request-queue/fs.d.ts.map +1 -0
- package/fs/request-queue/fs.js +30 -0
- package/fs/request-queue/fs.js.map +1 -0
- package/fs/request-queue/index.d.ts +9 -0
- package/fs/request-queue/index.d.ts.map +1 -0
- package/fs/request-queue/index.js +13 -0
- package/fs/request-queue/index.js.map +1 -0
- package/fs/request-queue/memory.d.ts +9 -0
- package/fs/request-queue/memory.d.ts.map +1 -0
- package/fs/request-queue/memory.js +24 -0
- package/fs/request-queue/memory.js.map +1 -0
- package/package.json +8 -7
- package/resource-clients/dataset.d.ts.map +1 -1
- package/resource-clients/dataset.js +9 -12
- package/resource-clients/dataset.js.map +1 -1
- package/resource-clients/key-value-store.d.ts.map +1 -1
- package/resource-clients/key-value-store.js +15 -29
- package/resource-clients/key-value-store.js.map +1 -1
- package/resource-clients/request-queue.d.ts +0 -1
- package/resource-clients/request-queue.d.ts.map +1 -1
- package/resource-clients/request-queue.js +45 -44
- package/resource-clients/request-queue.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/utils.d.ts +3 -31
- package/utils.d.ts.map +1 -1
- package/utils.js.map +1 -1
- package/workers/worker-utils.d.ts +1 -0
- package/workers/worker-utils.d.ts.map +1 -1
- package/workers/worker-utils.js +2 -74
- package/workers/worker-utils.js.map +1 -1
package/utils.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import type * as storage from '@crawlee/types';
|
|
2
|
-
import type { InternalKeyRecord } from './resource-clients/key-value-store';
|
|
3
|
-
import type { InternalRequest } from './resource-clients/request-queue';
|
|
4
2
|
/**
|
|
5
3
|
* Removes all properties with a null value
|
|
6
4
|
* from the provided object.
|
|
@@ -18,11 +16,9 @@ export interface WorkerData {
|
|
|
18
16
|
keyValueStoresDirectory: string;
|
|
19
17
|
requestQueuesDirectory: string;
|
|
20
18
|
}
|
|
21
|
-
export
|
|
22
|
-
export
|
|
23
|
-
|
|
24
|
-
export declare type WorkerDeleteEntryMessage = EntryDelete<'requestQueues'>;
|
|
25
|
-
declare type EntityType = 'datasets' | 'keyValueStores' | 'requestQueues';
|
|
19
|
+
export type WorkerReceivedMessage = WorkerUpdateMetadataMessage;
|
|
20
|
+
export type WorkerUpdateMetadataMessage = MetadataUpdate<'datasets', storage.DatasetInfo> | MetadataUpdate<'keyValueStores', storage.KeyValueStoreInfo> | MetadataUpdate<'requestQueues', storage.RequestQueueInfo>;
|
|
21
|
+
type EntityType = 'datasets' | 'keyValueStores' | 'requestQueues';
|
|
26
22
|
interface MetadataUpdate<Type extends EntityType, DataType> {
|
|
27
23
|
entityType: Type;
|
|
28
24
|
id: string;
|
|
@@ -32,29 +28,5 @@ interface MetadataUpdate<Type extends EntityType, DataType> {
|
|
|
32
28
|
writeMetadata: boolean;
|
|
33
29
|
persistStorage: boolean;
|
|
34
30
|
}
|
|
35
|
-
interface EntriesUpdate<Type extends EntityType, DataType> {
|
|
36
|
-
entityType: Type;
|
|
37
|
-
id: string;
|
|
38
|
-
action: 'update-entries';
|
|
39
|
-
entityDirectory: string;
|
|
40
|
-
data: DataType;
|
|
41
|
-
writeMetadata: boolean;
|
|
42
|
-
persistStorage: boolean;
|
|
43
|
-
}
|
|
44
|
-
interface EntryDelete<Type extends EntityType> {
|
|
45
|
-
entityType: Type;
|
|
46
|
-
id: string;
|
|
47
|
-
action: 'delete-entry';
|
|
48
|
-
entityDirectory: string;
|
|
49
|
-
writeMetadata: boolean;
|
|
50
|
-
persistStorage: boolean;
|
|
51
|
-
data: {
|
|
52
|
-
id: string;
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
interface KeyValueStoreItemData {
|
|
56
|
-
action: 'set' | 'delete';
|
|
57
|
-
record: InternalKeyRecord;
|
|
58
|
-
}
|
|
59
31
|
export {};
|
|
60
32
|
//# sourceMappingURL=utils.d.ts.map
|
package/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAK/C;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,CAQpD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAO9D;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAYhD;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAWhD;AAED,eAAO,MAAM,gBAAgB,0BAAgD,CAAC;AAE9E,MAAM,WAAW,UAAU;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,sBAAsB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,MAAM,qBAAqB,GAAG,2BAA2B,CAAC;AAEhE,MAAM,MAAM,2BAA2B,GACjC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC,GAC/C,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAC3D,cAAc,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAEhE,KAAK,UAAU,GAAG,UAAU,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAElE,UAAU,cAAc,CAAC,IAAI,SAAS,UAAU,EAAE,QAAQ;IACtD,UAAU,EAAE,IAAI,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,iBAAiB,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,QAAQ,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;CAC3B"}
|
package/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;;AAAA,6DAAoC;AAEpC,qDAAyC;AACzC,6CAAyC;AACzC,qCAA6C;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;;AAAA,6DAAoC;AAEpC,qDAAyC;AACzC,6CAAyC;AACzC,qCAA6C;AAE7C;;;GAGG;AACH,SAAgB,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QAChE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC/C,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;SACtF;KACJ;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AARD,oDAQC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,0BAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,0BAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAPD,oDAOC;AAAA,CAAC;AAEF,SAAgB,QAAQ,CAAC,KAAc;IACnC,IAAI;QACA,cAAC,CAAC,KAAK,CACH,cAAC,CAAC,UAAU,EAAE,EACd,cAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EACvB,cAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CACrB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEf,OAAO,IAAI,CAAC;KACf;IAAC,MAAM;QACJ,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAZD,4BAYC;AAED,SAAgB,QAAQ,CAAC,KAAc;IACnC,IAAI;QACA,cAAC,CAAC,MAAM,CAAC;YACL,EAAE,EAAE,cAAC,CAAC,GAAG;YACT,IAAI,EAAE,cAAC,CAAC,GAAG;SACd,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE5B,OAAO,IAAI,CAAC;KACf;IAAC,MAAM;QACJ,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAXD,4BAWC;AAEY,QAAA,gBAAgB,GAAG,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC"}
|
|
@@ -2,4 +2,5 @@ import type { WorkerReceivedMessage } from '../utils';
|
|
|
2
2
|
export declare function handleMessage(message: WorkerReceivedMessage & {
|
|
3
3
|
messageId: string;
|
|
4
4
|
}): Promise<void>;
|
|
5
|
+
export declare function lockAndWrite(filePath: string, data: unknown, stringify?: boolean, retry?: number, timeout?: number): Promise<void>;
|
|
5
6
|
//# sourceMappingURL=worker-utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-utils.d.ts","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"worker-utils.d.ts","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,qBAAqB,EAA+B,MAAM,UAAU,CAAC;AAInF,wBAAsB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,iBAezF;AAiBD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,UAAO,EAAE,KAAK,SAAK,EAAE,OAAO,SAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAW7H"}
|
package/workers/worker-utils.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.handleMessage = void 0;
|
|
3
|
+
exports.lockAndWrite = exports.handleMessage = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
6
6
|
const fs_extra_1 = require("fs-extra");
|
|
@@ -15,12 +15,6 @@ async function handleMessage(message) {
|
|
|
15
15
|
case 'update-metadata':
|
|
16
16
|
await updateMetadata(message);
|
|
17
17
|
break;
|
|
18
|
-
case 'update-entries':
|
|
19
|
-
await updateItems(message);
|
|
20
|
-
break;
|
|
21
|
-
case 'delete-entry':
|
|
22
|
-
await deleteEntry(message);
|
|
23
|
-
break;
|
|
24
18
|
default:
|
|
25
19
|
// We're keeping this to make eslint happy + in the event we add a new action without adding checks for it
|
|
26
20
|
// we should be aware of them
|
|
@@ -57,71 +51,5 @@ async function lockAndWrite(filePath, data, stringify = true, retry = 10, timeou
|
|
|
57
51
|
}
|
|
58
52
|
}
|
|
59
53
|
}
|
|
60
|
-
|
|
61
|
-
// Skip writing files to the disk if the client has the option set to false
|
|
62
|
-
if (!message.persistStorage) {
|
|
63
|
-
return;
|
|
64
|
-
}
|
|
65
|
-
// Ensure the directory for the entity exists
|
|
66
|
-
const dir = message.entityDirectory;
|
|
67
|
-
await (0, fs_extra_1.ensureDir)(dir);
|
|
68
|
-
switch (message.entityType) {
|
|
69
|
-
case 'requestQueues': {
|
|
70
|
-
// Write the entry to the file
|
|
71
|
-
const filePath = (0, node_path_1.resolve)(dir, `${message.data.id}.json`);
|
|
72
|
-
await lockAndWrite(filePath, message.data);
|
|
73
|
-
break;
|
|
74
|
-
}
|
|
75
|
-
case 'datasets': {
|
|
76
|
-
// Save all the new items to the disk
|
|
77
|
-
for (const [idx, data] of message.data) {
|
|
78
|
-
const filePath = (0, node_path_1.resolve)(dir, `${idx}.json`);
|
|
79
|
-
await lockAndWrite(filePath, data);
|
|
80
|
-
}
|
|
81
|
-
break;
|
|
82
|
-
}
|
|
83
|
-
case 'keyValueStores': {
|
|
84
|
-
// Create files for the record
|
|
85
|
-
const { action, record } = message.data;
|
|
86
|
-
const itemPath = (0, node_path_1.resolve)(dir, `${record.key}.${record.extension}`);
|
|
87
|
-
const itemMetadataPath = (0, node_path_1.resolve)(dir, `${record.key}.__metadata__.json`);
|
|
88
|
-
switch (action) {
|
|
89
|
-
case 'delete':
|
|
90
|
-
await (0, promises_1.rm)(itemPath, { force: true });
|
|
91
|
-
await (0, promises_1.rm)(itemMetadataPath, { force: true });
|
|
92
|
-
break;
|
|
93
|
-
case 'set': {
|
|
94
|
-
await (0, promises_1.rm)(itemPath, { force: true });
|
|
95
|
-
await (0, promises_1.rm)(itemMetadataPath, { force: true });
|
|
96
|
-
if (message.writeMetadata) {
|
|
97
|
-
await lockAndWrite(itemMetadataPath, {
|
|
98
|
-
key: record.key,
|
|
99
|
-
contentType: record.contentType ?? 'unknown/no content type',
|
|
100
|
-
extension: record.extension,
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
await lockAndWrite(itemPath, record.value, false);
|
|
104
|
-
break;
|
|
105
|
-
}
|
|
106
|
-
default:
|
|
107
|
-
}
|
|
108
|
-
break;
|
|
109
|
-
}
|
|
110
|
-
default:
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
async function deleteEntry(message) {
|
|
114
|
-
// Ensure the directory for the entity exists
|
|
115
|
-
const dir = message.entityDirectory;
|
|
116
|
-
await (0, fs_extra_1.ensureDir)(dir);
|
|
117
|
-
switch (message.entityType) {
|
|
118
|
-
case 'requestQueues': {
|
|
119
|
-
// Write the entry to the file
|
|
120
|
-
const filePath = (0, node_path_1.resolve)(dir, `${message.data.id}.json`);
|
|
121
|
-
await (0, promises_1.rm)(filePath, { force: true });
|
|
122
|
-
break;
|
|
123
|
-
}
|
|
124
|
-
default:
|
|
125
|
-
}
|
|
126
|
-
}
|
|
54
|
+
exports.lockAndWrite = lockAndWrite;
|
|
127
55
|
//# sourceMappingURL=worker-utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-utils.js","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,uCAAqC;AACrC,+
|
|
1
|
+
{"version":3,"file":"worker-utils.js","sourceRoot":"","sources":["../../src/workers/worker-utils.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,uCAAqC;AACrC,+CAA6C;AAC7C,mDAAkD;AAClD,yCAAoC;AACpC,6DAAiD;AACjD,qDAAuC;AAGvC,MAAM,SAAS,GAAG,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAExD,KAAK,UAAU,aAAa,CAAC,OAAsD;IACtF,QAAQ,OAAO,CAAC,MAAM,EAAE;QACpB,KAAK,iBAAiB;YAClB,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM;QACV;YACI,0GAA0G;YAC1G,6BAA6B;YAC7B,SAAS,CAAC,OAAO,CAAC,iCAAkC,OAAiC,CAAC,MAAM,EAAE,CAAC,CAAC;KACvG;IAED,gCAAU,EAAE,WAAW,CAAC;QACpB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;KAC/B,CAAC,CAAC;AACP,CAAC;AAfD,sCAeC;AAED,KAAK,UAAU,cAAc,CAAC,OAAoC;IAC9D,2HAA2H;IAC3H,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE;QACxB,OAAO;KACV;IAED,6CAA6C;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC;IACpC,MAAM,IAAA,oBAAS,EAAC,GAAG,CAAC,CAAC;IAErB,iCAAiC;IACjC,MAAM,QAAQ,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IACnD,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACxE,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAa,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE;IAC1G,IAAI;QACA,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAI,EAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC,CAAC;QACzF,MAAM,OAAO,EAAE,CAAC;KACnB;IAAC,OAAO,CAAM,EAAE;QACb,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE;YACnC,MAAM,IAAA,qBAAU,EAAC,OAAO,CAAC,CAAC;YAC1B,OAAO,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;SAC1E;KACJ;AACL,CAAC;AAXD,oCAWC"}
|