@crawlee/memory-storage 4.0.0-beta.6 → 4.0.0-beta.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/background-handler/fs-utils.d.ts +2 -1
- package/background-handler/fs-utils.d.ts.map +1 -1
- package/background-handler/fs-utils.js +2 -4
- package/background-handler/fs-utils.js.map +1 -1
- package/background-handler/index.d.ts +2 -1
- package/background-handler/index.d.ts.map +1 -1
- package/background-handler/index.js +4 -4
- package/background-handler/index.js.map +1 -1
- package/cache-helpers.d.ts.map +1 -1
- package/cache-helpers.js +9 -9
- package/cache-helpers.js.map +1 -1
- package/fs/key-value-store/fs.d.ts +1 -0
- package/fs/key-value-store/fs.d.ts.map +1 -1
- package/fs/key-value-store/fs.js +7 -3
- package/fs/key-value-store/fs.js.map +1 -1
- package/fs/key-value-store/index.d.ts +2 -0
- package/fs/key-value-store/index.d.ts.map +1 -1
- package/fs/key-value-store/index.js.map +1 -1
- package/memory-storage.d.ts +21 -10
- package/memory-storage.d.ts.map +1 -1
- package/memory-storage.js +127 -39
- package/memory-storage.js.map +1 -1
- package/package.json +5 -5
- package/resource-clients/dataset.d.ts +7 -16
- package/resource-clients/dataset.d.ts.map +1 -1
- package/resource-clients/dataset.js +41 -101
- package/resource-clients/dataset.js.map +1 -1
- package/resource-clients/key-value-store.d.ts +16 -8
- package/resource-clients/key-value-store.d.ts.map +1 -1
- package/resource-clients/key-value-store.js +62 -126
- package/resource-clients/key-value-store.js.map +1 -1
- package/resource-clients/request-queue.d.ts +3 -7
- package/resource-clients/request-queue.d.ts.map +1 -1
- package/resource-clients/request-queue.js +67 -129
- package/resource-clients/request-queue.js.map +1 -1
- package/utils.d.ts +0 -2
- package/utils.d.ts.map +1 -1
- package/utils.js +0 -2
- package/utils.js.map +1 -1
- package/resource-clients/dataset-collection.d.ts +0 -14
- package/resource-clients/dataset-collection.d.ts.map +0 -1
- package/resource-clients/dataset-collection.js +0 -49
- package/resource-clients/dataset-collection.js.map +0 -1
- package/resource-clients/key-value-store-collection.d.ts +0 -14
- package/resource-clients/key-value-store-collection.d.ts.map +0 -1
- package/resource-clients/key-value-store-collection.js +0 -53
- package/resource-clients/key-value-store-collection.js.map +0 -1
- package/resource-clients/request-queue-collection.d.ts +0 -14
- package/resource-clients/request-queue-collection.d.ts.map +0 -1
- package/resource-clients/request-queue-collection.js +0 -53
- package/resource-clients/request-queue-collection.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/memory-storage.js
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
/* eslint-disable import/no-duplicates */
|
|
2
|
-
import { readdir, rm } from 'node:fs/promises';
|
|
2
|
+
import { access, readdir, rm } from 'node:fs/promises';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { s } from '@sapphire/shapeshift';
|
|
5
5
|
import { ensureDirSync, move, moveSync, pathExistsSync } from 'fs-extra/esm';
|
|
6
|
-
import { promiseMap } from './background-handler/index.js';
|
|
6
|
+
import { promiseMap, scheduleBackgroundTask } from './background-handler/index.js';
|
|
7
|
+
import { findOrCacheDatasetByPossibleId, findOrCacheKeyValueStoreByPossibleId, findRequestQueueByPossibleId, } from './cache-helpers.js';
|
|
7
8
|
import { DatasetClient } from './resource-clients/dataset.js';
|
|
8
|
-
import { DatasetCollectionClient } from './resource-clients/dataset-collection.js';
|
|
9
9
|
import { KeyValueStoreClient } from './resource-clients/key-value-store.js';
|
|
10
|
-
import { KeyValueStoreCollectionClient } from './resource-clients/key-value-store-collection.js';
|
|
11
10
|
import { RequestQueueClient } from './resource-clients/request-queue.js';
|
|
12
|
-
import { RequestQueueCollectionClient } from './resource-clients/request-queue-collection.js';
|
|
13
11
|
export class MemoryStorage {
|
|
14
12
|
localDataDirectory;
|
|
15
13
|
datasetsDirectory;
|
|
@@ -17,15 +15,17 @@ export class MemoryStorage {
|
|
|
17
15
|
requestQueuesDirectory;
|
|
18
16
|
writeMetadata;
|
|
19
17
|
persistStorage;
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
logger;
|
|
19
|
+
keyValueStoreCache = [];
|
|
20
|
+
datasetClientCache = [];
|
|
21
|
+
requestQueueCache = [];
|
|
23
22
|
constructor(options = {}) {
|
|
24
23
|
s.object({
|
|
25
24
|
localDataDirectory: s.string().optional(),
|
|
26
25
|
writeMetadata: s.boolean().optional(),
|
|
27
26
|
persistStorage: s.boolean().optional(),
|
|
28
27
|
}).parse(options);
|
|
28
|
+
this.logger = options.logger;
|
|
29
29
|
// v3.0.0 used `crawlee_storage` as the default, we changed this in v3.0.1 to just `storage`,
|
|
30
30
|
// this function handles it without making BC breaks - it respects existing `crawlee_storage`
|
|
31
31
|
// directories, and uses the `storage` only if it's not there.
|
|
@@ -50,44 +50,128 @@ export class MemoryStorage {
|
|
|
50
50
|
? !['false', '0', ''].includes(process.env.CRAWLEE_PERSIST_STORAGE)
|
|
51
51
|
: true);
|
|
52
52
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Return a cache key that includes the resolved storage directory, so that
|
|
55
|
+
* two `MemoryStorage` instances pointing at different directories get separate
|
|
56
|
+
* cache partitions. Mirrors crawlee-python's `FileSystemStorageClient` which
|
|
57
|
+
* includes `configuration.storage_dir` in its cache key.
|
|
58
|
+
*/
|
|
59
|
+
getStorageClientCacheKey() {
|
|
60
|
+
return `MemoryStorage:${resolve(this.localDataDirectory)}`;
|
|
58
61
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
+
static resolveStorageName(options) {
|
|
63
|
+
const name = 'alias' in options ? options.alias : (options.name ?? options.id);
|
|
64
|
+
// Normalize the internal __default__ alias to the user-facing 'default' name.
|
|
65
|
+
return name === '__default__' ? 'default' : name;
|
|
62
66
|
}
|
|
63
|
-
|
|
64
|
-
|
|
67
|
+
async createDatasetClient(options = {}) {
|
|
68
|
+
const name = MemoryStorage.resolveStorageName(options);
|
|
69
|
+
if (name) {
|
|
70
|
+
const found = await findOrCacheDatasetByPossibleId(this, name);
|
|
71
|
+
if (found) {
|
|
72
|
+
return found;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const newStore = new DatasetClient({ name, baseStorageDirectory: this.datasetsDirectory, client: this });
|
|
76
|
+
this.datasetClientCache.push(newStore);
|
|
77
|
+
// Schedule the worker to write to the disk
|
|
78
|
+
const datasetInfo = newStore.toDatasetInfo();
|
|
79
|
+
scheduleBackgroundTask({
|
|
80
|
+
action: 'update-metadata',
|
|
81
|
+
entityType: 'datasets',
|
|
82
|
+
entityDirectory: newStore.datasetDirectory,
|
|
83
|
+
id: datasetInfo.name ?? datasetInfo.id,
|
|
84
|
+
data: datasetInfo,
|
|
85
|
+
writeMetadata: this.writeMetadata,
|
|
86
|
+
persistStorage: this.persistStorage,
|
|
87
|
+
}, this.logger);
|
|
88
|
+
return newStore;
|
|
89
|
+
}
|
|
90
|
+
async createKeyValueStoreClient(options = {}) {
|
|
91
|
+
const name = MemoryStorage.resolveStorageName(options);
|
|
92
|
+
if (name) {
|
|
93
|
+
const found = await findOrCacheKeyValueStoreByPossibleId(this, name);
|
|
94
|
+
if (found) {
|
|
95
|
+
return found;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const newStore = new KeyValueStoreClient({
|
|
99
|
+
name,
|
|
65
100
|
baseStorageDirectory: this.keyValueStoresDirectory,
|
|
66
101
|
client: this,
|
|
67
102
|
});
|
|
103
|
+
this.keyValueStoreCache.push(newStore);
|
|
104
|
+
// Schedule the worker to write to the disk
|
|
105
|
+
const kvStoreInfo = newStore.toKeyValueStoreInfo();
|
|
106
|
+
scheduleBackgroundTask({
|
|
107
|
+
action: 'update-metadata',
|
|
108
|
+
entityType: 'keyValueStores',
|
|
109
|
+
entityDirectory: newStore.keyValueStoreDirectory,
|
|
110
|
+
id: kvStoreInfo.name ?? kvStoreInfo.id,
|
|
111
|
+
data: kvStoreInfo,
|
|
112
|
+
writeMetadata: this.writeMetadata,
|
|
113
|
+
persistStorage: this.persistStorage,
|
|
114
|
+
}, this.logger);
|
|
115
|
+
return newStore;
|
|
68
116
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
117
|
+
async createRequestQueueClient(options = {}) {
|
|
118
|
+
const name = MemoryStorage.resolveStorageName(options);
|
|
119
|
+
if (name) {
|
|
120
|
+
const found = await findRequestQueueByPossibleId(this, name);
|
|
121
|
+
if (found) {
|
|
122
|
+
return found;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const newStore = new RequestQueueClient({
|
|
126
|
+
name,
|
|
75
127
|
baseStorageDirectory: this.requestQueuesDirectory,
|
|
76
128
|
client: this,
|
|
77
129
|
});
|
|
130
|
+
this.requestQueueCache.push(newStore);
|
|
131
|
+
// Schedule the worker to write to the disk
|
|
132
|
+
const queueInfo = newStore.toRequestQueueInfo();
|
|
133
|
+
scheduleBackgroundTask({
|
|
134
|
+
action: 'update-metadata',
|
|
135
|
+
entityType: 'requestQueues',
|
|
136
|
+
entityDirectory: newStore.requestQueueDirectory,
|
|
137
|
+
id: queueInfo.name ?? queueInfo.id,
|
|
138
|
+
data: queueInfo,
|
|
139
|
+
writeMetadata: this.writeMetadata,
|
|
140
|
+
persistStorage: this.persistStorage,
|
|
141
|
+
}, this.logger);
|
|
142
|
+
return newStore;
|
|
78
143
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
144
|
+
async storageExists(id, type) {
|
|
145
|
+
let clients;
|
|
146
|
+
let baseDir;
|
|
147
|
+
switch (type) {
|
|
148
|
+
case 'Dataset':
|
|
149
|
+
clients = this.datasetClientCache;
|
|
150
|
+
baseDir = this.datasetsDirectory;
|
|
151
|
+
break;
|
|
152
|
+
case 'KeyValueStore':
|
|
153
|
+
clients = this.keyValueStoreCache;
|
|
154
|
+
baseDir = this.keyValueStoresDirectory;
|
|
155
|
+
break;
|
|
156
|
+
case 'RequestQueue':
|
|
157
|
+
clients = this.requestQueueCache;
|
|
158
|
+
baseDir = this.requestQueuesDirectory;
|
|
159
|
+
break;
|
|
160
|
+
default:
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
// Check in-memory cache first
|
|
164
|
+
if (clients.some((store) => store.id === id)) {
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
// Check if a directory with that ID exists on disk
|
|
168
|
+
try {
|
|
169
|
+
await access(resolve(baseDir, id));
|
|
170
|
+
return true;
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
91
175
|
}
|
|
92
176
|
async setStatusMessage(message, options = {}) {
|
|
93
177
|
s.string().parse(message);
|
|
@@ -110,7 +194,7 @@ export class MemoryStorage {
|
|
|
110
194
|
if (keyValueStoreFolder.startsWith('__CRAWLEE_TEMPORARY') || keyValueStoreFolder.startsWith('__OLD')) {
|
|
111
195
|
keyValueStorePromises.push((await this.batchRemoveFiles(resolve(this.keyValueStoresDirectory, keyValueStoreFolder)))());
|
|
112
196
|
}
|
|
113
|
-
else if (keyValueStoreFolder === 'default') {
|
|
197
|
+
else if (keyValueStoreFolder === 'default' || keyValueStoreFolder === '__default__') {
|
|
114
198
|
keyValueStorePromises.push(this.handleDefaultKeyValueStore(resolve(this.keyValueStoresDirectory, keyValueStoreFolder))());
|
|
115
199
|
}
|
|
116
200
|
}
|
|
@@ -119,7 +203,9 @@ export class MemoryStorage {
|
|
|
119
203
|
const datasets = await readdir(this.datasetsDirectory).catch(() => []);
|
|
120
204
|
const datasetPromises = [];
|
|
121
205
|
for (const datasetFolder of datasets) {
|
|
122
|
-
if (datasetFolder === 'default' ||
|
|
206
|
+
if (datasetFolder === 'default' ||
|
|
207
|
+
datasetFolder === '__default__' ||
|
|
208
|
+
datasetFolder.startsWith('__CRAWLEE_TEMPORARY')) {
|
|
123
209
|
datasetPromises.push((await this.batchRemoveFiles(resolve(this.datasetsDirectory, datasetFolder)))());
|
|
124
210
|
}
|
|
125
211
|
}
|
|
@@ -128,7 +214,9 @@ export class MemoryStorage {
|
|
|
128
214
|
const requestQueues = await readdir(this.requestQueuesDirectory).catch(() => []);
|
|
129
215
|
const requestQueuePromises = [];
|
|
130
216
|
for (const requestQueueFolder of requestQueues) {
|
|
131
|
-
if (requestQueueFolder === 'default' ||
|
|
217
|
+
if (requestQueueFolder === 'default' ||
|
|
218
|
+
requestQueueFolder === '__default__' ||
|
|
219
|
+
requestQueueFolder.startsWith('__CRAWLEE_TEMPORARY')) {
|
|
132
220
|
requestQueuePromises.push((await this.batchRemoveFiles(resolve(this.requestQueuesDirectory, requestQueueFolder)))());
|
|
133
221
|
}
|
|
134
222
|
}
|
package/memory-storage.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-storage.js","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE7E,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAC;AACnF,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,6BAA6B,EAAE,MAAM,kDAAkD,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,4BAA4B,EAAE,MAAM,gDAAgD,CAAC;AAwB9F,MAAM,OAAO,aAAa;IACb,kBAAkB,CAAS;IAC3B,iBAAiB,CAAS;IAC1B,uBAAuB,CAAS;IAChC,sBAAsB,CAAS;IAC/B,aAAa,CAAU;IACvB,cAAc,CAAU;IAExB,qBAAqB,GAA0B,EAAE,CAAC;IAClD,qBAAqB,GAAoB,EAAE,CAAC;IAC5C,oBAAoB,GAAyB,EAAE,CAAC;IAEzD,YAAY,UAAgC,EAAE;QAC1C,CAAC,CAAC,MAAM,CAAC;YACL,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACzC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YACrC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SACzC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,6FAA6F;QAC7F,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAM,iBAAiB,GAAG,GAAG,EAAE;YAC3B,IAAI,cAAc,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,EAAE,CAAC;gBAC/C,OAAO,mBAAmB,CAAC;YAC/B,CAAC;YAED,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,iBAAiB,EAAE,CAAC;QAC/G,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;QACpF,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QACjF,IAAI,CAAC,aAAa;YACd,OAAO,CAAC,aAAa;gBACrB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,wBAAwB,CAAC;gBACrD,KAAK,CAAC;QACV,IAAI,CAAC,cAAc;YACf,OAAO,CAAC,cAAc;gBACtB,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB;oBAChC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAwB,CAAC;oBACpE,CAAC,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,QAAQ;QACJ,OAAO,IAAI,uBAAuB,CAAC;YAC/B,oBAAoB,EAAE,IAAI,CAAC,iBAAiB;YAC5C,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;IACP,CAAC;IAED,OAAO,CAAuC,EAAU;QACpD,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAErB,OAAO,IAAI,aAAa,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,cAAc;QACV,OAAO,IAAI,6BAA6B,CAAC;YACrC,oBAAoB,EAAE,IAAI,CAAC,uBAAuB;YAClD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;IACP,CAAC;IAED,aAAa,CAAC,EAAU;QACpB,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAErB,OAAO,IAAI,mBAAmB,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,uBAAuB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7G,CAAC;IAED,aAAa;QACT,OAAO,IAAI,4BAA4B,CAAC;YACpC,oBAAoB,EAAE,IAAI,CAAC,sBAAsB;YACjD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;IACP,CAAC;IAED,YAAY,CAAC,EAAU,EAAE,UAAuC,EAAE;QAC9D,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC,CAAC,MAAM,CAAC;YACL,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAChC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SACrC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO,IAAI,kBAAkB,CAAC;YAC1B,EAAE;YACF,oBAAoB,EAAE,IAAI,CAAC,sBAAsB;YACjD,MAAM,EAAE,IAAI;YACZ,GAAG,OAAO;SACb,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,UAA2C,EAAE;QACjF,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,MAAM,CAAC;YACL,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAClD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,mBAAmB;QACnB,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,qBAAqB,GAAoB,EAAE,CAAC;QAElD,KAAK,MAAM,mBAAmB,IAAI,cAAc,EAAE,CAAC;YAC/C,IAAI,mBAAmB,CAAC,UAAU,CAAC,qBAAqB,CAAC,IAAI,mBAAmB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnG,qBAAqB,CAAC,IAAI,CACtB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAC9F,CAAC;YACN,CAAC;iBAAM,IAAI,mBAAmB,KAAK,SAAS,EAAE,CAAC;gBAC3C,qBAAqB,CAAC,IAAI,CACtB,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,EAAE,CAChG,CAAC;YACN,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QAE/C,WAAW;QACX,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,eAAe,GAAoB,EAAE,CAAC;QAE5C,KAAK,MAAM,aAAa,IAAI,QAAQ,EAAE,CAAC;YACnC,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBACjF,eAAe,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QAEzC,iBAAiB;QACjB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACjF,MAAM,oBAAoB,GAAoB,EAAE,CAAC;QAEjD,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;YAC7C,IAAI,kBAAkB,KAAK,SAAS,IAAI,kBAAkB,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3F,oBAAoB,CAAC,IAAI,CACrB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAC5F,CAAC;YACN,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QAE9E,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAEO,0BAA0B,CAAC,MAAc;QAC7C,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,0CAA0C,CAAC,CAAC;QAElF,iGAAiG;QACjG,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAE5E,IAAI,iBAAiB,EAAE,CAAC;YACpB,qDAAqD;YACrD,aAAa,CAAC,aAAa,CAAC,CAAC;YAE7B,4DAA4D;YAC5D,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;gBACrC,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACjD,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBAEpD,IAAI,CAAC;oBACD,QAAQ,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACL,SAAS;gBACb,CAAC;YACL,CAAC;YAED,iDAAiD;YACjD,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,oBAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,oBAAoB,OAAO,IAAI,CAAC,CAAC;YAC5E,IAAI,IAAI,GAAG,KAAK,CAAC;YAEjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,CAAC;oBACD,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;oBACvC,IAAI,GAAG,IAAI,CAAC;gBAChB,CAAC;gBAAC,MAAM,CAAC;oBACL,oBAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,IAAI,CAAC,CAAC;gBAC9E,CAAC;YACL,CAAC;YAED,wDAAwD;YACxD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEhC,wBAAwB;YACxB,OAAO,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,MAAc,EAAE,OAAO,GAAG,CAAC;QACtD,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAE5C,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,0BAA0B,OAAO,IAAI,CAAC,CAAC;YAE/E,IAAI,CAAC;gBACD,qEAAqE;gBACrE,MAAM,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACL,+DAA+D;gBAC/D,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAED,OAAO,KAAK,IAAI,EAAE;gBACd,+BAA+B;gBAC/B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAC;gBAE/C,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,QAAQ,GAAoB,EAAE,CAAC;gBAEnC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC1B,SAAS,EAAE,CAAC;oBACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAEpE,gCAAgC;oBAChC,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;wBACzB,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;wBACnC,QAAQ,GAAG,EAAE,CAAC;oBAClB,CAAC;gBACL,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAEnC,2BAA2B;gBAC3B,MAAM,EAAE,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC;QACN,CAAC;QAED,OAAO,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"memory-storage.js","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE7E,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACnF,OAAO,EACH,8BAA8B,EAC9B,oCAAoC,EACpC,4BAA4B,GAC/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AA6BzE,MAAM,OAAO,aAAa;IACb,kBAAkB,CAAS;IAC3B,iBAAiB,CAAS;IAC1B,uBAAuB,CAAS;IAChC,sBAAsB,CAAS;IAC/B,aAAa,CAAU;IACvB,cAAc,CAAU;IACxB,MAAM,CAAiB;IAEvB,kBAAkB,GAA0B,EAAE,CAAC;IAC/C,kBAAkB,GAAoB,EAAE,CAAC;IACzC,iBAAiB,GAAyB,EAAE,CAAC;IAEtD,YAAY,UAAgC,EAAE;QAC1C,CAAC,CAAC,MAAM,CAAC;YACL,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACzC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YACrC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SACzC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAE7B,6FAA6F;QAC7F,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAM,iBAAiB,GAAG,GAAG,EAAE;YAC3B,IAAI,cAAc,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,EAAE,CAAC;gBAC/C,OAAO,mBAAmB,CAAC;YAC/B,CAAC;YAED,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,iBAAiB,EAAE,CAAC;QAC/G,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;QACpF,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QACjF,IAAI,CAAC,aAAa;YACd,OAAO,CAAC,aAAa;gBACrB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,wBAAwB,CAAC;gBACrD,KAAK,CAAC;QACV,IAAI,CAAC,cAAc;YACf,OAAO,CAAC,cAAc;gBACtB,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB;oBAChC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAwB,CAAC;oBACpE,CAAC,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,wBAAwB;QACpB,OAAO,iBAAiB,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;IAC/D,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,OAAuD;QACrF,MAAM,IAAI,GAAG,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/E,8EAA8E;QAC9E,OAAO,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,UAA8C,EAAE;QACtE,MAAM,IAAI,GAAG,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEvD,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,8BAA8B,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC/D,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,aAAa,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACzG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QAE7C,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,QAAQ,CAAC,gBAAgB;YAC1C,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;YACjB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,EACD,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC3B,UAAoD,EAAE;QAEtD,MAAM,IAAI,GAAG,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEvD,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,oCAAoC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrE,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;YACrC,IAAI;YACJ,oBAAoB,EAAE,IAAI,CAAC,uBAAuB;YAClD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QAEnD,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,QAAQ,CAAC,sBAAsB;YAChD,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;YACjB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,EACD,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC1B,UAAmD,EAAE;QAErD,MAAM,IAAI,GAAG,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEvD,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,4BAA4B,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7D,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;YACpC,IAAI;YACJ,oBAAoB,EAAE,IAAI,CAAC,sBAAsB;YACjD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEtC,2CAA2C;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,EAAE,CAAC;QAEhD,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,eAAe;YAC3B,eAAe,EAAE,QAAQ,CAAC,qBAAqB;YAC/C,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE;YAClC,IAAI,EAAE,SAAS;YACf,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,EACD,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAU,EAAE,IAAkD;QAC9E,IAAI,OAAyB,CAAC;QAC9B,IAAI,OAAe,CAAC;QAEpB,QAAQ,IAAI,EAAE,CAAC;YACX,KAAK,SAAS;gBACV,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBAClC,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjC,MAAM;YACV,KAAK,eAAe;gBAChB,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBAClC,OAAO,GAAG,IAAI,CAAC,uBAAuB,CAAC;gBACvC,MAAM;YACV,KAAK,cAAc;gBACf,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC;gBACjC,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC;gBACtC,MAAM;YACV;gBACI,OAAO,KAAK,CAAC;QACrB,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,mDAAmD;QACnD,IAAI,CAAC;YACD,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,UAA2C,EAAE;QACjF,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,MAAM,CAAC;YACL,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAClD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,mBAAmB;QACnB,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,qBAAqB,GAAoB,EAAE,CAAC;QAElD,KAAK,MAAM,mBAAmB,IAAI,cAAc,EAAE,CAAC;YAC/C,IAAI,mBAAmB,CAAC,UAAU,CAAC,qBAAqB,CAAC,IAAI,mBAAmB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnG,qBAAqB,CAAC,IAAI,CACtB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAC9F,CAAC;YACN,CAAC;iBAAM,IAAI,mBAAmB,KAAK,SAAS,IAAI,mBAAmB,KAAK,aAAa,EAAE,CAAC;gBACpF,qBAAqB,CAAC,IAAI,CACtB,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,EAAE,CAChG,CAAC;YACN,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QAE/C,WAAW;QACX,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,eAAe,GAAoB,EAAE,CAAC;QAE5C,KAAK,MAAM,aAAa,IAAI,QAAQ,EAAE,CAAC;YACnC,IACI,aAAa,KAAK,SAAS;gBAC3B,aAAa,KAAK,aAAa;gBAC/B,aAAa,CAAC,UAAU,CAAC,qBAAqB,CAAC,EACjD,CAAC;gBACC,eAAe,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QAEzC,iBAAiB;QACjB,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACjF,MAAM,oBAAoB,GAAoB,EAAE,CAAC;QAEjD,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;YAC7C,IACI,kBAAkB,KAAK,SAAS;gBAChC,kBAAkB,KAAK,aAAa;gBACpC,kBAAkB,CAAC,UAAU,CAAC,qBAAqB,CAAC,EACtD,CAAC;gBACC,oBAAoB,CAAC,IAAI,CACrB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAC5F,CAAC;YACN,CAAC;QACL,CAAC;QAED,KAAK,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QAE9E,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAEO,0BAA0B,CAAC,MAAc;QAC7C,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,0CAA0C,CAAC,CAAC;QAElF,iGAAiG;QACjG,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAE5E,IAAI,iBAAiB,EAAE,CAAC;YACpB,qDAAqD;YACrD,aAAa,CAAC,aAAa,CAAC,CAAC;YAE7B,4DAA4D;YAC5D,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;gBACrC,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACjD,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBAEpD,IAAI,CAAC;oBACD,QAAQ,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACL,SAAS;gBACb,CAAC;YACL,CAAC;YAED,iDAAiD;YACjD,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,oBAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,oBAAoB,OAAO,IAAI,CAAC,CAAC;YAC5E,IAAI,IAAI,GAAG,KAAK,CAAC;YAEjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,CAAC;oBACD,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;oBACvC,IAAI,GAAG,IAAI,CAAC;gBAChB,CAAC;gBAAC,MAAM,CAAC;oBACL,oBAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,IAAI,CAAC,CAAC;gBAC9E,CAAC;YACL,CAAC;YAED,wDAAwD;YACxD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEhC,wBAAwB;YACxB,OAAO,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,MAAc,EAAE,OAAO,GAAG,CAAC;QACtD,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAE5C,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,0BAA0B,OAAO,IAAI,CAAC,CAAC;YAE/E,IAAI,CAAC;gBACD,qEAAqE;gBACrE,MAAM,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACL,+DAA+D;gBAC/D,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAED,OAAO,KAAK,IAAI,EAAE;gBACd,+BAA+B;gBAC/B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAC;gBAE/C,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,QAAQ,GAAoB,EAAE,CAAC;gBAEnC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC1B,SAAS,EAAE,CAAC;oBACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAEpE,gCAAgC;oBAChC,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;wBACzB,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;wBACnC,QAAQ,GAAG,EAAE,CAAC;oBAClB,CAAC;gBACL,CAAC;gBAED,mCAAmC;gBACnC,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAEnC,2BAA2B;gBAC3B,MAAM,EAAE,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC;QACN,CAAC;QAED,OAAO,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/memory-storage",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.61",
|
|
4
4
|
"description": "A simple in-memory storage implementation of the Apify API",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
"homepage": "https://crawlee.dev",
|
|
35
35
|
"scripts": {
|
|
36
|
-
"build": "
|
|
36
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
37
37
|
"clean": "rimraf ./dist",
|
|
38
38
|
"compile": "tsc -p tsconfig.build.json",
|
|
39
39
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -42,14 +42,14 @@
|
|
|
42
42
|
"access": "public"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@
|
|
46
|
-
"@crawlee/types": "4.0.0-beta.6",
|
|
45
|
+
"@crawlee/types": "4.0.0-beta.61",
|
|
47
46
|
"@sapphire/async-queue": "^1.5.5",
|
|
48
47
|
"@sapphire/shapeshift": "^4.0.0",
|
|
49
48
|
"content-type": "^1.0.5",
|
|
50
49
|
"fs-extra": "^11.3.0",
|
|
51
50
|
"json5": "^2.2.3",
|
|
52
51
|
"mime-types": "^3.0.1",
|
|
52
|
+
"p-limit": "^6.2.0",
|
|
53
53
|
"proper-lockfile": "^4.1.2",
|
|
54
54
|
"tslib": "^2.8.1"
|
|
55
55
|
},
|
|
@@ -60,5 +60,5 @@
|
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
},
|
|
63
|
-
"gitHead": "
|
|
63
|
+
"gitHead": "4ff7bde2ee98666a553917858fe4fd12c264a9a6"
|
|
64
64
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Dictionary } from '@crawlee/types';
|
|
2
1
|
import type * as storage from '@crawlee/types';
|
|
2
|
+
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { MemoryStorage } from '../index.js';
|
|
4
4
|
import { BaseClient } from './common/base-client.js';
|
|
5
5
|
export interface DatasetClientOptions {
|
|
@@ -18,24 +18,15 @@ export declare class DatasetClient<Data extends Dictionary = Dictionary> extends
|
|
|
18
18
|
private readonly datasetEntries;
|
|
19
19
|
private readonly client;
|
|
20
20
|
constructor(options: DatasetClientOptions);
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
getMetadata(): Promise<storage.DatasetInfo>;
|
|
22
|
+
drop(): Promise<void>;
|
|
23
|
+
purge(): Promise<void>;
|
|
24
|
+
getData(options?: storage.DatasetClientListOptions): Promise<storage.PaginatedList<Data>>;
|
|
25
|
+
private getDataPage;
|
|
26
|
+
pushData(items: Data[]): Promise<void>;
|
|
27
27
|
toDatasetInfo(): storage.DatasetInfo;
|
|
28
28
|
private generateLocalEntryName;
|
|
29
29
|
private getStartAndEndIndexes;
|
|
30
|
-
/**
|
|
31
|
-
* To emulate API and split arrays of items into individual dataset items,
|
|
32
|
-
* we need to normalize the input items - which can be strings, objects
|
|
33
|
-
* or arrays of those - into objects, so that we can save them one by one
|
|
34
|
-
* later. We could potentially do this directly with strings, but let's
|
|
35
|
-
* not optimize prematurely.
|
|
36
|
-
*/
|
|
37
|
-
private normalizeItems;
|
|
38
|
-
private normalizeItem;
|
|
39
30
|
private updateTimestamps;
|
|
40
31
|
}
|
|
41
32
|
//# sourceMappingURL=dataset.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAcrD,MAAM,WAAW,oBAAoB;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,CAC3D,SAAQ,UACR,YAAW,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC;IAEtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,SAAS,SAAK;IACd,gBAAgB,EAAE,MAAM,CAAC;IAEzB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAkD;IACjF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,oBAAoB;IAOnC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;IAK3C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAYrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B,OAAO,CAAC,OAAO,GAAE,OAAO,CAAC,wBAA6B,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAgB/E,WAAW;IA2BnB,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB5C,aAAa,IAAI,OAAO,CAAC,WAAW;IAWpC,OAAO,CAAC,sBAAsB;IAI9B,OAAO,CAAC,qBAAqB;IAM7B,OAAO,CAAC,gBAAgB;CAqB3B"}
|
|
@@ -3,10 +3,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
3
3
|
import { rm } from 'node:fs/promises';
|
|
4
4
|
import { resolve } from 'node:path';
|
|
5
5
|
import { s } from '@sapphire/shapeshift';
|
|
6
|
-
import { move } from 'fs-extra/esm';
|
|
7
6
|
import { scheduleBackgroundTask } from '../background-handler/index.js';
|
|
8
|
-
import { findOrCacheDatasetByPossibleId } from '../cache-helpers.js';
|
|
9
|
-
import { StorageTypes } from '../consts.js';
|
|
10
7
|
import { createDatasetStorageImplementation } from '../fs/dataset/index.js';
|
|
11
8
|
import { BaseClient } from './common/base-client.js';
|
|
12
9
|
/**
|
|
@@ -34,110 +31,78 @@ export class DatasetClient extends BaseClient {
|
|
|
34
31
|
this.datasetDirectory = resolve(options.baseStorageDirectory, this.name ?? this.id);
|
|
35
32
|
this.client = options.client;
|
|
36
33
|
}
|
|
37
|
-
async
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
found.updateTimestamps(false);
|
|
41
|
-
return found.toDatasetInfo();
|
|
42
|
-
}
|
|
43
|
-
return undefined;
|
|
44
|
-
}
|
|
45
|
-
async update(newFields = {}) {
|
|
46
|
-
const parsed = s
|
|
47
|
-
.object({
|
|
48
|
-
name: s.string().lengthGreaterThan(0).optional(),
|
|
49
|
-
})
|
|
50
|
-
.parse(newFields);
|
|
51
|
-
// Check by id
|
|
52
|
-
const existingStoreById = await findOrCacheDatasetByPossibleId(this.client, this.name ?? this.id);
|
|
53
|
-
if (!existingStoreById) {
|
|
54
|
-
this.throwOnNonExisting(StorageTypes.Dataset);
|
|
55
|
-
}
|
|
56
|
-
// Skip if no changes
|
|
57
|
-
if (!parsed.name) {
|
|
58
|
-
return existingStoreById.toDatasetInfo();
|
|
59
|
-
}
|
|
60
|
-
// Check that name is not in use already
|
|
61
|
-
const existingStoreByName = this.client.datasetClientsHandled.find((store) => store.name?.toLowerCase() === parsed.name.toLowerCase());
|
|
62
|
-
if (existingStoreByName) {
|
|
63
|
-
this.throwOnDuplicateEntry(StorageTypes.Dataset, 'name', parsed.name);
|
|
64
|
-
}
|
|
65
|
-
existingStoreById.name = parsed.name;
|
|
66
|
-
const previousDir = existingStoreById.datasetDirectory;
|
|
67
|
-
existingStoreById.datasetDirectory = resolve(this.client.datasetsDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
|
|
68
|
-
await move(previousDir, existingStoreById.datasetDirectory, { overwrite: true });
|
|
69
|
-
// Update timestamps
|
|
70
|
-
existingStoreById.updateTimestamps(true);
|
|
71
|
-
return existingStoreById.toDatasetInfo();
|
|
34
|
+
async getMetadata() {
|
|
35
|
+
this.updateTimestamps(false);
|
|
36
|
+
return this.toDatasetInfo();
|
|
72
37
|
}
|
|
73
|
-
async
|
|
74
|
-
const storeIndex = this.client.
|
|
38
|
+
async drop() {
|
|
39
|
+
const storeIndex = this.client.datasetClientCache.findIndex((store) => store.id === this.id);
|
|
75
40
|
if (storeIndex !== -1) {
|
|
76
|
-
const [oldClient] = this.client.
|
|
41
|
+
const [oldClient] = this.client.datasetClientCache.splice(storeIndex, 1);
|
|
77
42
|
oldClient.itemCount = 0;
|
|
78
43
|
oldClient.datasetEntries.clear();
|
|
79
44
|
await rm(oldClient.datasetDirectory, { recursive: true, force: true });
|
|
80
45
|
}
|
|
81
46
|
}
|
|
82
|
-
async
|
|
83
|
-
|
|
47
|
+
async purge() {
|
|
48
|
+
this.itemCount = 0;
|
|
49
|
+
this.datasetEntries.clear();
|
|
50
|
+
// Remove item files from disk but keep the directory
|
|
51
|
+
if (this.client.persistStorage) {
|
|
52
|
+
const { readdir } = await import('node:fs/promises');
|
|
53
|
+
const entries = await readdir(this.datasetDirectory).catch(() => []);
|
|
54
|
+
for (const entry of entries) {
|
|
55
|
+
if (entry !== '__metadata__.json') {
|
|
56
|
+
await rm(resolve(this.datasetDirectory, entry), { force: true });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
this.updateTimestamps(true);
|
|
84
61
|
}
|
|
85
|
-
|
|
86
|
-
const { limit
|
|
62
|
+
getData(options = {}) {
|
|
63
|
+
const { desc, limit, offset } = s
|
|
87
64
|
.object({
|
|
88
65
|
desc: s.boolean().optional(),
|
|
89
66
|
limit: s.number().int().optional(),
|
|
90
67
|
offset: s.number().int().optional(),
|
|
91
68
|
})
|
|
92
69
|
.parse(options);
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
}
|
|
98
|
-
|
|
70
|
+
return this.getDataPage({
|
|
71
|
+
desc,
|
|
72
|
+
offset: offset ?? 0,
|
|
73
|
+
limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
async getDataPage(options = {}) {
|
|
77
|
+
const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
|
|
78
|
+
const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
|
|
99
79
|
const items = [];
|
|
100
80
|
for (let idx = start; idx < end; idx++) {
|
|
101
81
|
const entryNumber = this.generateLocalEntryName(idx);
|
|
102
|
-
items.push(await
|
|
82
|
+
items.push(await this.datasetEntries.get(entryNumber).get());
|
|
103
83
|
}
|
|
104
|
-
|
|
84
|
+
this.updateTimestamps(false);
|
|
105
85
|
return {
|
|
106
86
|
count: items.length,
|
|
107
87
|
desc: desc ?? false,
|
|
108
88
|
items: desc ? items.reverse() : items,
|
|
109
89
|
limit,
|
|
110
90
|
offset,
|
|
111
|
-
total:
|
|
91
|
+
total: this.itemCount,
|
|
112
92
|
};
|
|
113
93
|
}
|
|
114
|
-
async
|
|
115
|
-
const
|
|
116
|
-
.
|
|
117
|
-
s.string(),
|
|
118
|
-
s.object({}).passthrough(),
|
|
119
|
-
s.array(s.union([s.string(), s.object({}).passthrough()])),
|
|
120
|
-
])
|
|
121
|
-
.parse(items);
|
|
122
|
-
// Check by id
|
|
123
|
-
const existingStoreById = await findOrCacheDatasetByPossibleId(this.client, this.name ?? this.id);
|
|
124
|
-
if (!existingStoreById) {
|
|
125
|
-
this.throwOnNonExisting(StorageTypes.Dataset);
|
|
126
|
-
}
|
|
127
|
-
const normalized = this.normalizeItems(rawItems);
|
|
128
|
-
const addedIds = [];
|
|
129
|
-
for (const entry of normalized) {
|
|
130
|
-
const idx = this.generateLocalEntryName(++existingStoreById.itemCount);
|
|
94
|
+
async pushData(items) {
|
|
95
|
+
for (const entry of items) {
|
|
96
|
+
const idx = this.generateLocalEntryName(++this.itemCount);
|
|
131
97
|
const storageEntry = createDatasetStorageImplementation({
|
|
132
98
|
entityId: idx,
|
|
133
99
|
persistStorage: this.client.persistStorage,
|
|
134
|
-
storeDirectory:
|
|
100
|
+
storeDirectory: this.datasetDirectory,
|
|
135
101
|
});
|
|
136
102
|
await storageEntry.update(entry);
|
|
137
|
-
|
|
138
|
-
addedIds.push(idx);
|
|
103
|
+
this.datasetEntries.set(idx, storageEntry);
|
|
139
104
|
}
|
|
140
|
-
|
|
105
|
+
this.updateTimestamps(true);
|
|
141
106
|
}
|
|
142
107
|
toDatasetInfo() {
|
|
143
108
|
return {
|
|
@@ -157,31 +122,6 @@ export class DatasetClient extends BaseClient {
|
|
|
157
122
|
const end = Math.min(offset + limit, this.itemCount) + 1;
|
|
158
123
|
return [start, end];
|
|
159
124
|
}
|
|
160
|
-
/**
|
|
161
|
-
* To emulate API and split arrays of items into individual dataset items,
|
|
162
|
-
* we need to normalize the input items - which can be strings, objects
|
|
163
|
-
* or arrays of those - into objects, so that we can save them one by one
|
|
164
|
-
* later. We could potentially do this directly with strings, but let's
|
|
165
|
-
* not optimize prematurely.
|
|
166
|
-
*/
|
|
167
|
-
normalizeItems(items) {
|
|
168
|
-
if (typeof items === 'string') {
|
|
169
|
-
items = JSON.parse(items);
|
|
170
|
-
}
|
|
171
|
-
return Array.isArray(items) ? items.map((item) => this.normalizeItem(item)) : [this.normalizeItem(items)];
|
|
172
|
-
}
|
|
173
|
-
normalizeItem(item) {
|
|
174
|
-
if (typeof item === 'string') {
|
|
175
|
-
item = JSON.parse(item);
|
|
176
|
-
}
|
|
177
|
-
if (Array.isArray(item)) {
|
|
178
|
-
throw new Error(`Each dataset item can only be a single JSON object, not an array. Received: [${item.join(',\n')}]`);
|
|
179
|
-
}
|
|
180
|
-
if (typeof item !== 'object' || item === null) {
|
|
181
|
-
throw new Error(`Each dataset item must be a JSON object. Received: ${item}`);
|
|
182
|
-
}
|
|
183
|
-
return item;
|
|
184
|
-
}
|
|
185
125
|
updateTimestamps(hasBeenModified) {
|
|
186
126
|
this.accessedAt = new Date();
|
|
187
127
|
if (hasBeenModified) {
|
|
@@ -196,7 +136,7 @@ export class DatasetClient extends BaseClient {
|
|
|
196
136
|
id: this.name ?? this.id,
|
|
197
137
|
writeMetadata: this.client.writeMetadata,
|
|
198
138
|
persistStorage: this.client.persistStorage,
|
|
199
|
-
});
|
|
139
|
+
}, this.client.logger);
|
|
200
140
|
}
|
|
201
141
|
}
|
|
202
142
|
//# sourceMappingURL=dataset.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,OAAO,EAAE,kCAAkC,EAAE,MAAM,wBAAwB,CAAC;AAE5E,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD;;;GAGG;AACH,MAAM,gBAAgB,GAAG,eAAe,CAAC;AAEzC;;;GAGG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC;AASlC,MAAM,OAAO,aACT,SAAQ,UAAU;IAGlB,IAAI,CAAU;IACd,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,SAAS,GAAG,CAAC,CAAC;IACd,gBAAgB,CAAS;IAER,cAAc,GAAG,IAAI,GAAG,EAAuC,CAAC;IAChE,MAAM,CAAgB;IAEvC,YAAY,OAA6B;QACrC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,WAAW;QACb,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAE7F,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACzE,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;YACxB,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAEjC,MAAM,EAAE,CAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,qDAAqD;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACrE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,KAAK,KAAK,mBAAmB,EAAE,CAAC;oBAChC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBACrE,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,UAA4C,EAAE;QAClD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC;aAC5B,MAAM,CAAC;YACJ,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC5B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACtC,CAAC;aACD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEpB,OAAO,IAAI,CAAC,WAAW,CAAC;YACpB,IAAI;YACJ,MAAM,EAAE,MAAM,IAAI,CAAC;YACnB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,gBAAgB,EAAE,gBAAgB,CAAC;SAC/D,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,UAA4C,EAAE;QACpE,MAAM,EAAE,KAAK,GAAG,gBAAgB,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;QAE/D,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAC3C,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAC5D,KAAK,CACR,CAAC;QAEF,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,IAAI,KAAK;YACnB,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK;YACrC,KAAK;YACL,MAAM;YACN,KAAK,EAAE,IAAI,CAAC,SAAS;SACxB,CAAC;IACN,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa;QACxB,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,YAAY,GAAG,kCAAkC,CAAC;gBACpD,QAAQ,EAAE,GAAG;gBACb,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;gBAC1C,cAAc,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,aAAa;QACT,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;SAClB,CAAC;IACN,CAAC;IAEO,sBAAsB,CAAC,GAAW;QACtC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAChE,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAU,CAAC;IACjC,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,sBAAsB,CAClB;YACI,MAAM,EAAE,iBAAiB;YACzB,IAAI;YACJ,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,IAAI,CAAC,gBAAgB;YACtC,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE;YACxB,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACxC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;SAC7C,EACD,IAAI,CAAC,MAAM,CAAC,MAAM,CACrB,CAAC;IACN,CAAC;CACJ"}
|
|
@@ -12,8 +12,9 @@ export interface InternalKeyRecord {
|
|
|
12
12
|
value: Buffer | string;
|
|
13
13
|
contentType?: string;
|
|
14
14
|
extension: string;
|
|
15
|
+
filePath?: string;
|
|
15
16
|
}
|
|
16
|
-
export declare class KeyValueStoreClient extends BaseClient {
|
|
17
|
+
export declare class KeyValueStoreClient extends BaseClient implements storage.KeyValueStoreClient {
|
|
17
18
|
name?: string;
|
|
18
19
|
createdAt: Date;
|
|
19
20
|
accessedAt: Date;
|
|
@@ -22,10 +23,17 @@ export declare class KeyValueStoreClient extends BaseClient {
|
|
|
22
23
|
private readonly keyValueEntries;
|
|
23
24
|
private readonly client;
|
|
24
25
|
constructor(options: KeyValueStoreClientOptions);
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
listKeys(options?: storage.
|
|
26
|
+
getMetadata(): Promise<storage.KeyValueStoreInfo>;
|
|
27
|
+
drop(): Promise<void>;
|
|
28
|
+
purge(): Promise<void>;
|
|
29
|
+
listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreItemData[]>;
|
|
30
|
+
/**
|
|
31
|
+
* Generates a public file:// URL for accessing a specific record in the key-value store.
|
|
32
|
+
*
|
|
33
|
+
* Returns `undefined` if the record does not exist or has no associated file path (i.e., it is not stored as a file).
|
|
34
|
+
* @param key The key of the record to generate the public URL for.
|
|
35
|
+
*/
|
|
36
|
+
getPublicUrl(key: string): Promise<string | undefined>;
|
|
29
37
|
/**
|
|
30
38
|
* Tests whether a record with the given key exists in the key-value store without retrieving its value.
|
|
31
39
|
*
|
|
@@ -33,9 +41,9 @@ export declare class KeyValueStoreClient extends BaseClient {
|
|
|
33
41
|
* @returns `true` if the record exists, `false` if it does not.
|
|
34
42
|
*/
|
|
35
43
|
recordExists(key: string): Promise<boolean>;
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
44
|
+
getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
|
|
45
|
+
setValue(record: storage.KeyValueStoreRecord): Promise<void>;
|
|
46
|
+
deleteValue(key: string): Promise<void>;
|
|
39
47
|
toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
|
|
40
48
|
private updateTimestamps;
|
|
41
49
|
}
|