@crawlee/memory-storage 3.1.5-beta.2 → 3.1.5-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cache-helpers.d.ts.map +1 -1
- package/cache-helpers.js +51 -31
- package/cache-helpers.js.map +1 -1
- package/fs/common.d.ts +6 -0
- package/fs/common.d.ts.map +1 -0
- package/fs/common.js +3 -0
- package/fs/common.js.map +1 -0
- package/fs/dataset/fs.d.ts +10 -0
- package/fs/dataset/fs.d.ts.map +1 -0
- package/fs/dataset/fs.js +30 -0
- package/fs/dataset/fs.js.map +1 -0
- package/fs/dataset/index.d.ts +10 -0
- package/fs/dataset/index.d.ts.map +1 -0
- package/fs/dataset/index.js +13 -0
- package/fs/dataset/index.js.map +1 -0
- package/fs/dataset/memory.d.ts +8 -0
- package/fs/dataset/memory.d.ts.map +1 -0
- package/fs/dataset/memory.js +24 -0
- package/fs/dataset/memory.js.map +1 -0
- package/fs/key-value-store/fs.d.ts +15 -0
- package/fs/key-value-store/fs.d.ts.map +1 -0
- package/fs/key-value-store/fs.js +87 -0
- package/fs/key-value-store/fs.js.map +1 -0
- package/fs/key-value-store/index.d.ts +9 -0
- package/fs/key-value-store/index.d.ts.map +1 -0
- package/fs/key-value-store/index.js +13 -0
- package/fs/key-value-store/index.js.map +1 -0
- package/fs/key-value-store/memory.d.ts +9 -0
- package/fs/key-value-store/memory.d.ts.map +1 -0
- package/fs/key-value-store/memory.js +24 -0
- package/fs/key-value-store/memory.js.map +1 -0
- package/fs/request-queue/fs.d.ts +11 -0
- package/fs/request-queue/fs.d.ts.map +1 -0
- package/fs/request-queue/fs.js +30 -0
- package/fs/request-queue/fs.js.map +1 -0
- package/fs/request-queue/index.d.ts +9 -0
- package/fs/request-queue/index.d.ts.map +1 -0
- package/fs/request-queue/index.js +13 -0
- package/fs/request-queue/index.js.map +1 -0
- package/fs/request-queue/memory.d.ts +9 -0
- package/fs/request-queue/memory.d.ts.map +1 -0
- package/fs/request-queue/memory.js +24 -0
- package/fs/request-queue/memory.js.map +1 -0
- package/package.json +8 -7
- package/resource-clients/dataset.d.ts.map +1 -1
- package/resource-clients/dataset.js +9 -12
- package/resource-clients/dataset.js.map +1 -1
- package/resource-clients/key-value-store.d.ts.map +1 -1
- package/resource-clients/key-value-store.js +15 -29
- package/resource-clients/key-value-store.js.map +1 -1
- package/resource-clients/request-queue.d.ts +0 -1
- package/resource-clients/request-queue.d.ts.map +1 -1
- package/resource-clients/request-queue.js +45 -44
- package/resource-clients/request-queue.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/utils.d.ts +3 -31
- package/utils.d.ts.map +1 -1
- package/utils.js.map +1 -1
- package/workers/worker-utils.d.ts +1 -0
- package/workers/worker-utils.d.ts.map +1 -1
- package/workers/worker-utils.js +2 -74
- package/workers/worker-utils.js.map +1 -1
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RequestQueueFileSystemEntry = void 0;
|
|
4
|
+
const fs_extra_1 = require("fs-extra");
|
|
5
|
+
const promises_1 = require("node:fs/promises");
|
|
6
|
+
const node_path_1 = require("node:path");
|
|
7
|
+
const worker_utils_1 = require("../../workers/worker-utils");
|
|
8
|
+
class RequestQueueFileSystemEntry {
|
|
9
|
+
constructor(options) {
|
|
10
|
+
Object.defineProperty(this, "filePath", {
|
|
11
|
+
enumerable: true,
|
|
12
|
+
configurable: true,
|
|
13
|
+
writable: true,
|
|
14
|
+
value: void 0
|
|
15
|
+
});
|
|
16
|
+
this.filePath = (0, node_path_1.resolve)(options.storeDirectory, `${options.requestId}.json`);
|
|
17
|
+
}
|
|
18
|
+
async get() {
|
|
19
|
+
return JSON.parse(await (0, promises_1.readFile)(this.filePath, 'utf-8'));
|
|
20
|
+
}
|
|
21
|
+
async update(data) {
|
|
22
|
+
await (0, fs_extra_1.ensureDir)((0, node_path_1.dirname)(this.filePath));
|
|
23
|
+
await (0, worker_utils_1.lockAndWrite)(this.filePath, data);
|
|
24
|
+
}
|
|
25
|
+
async delete() {
|
|
26
|
+
await (0, promises_1.rm)(this.filePath, { force: true });
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.RequestQueueFileSystemEntry = RequestQueueFileSystemEntry;
|
|
30
|
+
//# sourceMappingURL=fs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/request-queue/fs.ts"],"names":[],"mappings":";;;AAAA,uCAAqC;AACrC,+CAAgD;AAChD,yCAA6C;AAG7C,6DAA0D;AAG1D,MAAa,2BAA2B;IAGpC,YAAY,OAA2C;QAFvD;;;;;WAAyB;QAGrB,IAAI,CAAC,QAAQ,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC,SAAS,OAAO,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,GAAG;QACL,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAqB;QAC9B,MAAM,IAAA,oBAAS,EAAC,IAAA,mBAAO,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxC,MAAM,IAAA,2BAAY,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,IAAA,aAAE,EAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;CACJ;AAnBD,kEAmBC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { InternalRequest } from '../../resource-clients/request-queue';
|
|
2
|
+
import type { StorageImplementation } from '../common';
|
|
3
|
+
export declare function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions): StorageImplementation<InternalRequest>;
|
|
4
|
+
export interface CreateStorageImplementationOptions {
|
|
5
|
+
persistStorage: boolean;
|
|
6
|
+
storeDirectory: string;
|
|
7
|
+
requestId: string;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAIvD,wBAAgB,uCAAuC,CAAC,OAAO,EAAE,kCAAkC,GAAG,qBAAqB,CAAC,eAAe,CAAC,CAM3I;AAED,MAAM,WAAW,kCAAkC;IAC/C,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createRequestQueueStorageImplementation = void 0;
|
|
4
|
+
const fs_1 = require("./fs");
|
|
5
|
+
const memory_1 = require("./memory");
|
|
6
|
+
function createRequestQueueStorageImplementation(options) {
|
|
7
|
+
if (options.persistStorage) {
|
|
8
|
+
return new fs_1.RequestQueueFileSystemEntry(options);
|
|
9
|
+
}
|
|
10
|
+
return new memory_1.RequestQueueMemoryEntry();
|
|
11
|
+
}
|
|
12
|
+
exports.createRequestQueueStorageImplementation = createRequestQueueStorageImplementation;
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/request-queue/index.ts"],"names":[],"mappings":";;;AAEA,6BAAmD;AACnD,qCAAmD;AAEnD,SAAgB,uCAAuC,CAAC,OAA2C;IAC/F,IAAI,OAAO,CAAC,cAAc,EAAE;QACxB,OAAO,IAAI,gCAA2B,CAAC,OAAO,CAAC,CAAC;KACnD;IAED,OAAO,IAAI,gCAAuB,EAAE,CAAC;AACzC,CAAC;AAND,0FAMC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { InternalRequest } from '../../resource-clients/request-queue';
|
|
2
|
+
import type { StorageImplementation } from '../common';
|
|
3
|
+
export declare class RequestQueueMemoryEntry implements StorageImplementation<InternalRequest> {
|
|
4
|
+
private data;
|
|
5
|
+
get(): Promise<InternalRequest>;
|
|
6
|
+
update(data: InternalRequest): void;
|
|
7
|
+
delete(): void;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAEvD,qBAAa,uBAAwB,YAAW,qBAAqB,CAAC,eAAe,CAAC;IAClF,OAAO,CAAC,IAAI,CAAmB;IAEzB,GAAG;IAIT,MAAM,CAAC,IAAI,EAAE,eAAe;IAI5B,MAAM;CAGT"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RequestQueueMemoryEntry = void 0;
|
|
4
|
+
class RequestQueueMemoryEntry {
|
|
5
|
+
constructor() {
|
|
6
|
+
Object.defineProperty(this, "data", {
|
|
7
|
+
enumerable: true,
|
|
8
|
+
configurable: true,
|
|
9
|
+
writable: true,
|
|
10
|
+
value: void 0
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
async get() {
|
|
14
|
+
return this.data;
|
|
15
|
+
}
|
|
16
|
+
update(data) {
|
|
17
|
+
this.data = data;
|
|
18
|
+
}
|
|
19
|
+
delete() {
|
|
20
|
+
// No-op
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
exports.RequestQueueMemoryEntry = RequestQueueMemoryEntry;
|
|
24
|
+
//# sourceMappingURL=memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/fs/request-queue/memory.ts"],"names":[],"mappings":";;;AAGA,MAAa,uBAAuB;IAApC;QACI;;;;;WAA+B;IAanC,CAAC;IAXG,KAAK,CAAC,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,MAAM,CAAC,IAAqB;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,MAAM;QACF,QAAQ;IACZ,CAAC;CACJ;AAdD,0DAcC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/memory-storage",
|
|
3
|
-
"version": "3.1.5-beta.
|
|
3
|
+
"version": "3.1.5-beta.21",
|
|
4
4
|
"description": "A simple in-memory storage implementation of the Apify API",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">= 16"
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
},
|
|
40
40
|
"homepage": "https://crawlee.dev",
|
|
41
41
|
"scripts": {
|
|
42
|
-
"build": "
|
|
42
|
+
"build": "yarn clean && yarn compile && yarn copy",
|
|
43
43
|
"clean": "rimraf ./dist",
|
|
44
44
|
"compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
|
|
45
45
|
"copy": "ts-node -T ../../scripts/copy.ts"
|
|
@@ -49,14 +49,15 @@
|
|
|
49
49
|
},
|
|
50
50
|
"dependencies": {
|
|
51
51
|
"@apify/log": "^2.0.0",
|
|
52
|
-
"@crawlee/types": "^3.1.5-beta.
|
|
53
|
-
"@crawlee/utils": "^3.1.5-beta.
|
|
52
|
+
"@crawlee/types": "^3.1.5-beta.21",
|
|
53
|
+
"@crawlee/utils": "^3.1.5-beta.21",
|
|
54
54
|
"@sapphire/shapeshift": "^3.0.0",
|
|
55
55
|
"content-type": "^1.0.4",
|
|
56
|
-
"fs-extra": "^
|
|
57
|
-
"json5": "^2.2.
|
|
56
|
+
"fs-extra": "^11.0.0",
|
|
57
|
+
"json5": "^2.2.3",
|
|
58
58
|
"mime-types": "^2.1.35",
|
|
59
59
|
"proper-lockfile": "^4.1.2",
|
|
60
60
|
"tslib": "^2.4.0"
|
|
61
|
-
}
|
|
61
|
+
},
|
|
62
|
+
"gitHead": "02e61e35d42248aba8b8cd04b887ab1e3d8ddb0e"
|
|
62
63
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAkBlD,MAAM,WAAW,oBAAoB;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,UAAW,YAAW,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC;IACtH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,SAAS,SAAK;IACd,gBAAgB,EAAE,MAAM,CAAC;IAEzB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAkD;IACjF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,oBAAoB;IAOnC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAW/C,MAAM,CAAC,SAAS,GAAE,OAAO,CAAC,0BAA+B,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;IAsCxF,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAYvB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;IAIhC,SAAS,CAAC,OAAO,GAAE,OAAO,CAAC,wBAA6B,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA0C/F,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmCxE,aAAa,IAAI,OAAO,CAAC,WAAW;IAWpC,OAAO,CAAC,sBAAsB;IAI9B,OAAO,CAAC,qBAAqB;IAM7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,gBAAgB;CAkB3B"}
|
|
@@ -10,6 +10,7 @@ const consts_1 = require("../consts");
|
|
|
10
10
|
const base_client_1 = require("./common/base-client");
|
|
11
11
|
const instance_1 = require("../workers/instance");
|
|
12
12
|
const cache_helpers_1 = require("../cache-helpers");
|
|
13
|
+
const dataset_1 = require("../fs/dataset");
|
|
13
14
|
/**
|
|
14
15
|
* This is what API returns in the x-apify-pagination-limit
|
|
15
16
|
* header when no limit query parameter is used.
|
|
@@ -136,7 +137,7 @@ class DatasetClient extends base_client_1.BaseClient {
|
|
|
136
137
|
const items = [];
|
|
137
138
|
for (let idx = start; idx < end; idx++) {
|
|
138
139
|
const entryNumber = this.generateLocalEntryName(idx);
|
|
139
|
-
items.push(existingStoreById.datasetEntries.get(entryNumber));
|
|
140
|
+
items.push(await existingStoreById.datasetEntries.get(entryNumber).get());
|
|
140
141
|
}
|
|
141
142
|
existingStoreById.updateTimestamps(false);
|
|
142
143
|
return {
|
|
@@ -159,20 +160,16 @@ class DatasetClient extends base_client_1.BaseClient {
|
|
|
159
160
|
const addedIds = [];
|
|
160
161
|
for (const entry of normalized) {
|
|
161
162
|
const idx = this.generateLocalEntryName(++existingStoreById.itemCount);
|
|
162
|
-
|
|
163
|
+
const storageEntry = (0, dataset_1.createDatasetStorageImplementation)({
|
|
164
|
+
entityId: idx,
|
|
165
|
+
persistStorage: this.client.persistStorage,
|
|
166
|
+
storeDirectory: existingStoreById.datasetDirectory,
|
|
167
|
+
});
|
|
168
|
+
await storageEntry.update(entry);
|
|
169
|
+
existingStoreById.datasetEntries.set(idx, storageEntry);
|
|
163
170
|
addedIds.push(idx);
|
|
164
171
|
}
|
|
165
|
-
const dataEntries = addedIds.map((id) => [id, existingStoreById.datasetEntries.get(id)]);
|
|
166
172
|
existingStoreById.updateTimestamps(true);
|
|
167
|
-
(0, instance_1.sendWorkerMessage)({
|
|
168
|
-
data: dataEntries,
|
|
169
|
-
action: 'update-entries',
|
|
170
|
-
entityType: 'datasets',
|
|
171
|
-
entityDirectory: existingStoreById.datasetDirectory,
|
|
172
|
-
id: existingStoreById.name ?? existingStoreById.id,
|
|
173
|
-
writeMetadata: this.client.writeMetadata,
|
|
174
|
-
persistStorage: this.client.persistStorage,
|
|
175
|
-
});
|
|
176
173
|
}
|
|
177
174
|
toDatasetInfo() {
|
|
178
175
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":";;;AAGA,qDAAyC;AACzC,6CAAyC;AACzC,+CAAsC;AACtC,yCAAoC;AACpC,uCAAgC;AAEhC,sCAAyC;AACzC,sDAAkD;AAClD,kDAAwD;AACxD,oDAAkE;AAElE;;;GAGG;AACH,MAAM,gBAAgB,GAAG,YAAe,CAAC;AAEzC;;;GAGG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC;AASlC,MAAa,aAAoD,SAAQ,wBAAU;IAW/E,YAAY,OAA6B;QACrC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE,CAAC,CAAC;QAXtC;;;;;WAAc;QACd;;;;mBAAY,IAAI,IAAI,EAAE;WAAC;QACvB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAY,CAAC;WAAC;QACd;;;;;WAAyB;QAEzB;;;;mBAAkC,IAAI,GAAG,
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":";;;AAGA,qDAAyC;AACzC,6CAAyC;AACzC,+CAAsC;AACtC,yCAAoC;AACpC,uCAAgC;AAEhC,sCAAyC;AACzC,sDAAkD;AAClD,kDAAwD;AACxD,oDAAkE;AAElE,2CAAmE;AAEnE;;;GAGG;AACH,MAAM,gBAAgB,GAAG,YAAe,CAAC;AAEzC;;;GAGG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC;AASlC,MAAa,aAAoD,SAAQ,wBAAU;IAW/E,YAAY,OAA6B;QACrC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE,CAAC,CAAC;QAXtC;;;;;WAAc;QACd;;;;mBAAY,IAAI,IAAI,EAAE;WAAC;QACvB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAY,CAAC;WAAC;QACd;;;;;WAAyB;QAEzB;;;;mBAAkC,IAAI,GAAG,EAAuC;WAAC;QACjF;;;;;WAAuC;QAInC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,gBAAgB,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,KAAK,GAAG,MAAM,IAAA,8CAA8B,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtF,IAAI,KAAK,EAAE;YACP,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC,aAAa,EAAE,CAAC;SAChC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,YAAgD,EAAE;QAC3D,MAAM,MAAM,GAAG,cAAC,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ;SAC/C,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,8CAA8B,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,OAAO,CAAC,CAAC;SACjD;QAED,qBAAqB;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YACd,OAAO,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAC5C;QAED,wCAAwC;QACxC,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,MAAM,CAAC,IAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAExI,IAAI,mBAAmB,EAAE;YACrB,IAAI,CAAC,qBAAqB,CAAC,qBAAY,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;SACzE;QAED,iBAAiB,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAErC,MAAM,WAAW,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;QAEvD,iBAAiB,CAAC,gBAAgB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,IAAI,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAE3I,MAAM,IAAA,eAAI,EAAC,WAAW,EAAE,iBAAiB,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjF,oBAAoB;QACpB,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAEzC,OAAO,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;YACnB,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC5E,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;YACxB,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAEjC,MAAM,IAAA,aAAE,EAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E;IACL,CAAC;IAED,KAAK,CAAC,aAAa;QACf,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,UAA4C,EAAE;QAC1D,MAAM,EACF,KAAK,GAAG,gBAAgB,EACxB,MAAM,GAAG,CAAC,EACV,IAAI,GACP,GAAG,cAAC,CAAC,MAAM,CAAC;YACT,IAAI,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;YACxB,KAAK,EAAE,cAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ;YAC5B,MAAM,EAAE,cAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ;SAChC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,8CAA8B,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,OAAO,CAAC,CAAC;SACjD;QAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,iBAAiB,CAAC,qBAAqB,CACxD,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,SAAS,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EACzE,KAAK,CACR,CAAC;QAEF,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,EAAE;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC,GAAG,EAAE,CAAC,CAAC;SAC9E;QAED,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,IAAI,KAAK;YACnB,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK;YACrC,KAAK;YACL,MAAM;YACN,KAAK,EAAE,iBAAiB,CAAC,SAAS;SACrC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAwC;QACpD,MAAM,QAAQ,GAAG,cAAC,CAAC,KAAK,CACpB,cAAC,CAAC,MAAM,EACR,cAAC,CAAC,MAAM,CAAO,EAAU,CAAC,CAAC,WAAW,EACtC,cAAC,CAAC,KAAK,CAAC,cAAC,CAAC,KAAK,CAAC,cAAC,CAAC,MAAM,EAAE,cAAC,CAAC,MAAM,CAAO,EAAU,CAAC,CAAC,WAAW,CAAC,CAAC,CACrE,CAAC,KAAK,CAAC,KAAK,CAAW,CAAC;QAEzB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,8CAA8B,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,OAAO,CAAC,CAAC;SACjD;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAEjD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,YAAY,GAAG,IAAA,4CAAkC,EAAC;gBACpD,QAAQ,EAAE,GAAG;gBACb,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;gBAC1C,cAAc,EAAE,iBAAiB,CAAC,gBAAgB;aACrD,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjC,iBAAiB,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACxD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACtB;QAED,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,aAAa;QACT,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;SAClB,CAAC;IACN,CAAC;IAEO,sBAAsB,CAAC,GAAW;QACtC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAChE,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAU,CAAC;IACjC,CAAC;IAED;;;;;;OAMG;IACK,cAAc,CAAC,KAAwC;QAC3D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC3B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;SAC7B;QAED,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,IAAmB;QACrC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAS,CAAC;SACnC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,gFAAgF,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACxH;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,KAAK,CAAC,sDAAsD,IAAI,EAAE,CAAC,CAAC;SACjF;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;SAChC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,IAAI;YACJ,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,IAAI,CAAC,gBAAgB;YACtC,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE;YACxB,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACxC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;SAC7C,CAAC,CAAC;IACP,CAAC;CACJ;AAxOD,sCAwOC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key-value-store.d.ts","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAQ/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAI9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"key-value-store.d.ts","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAQ/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAI9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAQlD,MAAM,WAAW,0BAA0B;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,mBAAoB,SAAQ,UAAU;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,sBAAsB,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA+D;IAC/F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,0BAA0B;IAOzC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;IAWrD,MAAM,CAAC,SAAS,GAAE,OAAO,CAAC,gCAAqC,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAsCpG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAWvB,QAAQ,CAAC,OAAO,GAAE,OAAO,CAAC,8BAAmC,GAAG,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC;IA6D5G,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,mCAAwC,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IA4CnI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoE7D,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB9C,mBAAmB,IAAI,OAAO,CAAC,iBAAiB;IAWhD,OAAO,CAAC,gBAAgB;CAkB3B"}
|
|
@@ -15,6 +15,7 @@ const utils_1 = require("../utils");
|
|
|
15
15
|
const base_client_1 = require("./common/base-client");
|
|
16
16
|
const instance_1 = require("../workers/instance");
|
|
17
17
|
const cache_helpers_1 = require("../cache-helpers");
|
|
18
|
+
const key_value_store_1 = require("../fs/key-value-store");
|
|
18
19
|
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
|
|
19
20
|
class KeyValueStoreClient extends base_client_1.BaseClient {
|
|
20
21
|
constructor(options) {
|
|
@@ -118,7 +119,8 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
|
|
|
118
119
|
this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
|
|
119
120
|
}
|
|
120
121
|
const items = [];
|
|
121
|
-
for (const
|
|
122
|
+
for (const storageEntry of existingStoreById.keyValueEntries.values()) {
|
|
123
|
+
const record = await storageEntry.get();
|
|
122
124
|
const size = Buffer.byteLength(record.value);
|
|
123
125
|
items.push({
|
|
124
126
|
key: record.key,
|
|
@@ -167,10 +169,11 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
|
|
|
167
169
|
if (!existingStoreById) {
|
|
168
170
|
this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
|
|
169
171
|
}
|
|
170
|
-
const
|
|
171
|
-
if (!
|
|
172
|
+
const storageEntry = existingStoreById.keyValueEntries.get(key);
|
|
173
|
+
if (!storageEntry) {
|
|
172
174
|
return undefined;
|
|
173
175
|
}
|
|
176
|
+
const entry = await storageEntry.get();
|
|
174
177
|
const record = {
|
|
175
178
|
key: entry.key,
|
|
176
179
|
value: entry.value,
|
|
@@ -191,7 +194,7 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
|
|
|
191
194
|
async setRecord(record) {
|
|
192
195
|
shapeshift_1.s.object({
|
|
193
196
|
key: shapeshift_1.s.string.lengthGreaterThan(0),
|
|
194
|
-
value: shapeshift_1.s.union(shapeshift_1.s.null, shapeshift_1.s.string, shapeshift_1.s.number, shapeshift_1.s.object({}).passthrough),
|
|
197
|
+
value: shapeshift_1.s.union(shapeshift_1.s.null, shapeshift_1.s.string, shapeshift_1.s.number, shapeshift_1.s.instance(Buffer), shapeshift_1.s.object({}).passthrough),
|
|
195
198
|
contentType: shapeshift_1.s.string.lengthGreaterThan(0).optional,
|
|
196
199
|
}).parse(record);
|
|
197
200
|
// Check by id
|
|
@@ -230,26 +233,20 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
|
|
|
230
233
|
}
|
|
231
234
|
value = Buffer.concat(chunks);
|
|
232
235
|
}
|
|
233
|
-
const
|
|
236
|
+
const _record = {
|
|
234
237
|
extension,
|
|
235
238
|
key,
|
|
236
239
|
value,
|
|
237
240
|
contentType,
|
|
238
241
|
};
|
|
239
|
-
|
|
240
|
-
existingStoreById.updateTimestamps(true);
|
|
241
|
-
(0, instance_1.sendWorkerMessage)({
|
|
242
|
-
action: 'update-entries',
|
|
243
|
-
data: {
|
|
244
|
-
action: 'set',
|
|
245
|
-
record: entry,
|
|
246
|
-
},
|
|
247
|
-
entityType: 'keyValueStores',
|
|
248
|
-
entityDirectory: existingStoreById.keyValueStoreDirectory,
|
|
249
|
-
id: existingStoreById.name ?? existingStoreById.id,
|
|
250
|
-
writeMetadata: this.client.writeMetadata,
|
|
242
|
+
const entry = (0, key_value_store_1.createKeyValueStorageImplementation)({
|
|
251
243
|
persistStorage: this.client.persistStorage,
|
|
244
|
+
storeDirectory: existingStoreById.keyValueStoreDirectory,
|
|
245
|
+
writeMetadata: existingStoreById.client.writeMetadata,
|
|
252
246
|
});
|
|
247
|
+
await entry.update(_record);
|
|
248
|
+
existingStoreById.keyValueEntries.set(key, entry);
|
|
249
|
+
existingStoreById.updateTimestamps(true);
|
|
253
250
|
}
|
|
254
251
|
async deleteRecord(key) {
|
|
255
252
|
shapeshift_1.s.string.parse(key);
|
|
@@ -262,18 +259,7 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
|
|
|
262
259
|
if (entry) {
|
|
263
260
|
existingStoreById.keyValueEntries.delete(key);
|
|
264
261
|
existingStoreById.updateTimestamps(true);
|
|
265
|
-
|
|
266
|
-
action: 'update-entries',
|
|
267
|
-
data: {
|
|
268
|
-
action: 'delete',
|
|
269
|
-
record: entry,
|
|
270
|
-
},
|
|
271
|
-
entityType: 'keyValueStores',
|
|
272
|
-
entityDirectory: existingStoreById.keyValueStoreDirectory,
|
|
273
|
-
id: existingStoreById.name ?? existingStoreById.id,
|
|
274
|
-
writeMetadata: this.client.writeMetadata,
|
|
275
|
-
persistStorage: this.client.persistStorage,
|
|
276
|
-
});
|
|
262
|
+
await entry.delete();
|
|
277
263
|
}
|
|
278
264
|
}
|
|
279
265
|
toKeyValueStoreInfo() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key-value-store.js","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":";;;;AACA,qDAAyC;AACzC,oEAA8B;AAC9B,6CAAyC;AACzC,6CAAuC;AACvC,yCAAoC;AACpC,+CAAsC;AACtC,uCAAgC;AAEhC,gDAAgD;AAChD,sCAAkE;AAClE,oCAA8C;AAC9C,sDAAkD;AAClD,kDAAwD;AACxD,oDAAwE;AAExE,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAgB3C,MAAa,mBAAoB,SAAQ,wBAAU;IAU/C,YAAY,OAAmC;QAC3C,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE,CAAC,CAAC;QAVtC;;;;;WAAc;QACd;;;;mBAAY,IAAI,IAAI,EAAE;WAAC;QACvB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;;WAA+B;QAE/B;;;;mBAAmC,IAAI,GAAG,
|
|
1
|
+
{"version":3,"file":"key-value-store.js","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":";;;;AACA,qDAAyC;AACzC,oEAA8B;AAC9B,6CAAyC;AACzC,6CAAuC;AACvC,yCAAoC;AACpC,+CAAsC;AACtC,uCAAgC;AAEhC,gDAAgD;AAChD,sCAAkE;AAClE,oCAA8C;AAC9C,sDAAkD;AAClD,kDAAwD;AACxD,oDAAwE;AAExE,2DAA4E;AAE5E,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAgB3C,MAAa,mBAAoB,SAAQ,wBAAU;IAU/C,YAAY,OAAmC;QAC3C,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE,CAAC,CAAC;QAVtC;;;;;WAAc;QACd;;;;mBAAY,IAAI,IAAI,EAAE;WAAC;QACvB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;;WAA+B;QAE/B;;;;mBAAmC,IAAI,GAAG,EAAoD;WAAC;QAC/F;;;;;WAAuC;QAInC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,sBAAsB,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1F,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,KAAK,GAAG,MAAM,IAAA,oDAAoC,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5F,IAAI,KAAK,EAAE;YACP,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC,mBAAmB,EAAE,CAAC;SACtC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,YAAsD,EAAE;QACjE,MAAM,MAAM,GAAG,cAAC,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ;SAC/C,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,oDAAoC,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAExG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,qBAAqB;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YACd,OAAO,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAED,wCAAwC;QACxC,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,MAAM,CAAC,IAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAExI,IAAI,mBAAmB,EAAE;YACrB,IAAI,CAAC,qBAAqB,CAAC,qBAAY,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;SAC/E;QAED,iBAAiB,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAErC,MAAM,WAAW,GAAG,iBAAiB,CAAC,sBAAsB,CAAC;QAE7D,iBAAiB,CAAC,sBAAsB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,MAAM,CAAC,IAAI,IAAI,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAEvJ,MAAM,IAAA,eAAI,EAAC,WAAW,EAAE,iBAAiB,CAAC,sBAAsB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvF,oBAAoB;QACpB,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAEzC,OAAO,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;YACnB,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC5E,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAElC,MAAM,IAAA,aAAE,EAAC,SAAS,CAAC,sBAAsB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;SAChF;IACL,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAkD,EAAE;QAC/D,MAAM,EACF,KAAK,GAAG,gCAAuB,EAC/B,iBAAiB,GACpB,GAAG,cAAC,CAAC,MAAM,CAAC;YACT,KAAK,EAAE,cAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ;YACvC,iBAAiB,EAAE,cAAC,CAAC,MAAM,CAAC,QAAQ;SACvC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,oDAAoC,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAExG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,YAAY,IAAI,iBAAiB,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE;YACnE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC;YAExC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC;gBACP,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI;aACP,CAAC,CAAC;SACN;QAED,iCAAiC;QACjC,oFAAoF;QACpF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAChB,OAAO,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,iBAAiB,EAAE;YACnB,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC;YACzE,IAAI,MAAM,KAAK,CAAC,CAAC;gBAAE,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEpD,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChD,MAAM,gBAAgB,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/D,MAAM,gCAAgC,GAAG,eAAe,KAAK,gBAAgB,CAAC;QAC9E,MAAM,qBAAqB,GAAG,gCAAgC;YAC1D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC;QAE3B,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,KAAK;YACL,iBAAiB;YACjB,WAAW,EAAE,CAAC,gCAAgC;YAC9C,qBAAqB;YACrB,KAAK,EAAE,YAAY;SACtB,CAAC;IACN,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAuD,EAAE;QAClF,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpB,cAAC,CAAC,MAAM,CAAC;YACL,MAAM,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;YAC1B,2CAA2C;YAC3C,8CAA8C;YAC9C,MAAM,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;YAC1B,eAAe,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;SACtC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,oDAAoC,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAExG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,YAAY,GAAG,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEhE,IAAI,CAAC,YAAY,EAAE;YACf,OAAO,SAAS,CAAC;SACpB;QAED,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAgC;YACxC,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,oBAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAW;SAChF,CAAC;QAEF,IAAI,OAAO,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,GAAG,sBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SAC9C;aAAM,IAAI,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SAC5C;aAAM;YACH,MAAM,CAAC,KAAK,GAAG,IAAA,4BAAc,EAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAY,CAAC,CAAC;SACpE;QAED,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAmC;QAC/C,cAAC,CAAC,MAAM,CAAC;YACL,GAAG,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAClC,KAAK,EAAE,cAAC,CAAC,KAAK,CAAC,cAAC,CAAC,IAAI,EAAE,cAAC,CAAC,MAAM,EAAE,cAAC,CAAC,MAAM,EAAE,cAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC;YACxF,WAAW,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ;SACtD,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEjB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,oDAAoC,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAExG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;QACvB,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;QAEpC,MAAM,aAAa,GAAG,IAAA,gBAAQ,EAAC,KAAK,CAAC,CAAC;QAEtC,MAAM,qBAAqB,GAAG,aAAa,IAAI,IAAA,gBAAQ,EAAC,KAAK,CAAC,CAAC;QAC/D,iEAAiE;QACjE,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,qBAAqB;gBAAE,WAAW,GAAG,0BAA0B,CAAC;iBAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,WAAW,GAAG,2BAA2B,CAAC;;gBACzE,WAAW,GAAG,iCAAiC,CAAC;SACxD;QAED,MAAM,SAAS,GAAG,oBAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,4BAA4B,CAAC;QAE9E,MAAM,iBAAiB,GAAG,SAAS,KAAK,MAAM,CAAC;QAE/C,IAAI,iBAAiB,IAAI,CAAC,qBAAqB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC1E,IAAI;gBACA,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;aAC1C;YAAC,OAAO,GAAQ,EAAE;gBACf,MAAM,GAAG,GAAG,8FAA8F,GAAG,CAAC,OAAO,EAAE,CAAC;gBACxH,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;aACxB;SACJ;QAED,IAAI,aAAa,EAAE;YACf,MAAM,MAAM,GAAG,EAAE,CAAC;YAClB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE;gBAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACtB;YACD,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SACjC;QAED,MAAM,OAAO,GAAG;YACZ,SAAS;YACT,GAAG;YACH,KAAK;YACL,WAAW;SACc,CAAC;QAE9B,MAAM,KAAK,GAAG,IAAA,qDAAmC,EAAC;YAC9C,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAC1C,cAAc,EAAE,iBAAiB,CAAC,sBAAsB;YACxD,aAAa,EAAE,iBAAiB,CAAC,MAAM,CAAC,aAAa;SACxD,CAAC,CAAC;QAEH,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5B,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAElD,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,iBAAiB,GAAG,MAAM,IAAA,oDAAoC,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAExG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzD,IAAI,KAAK,EAAE;YACP,iBAAiB,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9C,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;SACxB;IACL,CAAC;IAED,mBAAmB;QACf,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,GAAG;SACd,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;SAChC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACxC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,IAAI;YACJ,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,IAAI,CAAC,sBAAsB;YAC5C,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE;YACxB,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACxC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;SAC7C,CAAC,CAAC;IACP,CAAC;CACJ;AA1SD,kDA0SC"}
|
|
@@ -40,7 +40,6 @@ export declare class RequestQueueClient extends BaseClient implements storage.Re
|
|
|
40
40
|
deleteRequest(id: string): Promise<void>;
|
|
41
41
|
toRequestQueueInfo(): storage.RequestQueueInfo;
|
|
42
42
|
private updateTimestamps;
|
|
43
|
-
private updateItem;
|
|
44
43
|
private _jsonToRequest;
|
|
45
44
|
private _createInternalRequest;
|
|
46
45
|
private _calculateOrderNo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request-queue.d.ts","sourceRoot":"","sources":["../../src/resource-clients/request-queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAM/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"request-queue.d.ts","sourceRoot":"","sources":["../../src/resource-clients/request-queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAM/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAuBlD,MAAM,WAAW,yBAAyB;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,kBAAmB,SAAQ,UAAW,YAAW,OAAO,CAAC,kBAAkB;IACpF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,mBAAmB,SAAK;IACxB,mBAAmB,SAAK;IACxB,qBAAqB,EAAE,MAAM,CAAC;IAE9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6D;IACtF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,yBAAyB;IAOxC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAWpD,MAAM,CAAC,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAuC/F,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAYvB,QAAQ,CAAC,OAAO,GAAE,OAAO,CAAC,WAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IAmCvE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,OAAO,GAAE,OAAO,CAAC,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAgDrH,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,EAAE,EAAE,OAAO,GAAE,OAAO,CAAC,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC;IA2DlI,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAcnE,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,mBAAmB,EAAE,OAAO,GAAE,OAAO,CAAC,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAoD9H,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB9C,kBAAkB,IAAI,OAAO,CAAC,gBAAgB;IAgB9C,OAAO,CAAC,gBAAgB;IAmBxB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,iBAAiB;CAO5B"}
|
|
@@ -11,6 +11,7 @@ const utils_1 = require("../utils");
|
|
|
11
11
|
const base_client_1 = require("./common/base-client");
|
|
12
12
|
const instance_1 = require("../workers/instance");
|
|
13
13
|
const cache_helpers_1 = require("../cache-helpers");
|
|
14
|
+
const request_queue_1 = require("../fs/request-queue");
|
|
14
15
|
const requestShape = shapeshift_1.s.object({
|
|
15
16
|
id: shapeshift_1.s.string,
|
|
16
17
|
url: shapeshift_1.s.string.url({ allowedProtocols: ['http:', 'https:'] }),
|
|
@@ -139,10 +140,11 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
139
140
|
}
|
|
140
141
|
existingQueueById.updateTimestamps(false);
|
|
141
142
|
const items = [];
|
|
142
|
-
for (const
|
|
143
|
+
for (const storageEntry of existingQueueById.requests.values()) {
|
|
143
144
|
if (items.length === limit) {
|
|
144
145
|
break;
|
|
145
146
|
}
|
|
147
|
+
const request = await storageEntry.get();
|
|
146
148
|
if (request.orderNo) {
|
|
147
149
|
items.push(request);
|
|
148
150
|
}
|
|
@@ -162,9 +164,10 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
162
164
|
this.throwOnNonExisting(consts_1.StorageTypes.RequestQueue);
|
|
163
165
|
}
|
|
164
166
|
const requestModel = this._createInternalRequest(request, options.forefront);
|
|
165
|
-
const
|
|
167
|
+
const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);
|
|
166
168
|
// We already have the request present, so we return information about it
|
|
167
|
-
if (
|
|
169
|
+
if (existingRequestWithIdEntry) {
|
|
170
|
+
const existingRequestWithId = await existingRequestWithIdEntry.get();
|
|
168
171
|
existingQueueById.updateTimestamps(false);
|
|
169
172
|
return {
|
|
170
173
|
requestId: existingRequestWithId.id,
|
|
@@ -172,10 +175,16 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
172
175
|
wasAlreadyPresent: true,
|
|
173
176
|
};
|
|
174
177
|
}
|
|
175
|
-
|
|
176
|
-
|
|
178
|
+
const newEntry = (0, request_queue_1.createRequestQueueStorageImplementation)({
|
|
179
|
+
persistStorage: existingQueueById.client.persistStorage,
|
|
180
|
+
requestId: requestModel.id,
|
|
181
|
+
storeDirectory: existingQueueById.client.requestQueuesDirectory,
|
|
182
|
+
});
|
|
183
|
+
await newEntry.update(requestModel);
|
|
184
|
+
existingQueueById.requests.set(requestModel.id, newEntry);
|
|
185
|
+
// We add 1 to pending requests if the request was not handled yet
|
|
186
|
+
existingQueueById.pendingRequestCount += requestModel.orderNo !== null ? 1 : 0;
|
|
177
187
|
existingQueueById.updateTimestamps(true);
|
|
178
|
-
existingQueueById.updateItem(requestModel);
|
|
179
188
|
return {
|
|
180
189
|
requestId: requestModel.id,
|
|
181
190
|
// We return wasAlreadyHandled: false even though the request may
|
|
@@ -197,8 +206,9 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
197
206
|
};
|
|
198
207
|
for (const model of requests) {
|
|
199
208
|
const requestModel = this._createInternalRequest(model, options.forefront);
|
|
200
|
-
const
|
|
201
|
-
if (
|
|
209
|
+
const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);
|
|
210
|
+
if (existingRequestWithIdEntry) {
|
|
211
|
+
const existingRequestWithId = await existingRequestWithIdEntry.get();
|
|
202
212
|
result.processedRequests.push({
|
|
203
213
|
requestId: existingRequestWithId.id,
|
|
204
214
|
uniqueKey: existingRequestWithId.uniqueKey,
|
|
@@ -207,8 +217,15 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
207
217
|
});
|
|
208
218
|
continue;
|
|
209
219
|
}
|
|
210
|
-
|
|
211
|
-
|
|
220
|
+
const newEntry = (0, request_queue_1.createRequestQueueStorageImplementation)({
|
|
221
|
+
persistStorage: existingQueueById.client.persistStorage,
|
|
222
|
+
requestId: requestModel.id,
|
|
223
|
+
storeDirectory: existingQueueById.client.requestQueuesDirectory,
|
|
224
|
+
});
|
|
225
|
+
await newEntry.update(requestModel);
|
|
226
|
+
existingQueueById.requests.set(requestModel.id, newEntry);
|
|
227
|
+
// We add 1 to pending requests if the request was not handled yet
|
|
228
|
+
existingQueueById.pendingRequestCount += requestModel.orderNo !== null ? 1 : 0;
|
|
212
229
|
result.processedRequests.push({
|
|
213
230
|
requestId: requestModel.id,
|
|
214
231
|
uniqueKey: requestModel.uniqueKey,
|
|
@@ -217,7 +234,6 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
217
234
|
wasAlreadyHandled: false,
|
|
218
235
|
wasAlreadyPresent: false,
|
|
219
236
|
});
|
|
220
|
-
existingQueueById.updateItem(requestModel);
|
|
221
237
|
}
|
|
222
238
|
existingQueueById.updateTimestamps(true);
|
|
223
239
|
return result;
|
|
@@ -229,7 +245,7 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
229
245
|
this.throwOnNonExisting(consts_1.StorageTypes.RequestQueue);
|
|
230
246
|
}
|
|
231
247
|
existingQueueById.updateTimestamps(false);
|
|
232
|
-
const json = existingQueueById.requests.get(id)?.json;
|
|
248
|
+
const json = (await existingQueueById.requests.get(id)?.get())?.json;
|
|
233
249
|
return this._jsonToRequest(json);
|
|
234
250
|
}
|
|
235
251
|
async updateRequest(request, options = {}) {
|
|
@@ -242,25 +258,28 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
242
258
|
const requestModel = this._createInternalRequest(request, options.forefront);
|
|
243
259
|
// First we need to check the existing request to be
|
|
244
260
|
// able to return information about its handled state.
|
|
245
|
-
const
|
|
261
|
+
const existingRequestEntry = existingQueueById.requests.get(requestModel.id);
|
|
246
262
|
// Undefined means that the request is not present in the queue.
|
|
247
263
|
// We need to insert it, to behave the same as API.
|
|
248
|
-
if (!
|
|
264
|
+
if (!existingRequestEntry) {
|
|
249
265
|
return this.addRequest(request, options);
|
|
250
266
|
}
|
|
267
|
+
const existingRequest = await existingRequestEntry.get();
|
|
268
|
+
const newEntry = (0, request_queue_1.createRequestQueueStorageImplementation)({
|
|
269
|
+
persistStorage: existingQueueById.client.persistStorage,
|
|
270
|
+
requestId: requestModel.id,
|
|
271
|
+
storeDirectory: existingQueueById.client.requestQueuesDirectory,
|
|
272
|
+
});
|
|
273
|
+
await newEntry.update(requestModel);
|
|
251
274
|
// When updating the request, we need to make sure that
|
|
252
275
|
// the handled counts are updated correctly in all cases.
|
|
253
|
-
existingQueueById.requests.set(requestModel.id,
|
|
254
|
-
let handledCountAdjustment = 0;
|
|
276
|
+
existingQueueById.requests.set(requestModel.id, newEntry);
|
|
255
277
|
const isRequestHandledStateChanging = typeof existingRequest.orderNo !== typeof requestModel.orderNo;
|
|
256
278
|
const requestWasHandledBeforeUpdate = existingRequest.orderNo === null;
|
|
257
|
-
if (isRequestHandledStateChanging)
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
handledCountAdjustment = -handledCountAdjustment;
|
|
261
|
-
existingQueueById.pendingRequestCount += handledCountAdjustment;
|
|
279
|
+
if (isRequestHandledStateChanging) {
|
|
280
|
+
existingQueueById.pendingRequestCount += requestWasHandledBeforeUpdate ? 1 : -1;
|
|
281
|
+
}
|
|
262
282
|
existingQueueById.updateTimestamps(true);
|
|
263
|
-
existingQueueById.updateItem(requestModel);
|
|
264
283
|
return {
|
|
265
284
|
requestId: requestModel.id,
|
|
266
285
|
wasAlreadyHandled: requestWasHandledBeforeUpdate,
|
|
@@ -272,20 +291,13 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
272
291
|
if (!existingQueueById) {
|
|
273
292
|
this.throwOnNonExisting(consts_1.StorageTypes.RequestQueue);
|
|
274
293
|
}
|
|
275
|
-
const
|
|
276
|
-
if (
|
|
294
|
+
const entry = existingQueueById.requests.get(id);
|
|
295
|
+
if (entry) {
|
|
296
|
+
const request = await entry.get();
|
|
277
297
|
existingQueueById.requests.delete(id);
|
|
278
298
|
existingQueueById.pendingRequestCount -= request.orderNo ? 1 : 0;
|
|
279
299
|
existingQueueById.updateTimestamps(true);
|
|
280
|
-
|
|
281
|
-
action: 'delete-entry',
|
|
282
|
-
data: { id },
|
|
283
|
-
entityType: 'requestQueues',
|
|
284
|
-
entityDirectory: existingQueueById.requestQueueDirectory,
|
|
285
|
-
id: existingQueueById.name ?? existingQueueById.id,
|
|
286
|
-
writeMetadata: existingQueueById.client.writeMetadata,
|
|
287
|
-
persistStorage: existingQueueById.client.persistStorage,
|
|
288
|
-
});
|
|
300
|
+
await entry.delete();
|
|
289
301
|
}
|
|
290
302
|
}
|
|
291
303
|
toRequestQueueInfo() {
|
|
@@ -319,17 +331,6 @@ class RequestQueueClient extends base_client_1.BaseClient {
|
|
|
319
331
|
persistStorage: this.client.persistStorage,
|
|
320
332
|
});
|
|
321
333
|
}
|
|
322
|
-
updateItem(item) {
|
|
323
|
-
(0, instance_1.sendWorkerMessage)({
|
|
324
|
-
action: 'update-entries',
|
|
325
|
-
data: item,
|
|
326
|
-
entityType: 'requestQueues',
|
|
327
|
-
entityDirectory: this.requestQueueDirectory,
|
|
328
|
-
id: this.name ?? this.id,
|
|
329
|
-
writeMetadata: this.client.writeMetadata,
|
|
330
|
-
persistStorage: this.client.persistStorage,
|
|
331
|
-
});
|
|
332
|
-
}
|
|
333
334
|
_jsonToRequest(requestJson) {
|
|
334
335
|
if (!requestJson)
|
|
335
336
|
return undefined;
|