@crawlee/memory-storage 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/background-handler/fs-utils.d.ts +1 -1
- package/background-handler/fs-utils.d.ts.map +1 -1
- package/background-handler/fs-utils.js +17 -23
- package/background-handler/fs-utils.js.map +1 -1
- package/background-handler/index.d.ts +1 -1
- package/background-handler/index.d.ts.map +1 -1
- package/background-handler/index.js +9 -13
- package/background-handler/index.js.map +1 -1
- package/body-parser.js +5 -9
- package/body-parser.js.map +1 -1
- package/cache-helpers.d.ts +4 -4
- package/cache-helpers.d.ts.map +1 -1
- package/cache-helpers.js +43 -49
- package/cache-helpers.js.map +1 -1
- package/consts.js +4 -7
- package/consts.js.map +1 -1
- package/fs/common.js +1 -2
- package/fs/dataset/fs.d.ts +2 -2
- package/fs/dataset/fs.d.ts.map +1 -1
- package/fs/dataset/fs.js +13 -27
- package/fs/dataset/fs.js.map +1 -1
- package/fs/dataset/index.d.ts +1 -1
- package/fs/dataset/index.d.ts.map +1 -1
- package/fs/dataset/index.js +5 -8
- package/fs/dataset/index.js.map +1 -1
- package/fs/dataset/memory.d.ts +1 -1
- package/fs/dataset/memory.d.ts.map +1 -1
- package/fs/dataset/memory.js +2 -13
- package/fs/dataset/memory.js.map +1 -1
- package/fs/key-value-store/fs.d.ts +3 -3
- package/fs/key-value-store/fs.d.ts.map +1 -1
- package/fs/key-value-store/fs.js +27 -62
- package/fs/key-value-store/fs.js.map +1 -1
- package/fs/key-value-store/index.d.ts +2 -2
- package/fs/key-value-store/index.d.ts.map +1 -1
- package/fs/key-value-store/index.js +5 -8
- package/fs/key-value-store/index.js.map +1 -1
- package/fs/key-value-store/memory.d.ts +2 -2
- package/fs/key-value-store/memory.d.ts.map +1 -1
- package/fs/key-value-store/memory.js +2 -13
- package/fs/key-value-store/memory.js.map +1 -1
- package/fs/request-queue/fs.d.ts +3 -3
- package/fs/request-queue/fs.d.ts.map +1 -1
- package/fs/request-queue/fs.js +23 -57
- package/fs/request-queue/fs.js.map +1 -1
- package/fs/request-queue/index.d.ts +2 -2
- package/fs/request-queue/index.d.ts.map +1 -1
- package/fs/request-queue/index.js +5 -8
- package/fs/request-queue/index.js.map +1 -1
- package/fs/request-queue/memory.d.ts +2 -2
- package/fs/request-queue/memory.d.ts.map +1 -1
- package/fs/request-queue/memory.js +3 -19
- package/fs/request-queue/memory.js.map +1 -1
- package/index.d.ts +1 -1
- package/index.d.ts.map +1 -1
- package/index.js +1 -4
- package/index.js.map +1 -1
- package/memory-storage.d.ts +3 -3
- package/memory-storage.d.ts.map +1 -1
- package/memory-storage.js +68 -117
- package/memory-storage.js.map +1 -1
- package/package.json +14 -20
- package/resource-clients/common/base-client.d.ts +1 -1
- package/resource-clients/common/base-client.d.ts.map +1 -1
- package/resource-clients/common/base-client.js +2 -11
- package/resource-clients/common/base-client.js.map +1 -1
- package/resource-clients/dataset-collection.d.ts +1 -1
- package/resource-clients/dataset-collection.d.ts.map +1 -1
- package/resource-clients/dataset-collection.js +13 -27
- package/resource-clients/dataset-collection.js.map +1 -1
- package/resource-clients/dataset.d.ts +2 -2
- package/resource-clients/dataset.d.ts.map +1 -1
- package/resource-clients/dataset.js +47 -87
- package/resource-clients/dataset.js.map +1 -1
- package/resource-clients/key-value-store-collection.d.ts +1 -1
- package/resource-clients/key-value-store-collection.d.ts.map +1 -1
- package/resource-clients/key-value-store-collection.js +13 -27
- package/resource-clients/key-value-store-collection.js.map +1 -1
- package/resource-clients/key-value-store.d.ts +2 -2
- package/resource-clients/key-value-store.d.ts.map +1 -1
- package/resource-clients/key-value-store.js +76 -109
- package/resource-clients/key-value-store.js.map +1 -1
- package/resource-clients/request-queue-collection.d.ts +1 -1
- package/resource-clients/request-queue-collection.d.ts.map +1 -1
- package/resource-clients/request-queue-collection.js +13 -27
- package/resource-clients/request-queue-collection.js.map +1 -1
- package/resource-clients/request-queue.d.ts +2 -2
- package/resource-clients/request-queue.d.ts.map +1 -1
- package/resource-clients/request-queue.js +81 -137
- package/resource-clients/request-queue.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/utils.js +12 -20
- package/utils.js.map +1 -1
- package/index.mjs +0 -4
package/fs/dataset/fs.js
CHANGED
|
@@ -1,31 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class DatasetFileSystemEntry {
|
|
1
|
+
import { readFile, rm } from 'node:fs/promises';
|
|
2
|
+
import { dirname, resolve } from 'node:path';
|
|
3
|
+
import { AsyncQueue } from '@sapphire/async-queue';
|
|
4
|
+
import { ensureDir } from 'fs-extra/esm';
|
|
5
|
+
import { lockAndWrite } from '../../background-handler/fs-utils.js';
|
|
6
|
+
export class DatasetFileSystemEntry {
|
|
7
|
+
filePath;
|
|
8
|
+
fsQueue = new AsyncQueue();
|
|
10
9
|
constructor(options) {
|
|
11
|
-
|
|
12
|
-
enumerable: true,
|
|
13
|
-
configurable: true,
|
|
14
|
-
writable: true,
|
|
15
|
-
value: void 0
|
|
16
|
-
});
|
|
17
|
-
Object.defineProperty(this, "fsQueue", {
|
|
18
|
-
enumerable: true,
|
|
19
|
-
configurable: true,
|
|
20
|
-
writable: true,
|
|
21
|
-
value: new async_queue_1.AsyncQueue()
|
|
22
|
-
});
|
|
23
|
-
this.filePath = (0, node_path_1.resolve)(options.storeDirectory, `${options.entityId}.json`);
|
|
10
|
+
this.filePath = resolve(options.storeDirectory, `${options.entityId}.json`);
|
|
24
11
|
}
|
|
25
12
|
async get() {
|
|
26
13
|
await this.fsQueue.wait();
|
|
27
14
|
try {
|
|
28
|
-
return JSON.parse(await
|
|
15
|
+
return JSON.parse(await readFile(this.filePath, 'utf-8'));
|
|
29
16
|
}
|
|
30
17
|
finally {
|
|
31
18
|
this.fsQueue.shift();
|
|
@@ -34,8 +21,8 @@ class DatasetFileSystemEntry {
|
|
|
34
21
|
async update(data) {
|
|
35
22
|
await this.fsQueue.wait();
|
|
36
23
|
try {
|
|
37
|
-
await
|
|
38
|
-
await
|
|
24
|
+
await ensureDir(dirname(this.filePath));
|
|
25
|
+
await lockAndWrite(this.filePath, data);
|
|
39
26
|
}
|
|
40
27
|
finally {
|
|
41
28
|
this.fsQueue.shift();
|
|
@@ -43,9 +30,8 @@ class DatasetFileSystemEntry {
|
|
|
43
30
|
}
|
|
44
31
|
async delete() {
|
|
45
32
|
await this.fsQueue.wait();
|
|
46
|
-
await
|
|
33
|
+
await rm(this.filePath, { force: true });
|
|
47
34
|
this.fsQueue.shift();
|
|
48
35
|
}
|
|
49
36
|
}
|
|
50
|
-
exports.DatasetFileSystemEntry = DatasetFileSystemEntry;
|
|
51
37
|
//# sourceMappingURL=fs.js.map
|
package/fs/dataset/fs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/dataset/fs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/dataset/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAIpE,MAAM,OAAO,sBAAsB;IACvB,QAAQ,CAAS;IACjB,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;IAEnC,YAAY,OAA2C;QACnD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC,QAAQ,OAAO,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9D,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC5C,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ"}
|
package/fs/dataset/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Dictionary } from '@crawlee/types';
|
|
2
|
-
import type { StorageImplementation } from '../common';
|
|
2
|
+
import type { StorageImplementation } from '../common.js';
|
|
3
3
|
export declare function createDatasetStorageImplementation<Data extends Dictionary>(options: CreateStorageImplementationOptions): StorageImplementation<Data>;
|
|
4
4
|
export interface CreateStorageImplementationOptions {
|
|
5
5
|
persistStorage: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/dataset/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/dataset/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAI1D,wBAAgB,kCAAkC,CAAC,IAAI,SAAS,UAAU,EACtE,OAAO,EAAE,kCAAkC,GAC5C,qBAAqB,CAAC,IAAI,CAAC,CAM7B;AAED,MAAM,WAAW,kCAAkC;IAC/C,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;CACpB"}
|
package/fs/dataset/index.js
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const fs_1 = require("./fs");
|
|
5
|
-
const memory_1 = require("./memory");
|
|
6
|
-
function createDatasetStorageImplementation(options) {
|
|
1
|
+
import { DatasetFileSystemEntry } from './fs.js';
|
|
2
|
+
import { DatasetMemoryEntry } from './memory.js';
|
|
3
|
+
export function createDatasetStorageImplementation(options) {
|
|
7
4
|
if (options.persistStorage) {
|
|
8
|
-
return new
|
|
5
|
+
return new DatasetFileSystemEntry(options);
|
|
9
6
|
}
|
|
10
|
-
return new
|
|
7
|
+
return new DatasetMemoryEntry();
|
|
11
8
|
}
|
|
12
9
|
//# sourceMappingURL=index.js.map
|
package/fs/dataset/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/dataset/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/dataset/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,UAAU,kCAAkC,CAC9C,OAA2C;IAE3C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,OAAO,IAAI,sBAAsB,CAAO,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,IAAI,kBAAkB,EAAQ,CAAC;AAC1C,CAAC"}
|
package/fs/dataset/memory.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/dataset/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/dataset/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE1D,qBAAa,kBAAkB,CAAC,IAAI,CAAE,YAAW,qBAAqB,CAAC,IAAI,CAAC;IACxE,OAAO,CAAC,IAAI,CAAQ;IAEd,GAAG;IAIT,MAAM,CAAC,IAAI,EAAE,IAAI;IAIjB,MAAM;CAGT"}
|
package/fs/dataset/memory.js
CHANGED
|
@@ -1,15 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.DatasetMemoryEntry = void 0;
|
|
4
|
-
class DatasetMemoryEntry {
|
|
5
|
-
constructor() {
|
|
6
|
-
Object.defineProperty(this, "data", {
|
|
7
|
-
enumerable: true,
|
|
8
|
-
configurable: true,
|
|
9
|
-
writable: true,
|
|
10
|
-
value: void 0
|
|
11
|
-
});
|
|
12
|
-
}
|
|
1
|
+
export class DatasetMemoryEntry {
|
|
2
|
+
data;
|
|
13
3
|
async get() {
|
|
14
4
|
return this.data;
|
|
15
5
|
}
|
|
@@ -20,5 +10,4 @@ class DatasetMemoryEntry {
|
|
|
20
10
|
// No-op
|
|
21
11
|
}
|
|
22
12
|
}
|
|
23
|
-
exports.DatasetMemoryEntry = DatasetMemoryEntry;
|
|
24
13
|
//# sourceMappingURL=memory.js.map
|
package/fs/dataset/memory.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/fs/dataset/memory.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/fs/dataset/memory.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,kBAAkB;IACnB,IAAI,CAAQ;IAEpB,KAAK,CAAC,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,MAAM,CAAC,IAAU;QACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,MAAM;QACF,QAAQ;IACZ,CAAC;CACJ"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
|
|
2
|
-
import type { StorageImplementation } from '../common';
|
|
3
|
-
import type { CreateStorageImplementationOptions } from '.';
|
|
1
|
+
import type { InternalKeyRecord } from '../../resource-clients/key-value-store.js';
|
|
2
|
+
import type { StorageImplementation } from '../common.js';
|
|
3
|
+
import type { CreateStorageImplementationOptions } from './index.js';
|
|
4
4
|
export declare class KeyValueFileSystemEntry implements StorageImplementation<InternalKeyRecord> {
|
|
5
5
|
private storeDirectory;
|
|
6
6
|
private writeMetadata;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../../../src/fs/key-value-store/fs.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../../../src/fs/key-value-store/fs.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AAEnF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,kCAAkC,EAAE,MAAM,YAAY,CAAC;AAErE,qBAAa,uBAAwB,YAAW,qBAAqB,CAAC,iBAAiB,CAAC;IACpF,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,aAAa,CAAU;IAE/B,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,SAAS,CAAoC;IACrD,OAAO,CAAC,OAAO,CAAoB;gBAEvB,OAAO,EAAE,kCAAkC;IAKjD,GAAG,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAiCjC,MAAM,CAAC,IAAI,EAAE,iBAAiB;IA2B9B,MAAM;CAMf"}
|
package/fs/key-value-store/fs.js
CHANGED
|
@@ -1,53 +1,19 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
1
|
+
import { readFile, rm } from 'node:fs/promises';
|
|
2
|
+
import { dirname, resolve } from 'node:path';
|
|
3
|
+
import { basename } from 'node:path/win32';
|
|
4
|
+
import { AsyncQueue } from '@sapphire/async-queue';
|
|
5
|
+
import { ensureDir } from 'fs-extra/esm';
|
|
6
|
+
import mime from 'mime-types';
|
|
7
|
+
import { lockAndWrite } from '../../background-handler/fs-utils.js';
|
|
8
|
+
import { memoryStorageLog } from '../../utils.js';
|
|
9
|
+
export class KeyValueFileSystemEntry {
|
|
10
|
+
storeDirectory;
|
|
11
|
+
writeMetadata;
|
|
12
|
+
filePath;
|
|
13
|
+
fileMetadataPath;
|
|
14
|
+
rawRecord;
|
|
15
|
+
fsQueue = new AsyncQueue();
|
|
14
16
|
constructor(options) {
|
|
15
|
-
Object.defineProperty(this, "storeDirectory", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "writeMetadata", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
27
|
-
Object.defineProperty(this, "filePath", {
|
|
28
|
-
enumerable: true,
|
|
29
|
-
configurable: true,
|
|
30
|
-
writable: true,
|
|
31
|
-
value: void 0
|
|
32
|
-
});
|
|
33
|
-
Object.defineProperty(this, "fileMetadataPath", {
|
|
34
|
-
enumerable: true,
|
|
35
|
-
configurable: true,
|
|
36
|
-
writable: true,
|
|
37
|
-
value: void 0
|
|
38
|
-
});
|
|
39
|
-
Object.defineProperty(this, "rawRecord", {
|
|
40
|
-
enumerable: true,
|
|
41
|
-
configurable: true,
|
|
42
|
-
writable: true,
|
|
43
|
-
value: void 0
|
|
44
|
-
});
|
|
45
|
-
Object.defineProperty(this, "fsQueue", {
|
|
46
|
-
enumerable: true,
|
|
47
|
-
configurable: true,
|
|
48
|
-
writable: true,
|
|
49
|
-
value: new async_queue_1.AsyncQueue()
|
|
50
|
-
});
|
|
51
17
|
this.storeDirectory = options.storeDirectory;
|
|
52
18
|
this.writeMetadata = options.writeMetadata;
|
|
53
19
|
}
|
|
@@ -55,14 +21,14 @@ class KeyValueFileSystemEntry {
|
|
|
55
21
|
await this.fsQueue.wait();
|
|
56
22
|
let file;
|
|
57
23
|
try {
|
|
58
|
-
file = await
|
|
24
|
+
file = await readFile(this.filePath);
|
|
59
25
|
}
|
|
60
26
|
catch {
|
|
61
27
|
try {
|
|
62
28
|
// Try without extension
|
|
63
|
-
file = await
|
|
64
|
-
|
|
65
|
-
`Key-value entry "${this.rawRecord.key}" for store ${
|
|
29
|
+
file = await readFile(resolve(this.storeDirectory, this.rawRecord.key));
|
|
30
|
+
memoryStorageLog.warning([
|
|
31
|
+
`Key-value entry "${this.rawRecord.key}" for store ${basename(this.storeDirectory)} does not have a file extension, assuming it as text.`,
|
|
66
32
|
'If you want to have correct interpretation of the file, you should add a file extension to the entry.',
|
|
67
33
|
].join('\n'));
|
|
68
34
|
file = file.toString('utf-8');
|
|
@@ -82,21 +48,21 @@ class KeyValueFileSystemEntry {
|
|
|
82
48
|
}
|
|
83
49
|
async update(data) {
|
|
84
50
|
await this.fsQueue.wait();
|
|
85
|
-
const contentType =
|
|
51
|
+
const contentType = mime.contentType(data.key);
|
|
86
52
|
const fileName =
|
|
87
53
|
// the content type might include charset, e.g. `text/html; charset=utf-8`, so we check via `startsWith` instead of `===`
|
|
88
54
|
contentType && data.contentType && contentType.startsWith(data.contentType)
|
|
89
55
|
? data.key
|
|
90
56
|
: `${data.key}.${data.extension}`;
|
|
91
|
-
this.filePath
|
|
92
|
-
this.fileMetadataPath
|
|
57
|
+
this.filePath ??= resolve(this.storeDirectory, fileName);
|
|
58
|
+
this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);
|
|
93
59
|
const { value, ...rest } = data;
|
|
94
60
|
this.rawRecord = rest;
|
|
95
61
|
try {
|
|
96
|
-
await
|
|
97
|
-
await
|
|
62
|
+
await ensureDir(dirname(this.filePath));
|
|
63
|
+
await lockAndWrite(this.filePath, value, false);
|
|
98
64
|
if (this.writeMetadata) {
|
|
99
|
-
await
|
|
65
|
+
await lockAndWrite(this.fileMetadataPath, JSON.stringify(rest), true);
|
|
100
66
|
}
|
|
101
67
|
}
|
|
102
68
|
finally {
|
|
@@ -105,10 +71,9 @@ class KeyValueFileSystemEntry {
|
|
|
105
71
|
}
|
|
106
72
|
async delete() {
|
|
107
73
|
await this.fsQueue.wait();
|
|
108
|
-
await
|
|
109
|
-
await
|
|
74
|
+
await rm(this.filePath, { force: true });
|
|
75
|
+
await rm(this.fileMetadataPath, { force: true });
|
|
110
76
|
this.fsQueue.shift();
|
|
111
77
|
}
|
|
112
78
|
}
|
|
113
|
-
exports.KeyValueFileSystemEntry = KeyValueFileSystemEntry;
|
|
114
79
|
//# sourceMappingURL=fs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/key-value-store/fs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/key-value-store/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,IAAI,MAAM,YAAY,CAAC;AAE9B,OAAO,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAEpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAIlD,MAAM,OAAO,uBAAuB;IACxB,cAAc,CAAS;IACvB,aAAa,CAAU;IAEvB,QAAQ,CAAU;IAClB,gBAAgB,CAAU;IAC1B,SAAS,CAAoC;IAC7C,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;IAEnC,YAAY,OAA2C;QACnD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,IAAqB,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACL,IAAI,CAAC;gBACD,wBAAwB;gBACxB,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxE,gBAAgB,CAAC,OAAO,CACpB;oBACI,oBAAoB,IAAI,CAAC,SAAS,CAAC,GAAG,eAAe,QAAQ,CACzD,IAAI,CAAC,cAAc,CACtB,uDAAuD;oBACxD,uGAAuG;iBAC1G,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;gBACF,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YAAC,MAAM,CAAC;gBACL,iDAAiD;gBACjD,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/D,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;QAED,OAAO;YACH,GAAG,IAAI,CAAC,SAAS;YACjB,KAAK,EAAE,IAAI;SACd,CAAC;IACN,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAuB;QAChC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ;QACV,yHAAyH;QACzH,WAAW,IAAI,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC;YACvE,CAAC,CAAC,IAAI,CAAC,GAAG;YACV,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAE1C,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QACzD,IAAI,CAAC,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC;QAExF,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACD,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAEhD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACrB,MAAM,YAAY,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,MAAM,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
|
|
2
|
-
import type { StorageImplementation } from '../common';
|
|
1
|
+
import type { InternalKeyRecord } from '../../resource-clients/key-value-store.js';
|
|
2
|
+
import type { StorageImplementation } from '../common.js';
|
|
3
3
|
export declare function createKeyValueStorageImplementation(options: CreateStorageImplementationOptions): StorageImplementation<InternalKeyRecord>;
|
|
4
4
|
export interface CreateStorageImplementationOptions {
|
|
5
5
|
persistStorage: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/key-value-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/key-value-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AACnF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAI1D,wBAAgB,mCAAmC,CAC/C,OAAO,EAAE,kCAAkC,GAC5C,qBAAqB,CAAC,iBAAiB,CAAC,CAM1C;AAED,MAAM,WAAW,kCAAkC;IAC/C,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;CAC1B"}
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const fs_1 = require("./fs");
|
|
5
|
-
const memory_1 = require("./memory");
|
|
6
|
-
function createKeyValueStorageImplementation(options) {
|
|
1
|
+
import { KeyValueFileSystemEntry } from './fs.js';
|
|
2
|
+
import { KeyValueMemoryEntry } from './memory.js';
|
|
3
|
+
export function createKeyValueStorageImplementation(options) {
|
|
7
4
|
if (options.persistStorage) {
|
|
8
|
-
return new
|
|
5
|
+
return new KeyValueFileSystemEntry(options);
|
|
9
6
|
}
|
|
10
|
-
return new
|
|
7
|
+
return new KeyValueMemoryEntry();
|
|
11
8
|
}
|
|
12
9
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/key-value-store/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/key-value-store/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,UAAU,mCAAmC,CAC/C,OAA2C;IAE3C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,OAAO,IAAI,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,IAAI,mBAAmB,EAAE,CAAC;AACrC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
|
|
2
|
-
import type { StorageImplementation } from '../common';
|
|
1
|
+
import type { InternalKeyRecord } from '../../resource-clients/key-value-store.js';
|
|
2
|
+
import type { StorageImplementation } from '../common.js';
|
|
3
3
|
export declare class KeyValueMemoryEntry implements StorageImplementation<InternalKeyRecord> {
|
|
4
4
|
private data;
|
|
5
5
|
get(): Promise<InternalKeyRecord>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/key-value-store/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/key-value-store/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AACnF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE1D,qBAAa,mBAAoB,YAAW,qBAAqB,CAAC,iBAAiB,CAAC;IAChF,OAAO,CAAC,IAAI,CAAqB;IAE3B,GAAG;IAIT,MAAM,CAAC,IAAI,EAAE,iBAAiB;IAI9B,MAAM;CAGT"}
|
|
@@ -1,15 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.KeyValueMemoryEntry = void 0;
|
|
4
|
-
class KeyValueMemoryEntry {
|
|
5
|
-
constructor() {
|
|
6
|
-
Object.defineProperty(this, "data", {
|
|
7
|
-
enumerable: true,
|
|
8
|
-
configurable: true,
|
|
9
|
-
writable: true,
|
|
10
|
-
value: void 0
|
|
11
|
-
});
|
|
12
|
-
}
|
|
1
|
+
export class KeyValueMemoryEntry {
|
|
2
|
+
data;
|
|
13
3
|
async get() {
|
|
14
4
|
return this.data;
|
|
15
5
|
}
|
|
@@ -20,5 +10,4 @@ class KeyValueMemoryEntry {
|
|
|
20
10
|
// No-op
|
|
21
11
|
}
|
|
22
12
|
}
|
|
23
|
-
exports.KeyValueMemoryEntry = KeyValueMemoryEntry;
|
|
24
13
|
//# sourceMappingURL=memory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/fs/key-value-store/memory.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/fs/key-value-store/memory.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,mBAAmB;IACpB,IAAI,CAAqB;IAEjC,KAAK,CAAC,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,MAAM,CAAC,IAAuB;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,MAAM;QACF,QAAQ;IACZ,CAAC;CACJ"}
|
package/fs/request-queue/fs.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { InternalRequest } from '../../resource-clients/request-queue';
|
|
2
|
-
import type { StorageImplementation } from '../common';
|
|
3
|
-
import type { CreateStorageImplementationOptions } from '.';
|
|
1
|
+
import type { InternalRequest } from '../../resource-clients/request-queue.js';
|
|
2
|
+
import type { StorageImplementation } from '../common.js';
|
|
3
|
+
import type { CreateStorageImplementationOptions } from './index.js';
|
|
4
4
|
export declare class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {
|
|
5
5
|
private filePath;
|
|
6
6
|
private fsQueue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/fs.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/fs.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,kCAAkC,EAAE,MAAM,YAAY,CAAC;AAErE,qBAAa,2BAA4B,YAAW,qBAAqB,CAAC,eAAe,CAAC;IACtF,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,IAAI,CAAC,CAAkB;IAC/B,OAAO,CAAC,eAAe,CAAS;IAEhC;;;OAGG;IACH,OAAO,CAAC,YAAY,CAAC,CAAiB;IAE/B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEnB,OAAO,EAAE,kCAAkC;IAIjD,GAAG,CAAC,KAAK,UAAQ;IAuBjB,MAAM,CAAC,IAAI,EAAE,eAAe;IAmB5B,MAAM;IAMZ,OAAO,CAAC,wBAAwB;CAUnC"}
|
package/fs/request-queue/fs.js
CHANGED
|
@@ -1,54 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
import { readFile, rm } from 'node:fs/promises';
|
|
2
|
+
import { dirname, resolve } from 'node:path';
|
|
3
|
+
import { AsyncQueue } from '@sapphire/async-queue';
|
|
4
|
+
import { ensureDir } from 'fs-extra/esm';
|
|
5
|
+
import { lockAndCallback, lockAndWrite } from '../../background-handler/fs-utils.js';
|
|
6
|
+
export class RequestQueueFileSystemEntry {
|
|
7
|
+
filePath;
|
|
8
|
+
fsQueue = new AsyncQueue();
|
|
9
|
+
data;
|
|
10
|
+
directoryExists = false;
|
|
11
|
+
/**
|
|
12
|
+
* A "sweep" timeout that is created/refreshed whenever this entry is accessed/updated.
|
|
13
|
+
* It exists to ensure that the entry is not kept in memory indefinitely, by sweeping it after 60 seconds of inactivity (in order to keep memory usage low)
|
|
14
|
+
*/
|
|
15
|
+
sweepTimeout;
|
|
16
|
+
orderNo;
|
|
10
17
|
constructor(options) {
|
|
11
|
-
|
|
12
|
-
enumerable: true,
|
|
13
|
-
configurable: true,
|
|
14
|
-
writable: true,
|
|
15
|
-
value: void 0
|
|
16
|
-
});
|
|
17
|
-
Object.defineProperty(this, "fsQueue", {
|
|
18
|
-
enumerable: true,
|
|
19
|
-
configurable: true,
|
|
20
|
-
writable: true,
|
|
21
|
-
value: new async_queue_1.AsyncQueue()
|
|
22
|
-
});
|
|
23
|
-
Object.defineProperty(this, "data", {
|
|
24
|
-
enumerable: true,
|
|
25
|
-
configurable: true,
|
|
26
|
-
writable: true,
|
|
27
|
-
value: void 0
|
|
28
|
-
});
|
|
29
|
-
Object.defineProperty(this, "directoryExists", {
|
|
30
|
-
enumerable: true,
|
|
31
|
-
configurable: true,
|
|
32
|
-
writable: true,
|
|
33
|
-
value: false
|
|
34
|
-
});
|
|
35
|
-
/**
|
|
36
|
-
* A "sweep" timeout that is created/refreshed whenever this entry is accessed/updated.
|
|
37
|
-
* It exists to ensure that the entry is not kept in memory indefinitely, by sweeping it after 60 seconds of inactivity (in order to keep memory usage low)
|
|
38
|
-
*/
|
|
39
|
-
Object.defineProperty(this, "sweepTimeout", {
|
|
40
|
-
enumerable: true,
|
|
41
|
-
configurable: true,
|
|
42
|
-
writable: true,
|
|
43
|
-
value: void 0
|
|
44
|
-
});
|
|
45
|
-
Object.defineProperty(this, "orderNo", {
|
|
46
|
-
enumerable: true,
|
|
47
|
-
configurable: true,
|
|
48
|
-
writable: true,
|
|
49
|
-
value: void 0
|
|
50
|
-
});
|
|
51
|
-
this.filePath = (0, node_path_1.resolve)(options.storeDirectory, `${options.requestId}.json`);
|
|
18
|
+
this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);
|
|
52
19
|
}
|
|
53
20
|
async get(force = false) {
|
|
54
21
|
await this.fsQueue.wait();
|
|
@@ -58,8 +25,8 @@ class RequestQueueFileSystemEntry {
|
|
|
58
25
|
return this.data;
|
|
59
26
|
}
|
|
60
27
|
try {
|
|
61
|
-
return await
|
|
62
|
-
const req = JSON.parse(await
|
|
28
|
+
return await lockAndCallback(this.filePath, async () => {
|
|
29
|
+
const req = JSON.parse(await readFile(this.filePath, 'utf-8'));
|
|
63
30
|
this.data = req;
|
|
64
31
|
this.orderNo = req.orderNo;
|
|
65
32
|
return req;
|
|
@@ -74,10 +41,10 @@ class RequestQueueFileSystemEntry {
|
|
|
74
41
|
this.data = data;
|
|
75
42
|
try {
|
|
76
43
|
if (!this.directoryExists) {
|
|
77
|
-
await
|
|
44
|
+
await ensureDir(dirname(this.filePath));
|
|
78
45
|
this.directoryExists = true;
|
|
79
46
|
}
|
|
80
|
-
await
|
|
47
|
+
await lockAndWrite(this.filePath, data);
|
|
81
48
|
this.orderNo = data.orderNo;
|
|
82
49
|
}
|
|
83
50
|
finally {
|
|
@@ -87,7 +54,7 @@ class RequestQueueFileSystemEntry {
|
|
|
87
54
|
}
|
|
88
55
|
async delete() {
|
|
89
56
|
await this.fsQueue.wait();
|
|
90
|
-
await
|
|
57
|
+
await rm(this.filePath, { force: true });
|
|
91
58
|
this.fsQueue.shift();
|
|
92
59
|
}
|
|
93
60
|
setOrRefreshSweepTimeout() {
|
|
@@ -98,9 +65,8 @@ class RequestQueueFileSystemEntry {
|
|
|
98
65
|
this.sweepTimeout = setTimeout(() => {
|
|
99
66
|
this.sweepTimeout = undefined;
|
|
100
67
|
this.data = undefined;
|
|
101
|
-
},
|
|
68
|
+
}, 60_000).unref();
|
|
102
69
|
}
|
|
103
70
|
}
|
|
104
71
|
}
|
|
105
|
-
exports.RequestQueueFileSystemEntry = RequestQueueFileSystemEntry;
|
|
106
72
|
//# sourceMappingURL=fs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/request-queue/fs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../../src/fs/request-queue/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAKrF,MAAM,OAAO,2BAA2B;IAC5B,QAAQ,CAAS;IACjB,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;IAC3B,IAAI,CAAmB;IACvB,eAAe,GAAG,KAAK,CAAC;IAEhC;;;OAGG;IACK,YAAY,CAAkB;IAE/B,OAAO,CAAiB;IAE/B,YAAY,OAA2C;QACnD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC,SAAS,OAAO,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEhC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,CAAC;YACD,OAAO,MAAM,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACnD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/D,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;gBAEhB,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;gBAE3B,OAAO,GAAG,CAAC;YACf,CAAC,CAAC,CAAC;QACP,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAqB;QAC9B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACxB,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACxC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAChC,CAAC;YAED,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAExC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAChC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAEO,wBAAwB;QAC5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;gBAC9B,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YAC1B,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RequestQueueFileSystemEntry } from './fs';
|
|
2
|
-
import { RequestQueueMemoryEntry } from './memory';
|
|
1
|
+
import { RequestQueueFileSystemEntry } from './fs.js';
|
|
2
|
+
import { RequestQueueMemoryEntry } from './memory.js';
|
|
3
3
|
export declare function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions): RequestQueueMemoryEntry | RequestQueueFileSystemEntry;
|
|
4
4
|
export interface CreateStorageImplementationOptions {
|
|
5
5
|
persistStorage: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAEtD,wBAAgB,uCAAuC,CAAC,OAAO,EAAE,kCAAkC,yDAMlG;AAED,MAAM,WAAW,kCAAkC;IAC/C,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const fs_1 = require("./fs");
|
|
5
|
-
const memory_1 = require("./memory");
|
|
6
|
-
function createRequestQueueStorageImplementation(options) {
|
|
1
|
+
import { RequestQueueFileSystemEntry } from './fs.js';
|
|
2
|
+
import { RequestQueueMemoryEntry } from './memory.js';
|
|
3
|
+
export function createRequestQueueStorageImplementation(options) {
|
|
7
4
|
if (options.persistStorage) {
|
|
8
|
-
return new
|
|
5
|
+
return new RequestQueueFileSystemEntry(options);
|
|
9
6
|
}
|
|
10
|
-
return new
|
|
7
|
+
return new RequestQueueMemoryEntry();
|
|
11
8
|
}
|
|
12
9
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/request-queue/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/fs/request-queue/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAEtD,MAAM,UAAU,uCAAuC,CAAC,OAA2C;IAC/F,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,OAAO,IAAI,2BAA2B,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,IAAI,uBAAuB,EAAE,CAAC;AACzC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { InternalRequest } from '../../resource-clients/request-queue';
|
|
2
|
-
import type { StorageImplementation } from '../common';
|
|
1
|
+
import type { InternalRequest } from '../../resource-clients/request-queue.js';
|
|
2
|
+
import type { StorageImplementation } from '../common.js';
|
|
3
3
|
export declare class RequestQueueMemoryEntry implements StorageImplementation<InternalRequest> {
|
|
4
4
|
private data;
|
|
5
5
|
orderNo?: number | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/fs/request-queue/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE1D,qBAAa,uBAAwB,YAAW,qBAAqB,CAAC,eAAe,CAAC;IAClF,OAAO,CAAC,IAAI,CAAmB;IAExB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB,GAAG;IAIT,MAAM,CAAC,IAAI,EAAE,eAAe;IAK5B,MAAM;CAGT"}
|