@crawlee/memory-storage 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/background-handler/fs-utils.d.ts +1 -1
- package/background-handler/fs-utils.d.ts.map +1 -1
- package/background-handler/fs-utils.js +17 -23
- package/background-handler/fs-utils.js.map +1 -1
- package/background-handler/index.d.ts +1 -1
- package/background-handler/index.d.ts.map +1 -1
- package/background-handler/index.js +9 -13
- package/background-handler/index.js.map +1 -1
- package/body-parser.js +5 -9
- package/body-parser.js.map +1 -1
- package/cache-helpers.d.ts +4 -4
- package/cache-helpers.d.ts.map +1 -1
- package/cache-helpers.js +43 -49
- package/cache-helpers.js.map +1 -1
- package/consts.js +4 -7
- package/consts.js.map +1 -1
- package/fs/common.js +1 -2
- package/fs/dataset/fs.d.ts +2 -2
- package/fs/dataset/fs.d.ts.map +1 -1
- package/fs/dataset/fs.js +13 -27
- package/fs/dataset/fs.js.map +1 -1
- package/fs/dataset/index.d.ts +1 -1
- package/fs/dataset/index.d.ts.map +1 -1
- package/fs/dataset/index.js +5 -8
- package/fs/dataset/index.js.map +1 -1
- package/fs/dataset/memory.d.ts +1 -1
- package/fs/dataset/memory.d.ts.map +1 -1
- package/fs/dataset/memory.js +2 -13
- package/fs/dataset/memory.js.map +1 -1
- package/fs/key-value-store/fs.d.ts +3 -3
- package/fs/key-value-store/fs.d.ts.map +1 -1
- package/fs/key-value-store/fs.js +27 -62
- package/fs/key-value-store/fs.js.map +1 -1
- package/fs/key-value-store/index.d.ts +2 -2
- package/fs/key-value-store/index.d.ts.map +1 -1
- package/fs/key-value-store/index.js +5 -8
- package/fs/key-value-store/index.js.map +1 -1
- package/fs/key-value-store/memory.d.ts +2 -2
- package/fs/key-value-store/memory.d.ts.map +1 -1
- package/fs/key-value-store/memory.js +2 -13
- package/fs/key-value-store/memory.js.map +1 -1
- package/fs/request-queue/fs.d.ts +3 -3
- package/fs/request-queue/fs.d.ts.map +1 -1
- package/fs/request-queue/fs.js +23 -57
- package/fs/request-queue/fs.js.map +1 -1
- package/fs/request-queue/index.d.ts +2 -2
- package/fs/request-queue/index.d.ts.map +1 -1
- package/fs/request-queue/index.js +5 -8
- package/fs/request-queue/index.js.map +1 -1
- package/fs/request-queue/memory.d.ts +2 -2
- package/fs/request-queue/memory.d.ts.map +1 -1
- package/fs/request-queue/memory.js +3 -19
- package/fs/request-queue/memory.js.map +1 -1
- package/index.d.ts +1 -1
- package/index.d.ts.map +1 -1
- package/index.js +1 -4
- package/index.js.map +1 -1
- package/memory-storage.d.ts +3 -3
- package/memory-storage.d.ts.map +1 -1
- package/memory-storage.js +68 -117
- package/memory-storage.js.map +1 -1
- package/package.json +14 -20
- package/resource-clients/common/base-client.d.ts +1 -1
- package/resource-clients/common/base-client.d.ts.map +1 -1
- package/resource-clients/common/base-client.js +2 -11
- package/resource-clients/common/base-client.js.map +1 -1
- package/resource-clients/dataset-collection.d.ts +1 -1
- package/resource-clients/dataset-collection.d.ts.map +1 -1
- package/resource-clients/dataset-collection.js +13 -27
- package/resource-clients/dataset-collection.js.map +1 -1
- package/resource-clients/dataset.d.ts +2 -2
- package/resource-clients/dataset.d.ts.map +1 -1
- package/resource-clients/dataset.js +47 -87
- package/resource-clients/dataset.js.map +1 -1
- package/resource-clients/key-value-store-collection.d.ts +1 -1
- package/resource-clients/key-value-store-collection.d.ts.map +1 -1
- package/resource-clients/key-value-store-collection.js +13 -27
- package/resource-clients/key-value-store-collection.js.map +1 -1
- package/resource-clients/key-value-store.d.ts +2 -2
- package/resource-clients/key-value-store.d.ts.map +1 -1
- package/resource-clients/key-value-store.js +76 -109
- package/resource-clients/key-value-store.js.map +1 -1
- package/resource-clients/request-queue-collection.d.ts +1 -1
- package/resource-clients/request-queue-collection.d.ts.map +1 -1
- package/resource-clients/request-queue-collection.js +13 -27
- package/resource-clients/request-queue-collection.js.map +1 -1
- package/resource-clients/request-queue.d.ts +2 -2
- package/resource-clients/request-queue.d.ts.map +1 -1
- package/resource-clients/request-queue.js +81 -137
- package/resource-clients/request-queue.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/utils.js +12 -20
- package/utils.js.map +1 -1
- package/index.mjs +0 -4
package/utils.js
CHANGED
|
@@ -1,20 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
|
|
6
|
-
exports.isBuffer = isBuffer;
|
|
7
|
-
exports.isStream = isStream;
|
|
8
|
-
const tslib_1 = require("tslib");
|
|
9
|
-
const node_crypto_1 = require("node:crypto");
|
|
10
|
-
const shapeshift_1 = require("@sapphire/shapeshift");
|
|
11
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
12
|
-
const consts_1 = require("./consts");
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { s } from '@sapphire/shapeshift';
|
|
3
|
+
import defaultLog from '@apify/log';
|
|
4
|
+
import { REQUEST_ID_LENGTH } from './consts.js';
|
|
13
5
|
/**
|
|
14
6
|
* Removes all properties with a null value
|
|
15
7
|
* from the provided object.
|
|
16
8
|
*/
|
|
17
|
-
function purgeNullsFromObject(object) {
|
|
9
|
+
export function purgeNullsFromObject(object) {
|
|
18
10
|
if (object && typeof object === 'object' && !Array.isArray(object)) {
|
|
19
11
|
for (const [key, value] of Object.entries(object)) {
|
|
20
12
|
if (value === null)
|
|
@@ -26,26 +18,26 @@ function purgeNullsFromObject(object) {
|
|
|
26
18
|
/**
|
|
27
19
|
* Creates a standard request ID (same as Platform).
|
|
28
20
|
*/
|
|
29
|
-
function uniqueKeyToRequestId(uniqueKey) {
|
|
30
|
-
const str =
|
|
21
|
+
export function uniqueKeyToRequestId(uniqueKey) {
|
|
22
|
+
const str = createHash('sha256')
|
|
31
23
|
.update(uniqueKey)
|
|
32
24
|
.digest('base64')
|
|
33
25
|
.replace(/(\+|\/|=)/g, '');
|
|
34
|
-
return str.length >
|
|
26
|
+
return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;
|
|
35
27
|
}
|
|
36
|
-
function isBuffer(value) {
|
|
28
|
+
export function isBuffer(value) {
|
|
37
29
|
try {
|
|
38
|
-
|
|
30
|
+
s.union([s.instance(Buffer), s.instance(ArrayBuffer), s.typedArray()]).parse(value);
|
|
39
31
|
return true;
|
|
40
32
|
}
|
|
41
33
|
catch {
|
|
42
34
|
return false;
|
|
43
35
|
}
|
|
44
36
|
}
|
|
45
|
-
function isStream(value) {
|
|
37
|
+
export function isStream(value) {
|
|
46
38
|
return (typeof value === 'object' &&
|
|
47
39
|
value &&
|
|
48
40
|
['on', 'pipe'].every((key) => key in value && typeof value[key] === 'function'));
|
|
49
41
|
}
|
|
50
|
-
|
|
42
|
+
export const memoryStorageLog = defaultLog.child({ prefix: 'MemoryStorage' });
|
|
51
43
|
//# sourceMappingURL=utils.js.map
|
package/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,UAAU,MAAM,YAAY,CAAC;AAEpC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACnC,IAAI,CAAC;QACD,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpF,OAAO,IAAI,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAU;IAC/B,OAAO,CACH,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK;QACL,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAClF,CAAC;AACN,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC"}
|
package/index.mjs
DELETED