@crawlee/core 3.13.1-beta.8 → 3.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/autoscaling/autoscaled_pool.d.ts +1 -1
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +6 -5
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +2 -2
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +4 -2
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +1 -1
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +2 -2
- package/configuration.d.ts.map +1 -1
- package/configuration.js +1 -1
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +7 -7
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts.map +1 -1
- package/crawlers/error_snapshotter.d.ts +1 -1
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_tracker.d.ts +1 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +7 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +9 -3
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +1 -1
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +4 -4
- package/enqueue_links/shared.js.map +1 -1
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +1 -1
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +1 -1
- package/events/local_event_manager.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +1 -1
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/log.d.ts +1 -1
- package/log.d.ts.map +1 -1
- package/log.js +1 -1
- package/log.js.map +1 -1
- package/package.json +5 -5
- package/proxy_configuration.d.ts +0 -1
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +2 -5
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts.map +1 -1
- package/request.js +1 -1
- package/request.js.map +1 -1
- package/serialization.d.ts +1 -1
- package/serialization.d.ts.map +1 -1
- package/serialization.js +6 -6
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +17 -9
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +49 -25
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +2 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +2 -2
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +2 -2
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +1 -1
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +7 -6
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +4 -4
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +1 -1
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +3 -3
- package/storages/request_list.js.map +1 -1
- package/storages/request_provider.d.ts +3 -3
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +5 -5
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +2 -2
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +1 -1
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +2 -2
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +1 -1
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +1 -1
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +7 -5
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +1 -1
- package/storages/utils.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
package/serialization.js
CHANGED
|
@@ -4,19 +4,19 @@ exports.serializeArray = serializeArray;
|
|
|
4
4
|
exports.deserializeArray = deserializeArray;
|
|
5
5
|
exports.createDeserialize = createDeserialize;
|
|
6
6
|
const tslib_1 = require("tslib");
|
|
7
|
+
const node_stream_1 = require("node:stream");
|
|
7
8
|
const node_util_1 = tslib_1.__importDefault(require("node:util"));
|
|
8
9
|
const node_zlib_1 = tslib_1.__importDefault(require("node:zlib"));
|
|
9
|
-
const stream_1 = require("stream");
|
|
10
10
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
11
11
|
const StreamArray_1 = tslib_1.__importDefault(require("stream-json/streamers/StreamArray"));
|
|
12
|
-
const pipeline = node_util_1.default.promisify(
|
|
12
|
+
const pipeline = node_util_1.default.promisify(node_stream_1.pipeline);
|
|
13
13
|
/**
|
|
14
14
|
* Transforms an array of items to a JSON in a streaming
|
|
15
15
|
* fashion to save memory. It operates in batches to speed
|
|
16
16
|
* up the process.
|
|
17
17
|
* @internal
|
|
18
18
|
*/
|
|
19
|
-
class ArrayToJson extends
|
|
19
|
+
class ArrayToJson extends node_stream_1.Readable {
|
|
20
20
|
constructor(data, options = {}) {
|
|
21
21
|
super({
|
|
22
22
|
...options,
|
|
@@ -95,7 +95,7 @@ async function serializeArray(data) {
|
|
|
95
95
|
async function deserializeArray(compressedData) {
|
|
96
96
|
(0, ow_1.default)(compressedData, ow_1.default.uint8Array);
|
|
97
97
|
const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
|
|
98
|
-
await pipeline(
|
|
98
|
+
await pipeline(node_stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), StreamArray_1.default.withParser(), collector);
|
|
99
99
|
return chunks;
|
|
100
100
|
}
|
|
101
101
|
/**
|
|
@@ -111,7 +111,7 @@ function createDeserialize(compressedData) {
|
|
|
111
111
|
(0, ow_1.default)(compressedData, ow_1.default.uint8Array);
|
|
112
112
|
const streamArray = StreamArray_1.default.withParser();
|
|
113
113
|
const destination = pluckValue(streamArray);
|
|
114
|
-
(0,
|
|
114
|
+
(0, node_stream_1.pipeline)(node_stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), destination,
|
|
115
115
|
// @ts-expect-error Something's wrong here, the types are wrong but tests fail if we correct the code to make them right
|
|
116
116
|
(err) => destination.emit(err));
|
|
117
117
|
return destination;
|
|
@@ -119,7 +119,7 @@ function createDeserialize(compressedData) {
|
|
|
119
119
|
function createChunkCollector(options = {}) {
|
|
120
120
|
const { fromValuesStream = false } = options;
|
|
121
121
|
const chunks = [];
|
|
122
|
-
const collector = new
|
|
122
|
+
const collector = new node_stream_1.Writable({
|
|
123
123
|
decodeStrings: false,
|
|
124
124
|
objectMode: fromValuesStream,
|
|
125
125
|
write(chunk, _nil, callback) {
|
package/serialization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;AAiEA,wCAMC;AAWD,4CAMC;AAWD,8CAcC;;AAjHD,
|
|
1
|
+
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;AAiEA,wCAMC;AAWD,4CAMC;AAWD,8CAcC;;AAjHD,6CAA6E;AAC7E,kEAA6B;AAC7B,kEAA6B;AAE7B,oDAAoB;AACpB,4FAA4D;AAE5D,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,sBAAc,CAAC,CAAC;AAIhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,sBAAQ;IAIjC,YACY,IAAS,EACjB,UAAkC,EAAE;QAEpC,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QAPH;;;;mBAAQ,IAAI;WAAK;QAJb;;;;mBAAS,CAAC;WAAC;QACF;;;;;WAAkB;QAW/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,mBAAI,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAEpE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAmC;IACjG,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CAAC,sBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,mBAAI,CAAC,YAAY,EAAE,EAAE,qBAAW,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAE1G,OAAO,MAAa,CAAC;AACzB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAmC;IACjE,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,sBAAc,EACV,sBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wHAAwH;IACxH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CACzB,UAA0C,EAAE;IAE5C,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,sBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { Log } from '@apify/log';
|
|
2
1
|
import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';
|
|
3
2
|
import type { Cookie, SerializedCookieJar } from 'tough-cookie';
|
|
4
3
|
import { CookieJar } from 'tough-cookie';
|
|
4
|
+
import type { Log } from '@apify/log';
|
|
5
5
|
import type { ResponseLike } from '../cookie_utils';
|
|
6
6
|
/**
|
|
7
7
|
* Persistable {@apilink Session} state.
|
|
@@ -74,16 +74,24 @@ export declare class Session {
|
|
|
74
74
|
readonly id: string;
|
|
75
75
|
private maxAgeSecs;
|
|
76
76
|
userData: Dictionary;
|
|
77
|
-
private
|
|
78
|
-
private
|
|
79
|
-
private
|
|
80
|
-
private
|
|
81
|
-
private
|
|
82
|
-
private
|
|
77
|
+
private _maxErrorScore;
|
|
78
|
+
private _errorScoreDecrement;
|
|
79
|
+
private _createdAt;
|
|
80
|
+
private _expiresAt;
|
|
81
|
+
private _usageCount;
|
|
82
|
+
private _maxUsageCount;
|
|
83
83
|
private sessionPool;
|
|
84
|
-
private
|
|
85
|
-
private
|
|
84
|
+
private _errorScore;
|
|
85
|
+
private _cookieJar;
|
|
86
86
|
private log;
|
|
87
|
+
get errorScore(): number;
|
|
88
|
+
get usageCount(): number;
|
|
89
|
+
get maxErrorScore(): number;
|
|
90
|
+
get errorScoreDecrement(): number;
|
|
91
|
+
get expiresAt(): Date;
|
|
92
|
+
get createdAt(): Date;
|
|
93
|
+
get maxUsageCount(): number;
|
|
94
|
+
get cookieJar(): CookieJar;
|
|
87
95
|
/**
|
|
88
96
|
* Session configuration.
|
|
89
97
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAUpD;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,mBAAmB,CAAC;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,gBAAgB,EAAE,WAAW,CAAC;IAEnD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,GAAG,CAAM;IAEjB,IAAI,UAAU,WAEb;IAED,IAAI,UAAU,WAEb;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,mBAAmB,WAEtB;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,SAAS,cAEZ;IAED;;OAEG;gBACS,OAAO,EAAE,cAAc;IAuDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAEvD;;;;;;;;;;;;OAYG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,4BAA4B,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO;IAahG;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,YAAY;IAW7C;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -3,12 +3,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Session = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const node_events_1 = require("node:events");
|
|
6
|
-
const utilities_1 = require("@apify/utilities");
|
|
7
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
8
7
|
const tough_cookie_1 = require("tough-cookie");
|
|
9
|
-
const
|
|
8
|
+
const utilities_1 = require("@apify/utilities");
|
|
10
9
|
const cookie_utils_1 = require("../cookie_utils");
|
|
11
10
|
const log_1 = require("../log");
|
|
11
|
+
const events_1 = require("./events");
|
|
12
12
|
/**
|
|
13
13
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
14
14
|
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
@@ -16,6 +16,30 @@ const log_1 = require("../log");
|
|
|
16
16
|
* @category Scaling
|
|
17
17
|
*/
|
|
18
18
|
class Session {
|
|
19
|
+
get errorScore() {
|
|
20
|
+
return this._errorScore;
|
|
21
|
+
}
|
|
22
|
+
get usageCount() {
|
|
23
|
+
return this._usageCount;
|
|
24
|
+
}
|
|
25
|
+
get maxErrorScore() {
|
|
26
|
+
return this._maxErrorScore;
|
|
27
|
+
}
|
|
28
|
+
get errorScoreDecrement() {
|
|
29
|
+
return this._errorScoreDecrement;
|
|
30
|
+
}
|
|
31
|
+
get expiresAt() {
|
|
32
|
+
return this._expiresAt;
|
|
33
|
+
}
|
|
34
|
+
get createdAt() {
|
|
35
|
+
return this._createdAt;
|
|
36
|
+
}
|
|
37
|
+
get maxUsageCount() {
|
|
38
|
+
return this._maxUsageCount;
|
|
39
|
+
}
|
|
40
|
+
get cookieJar() {
|
|
41
|
+
return this._cookieJar;
|
|
42
|
+
}
|
|
19
43
|
/**
|
|
20
44
|
* Session configuration.
|
|
21
45
|
*/
|
|
@@ -38,37 +62,37 @@ class Session {
|
|
|
38
62
|
writable: true,
|
|
39
63
|
value: void 0
|
|
40
64
|
});
|
|
41
|
-
Object.defineProperty(this, "
|
|
65
|
+
Object.defineProperty(this, "_maxErrorScore", {
|
|
42
66
|
enumerable: true,
|
|
43
67
|
configurable: true,
|
|
44
68
|
writable: true,
|
|
45
69
|
value: void 0
|
|
46
70
|
});
|
|
47
|
-
Object.defineProperty(this, "
|
|
71
|
+
Object.defineProperty(this, "_errorScoreDecrement", {
|
|
48
72
|
enumerable: true,
|
|
49
73
|
configurable: true,
|
|
50
74
|
writable: true,
|
|
51
75
|
value: void 0
|
|
52
76
|
});
|
|
53
|
-
Object.defineProperty(this, "
|
|
77
|
+
Object.defineProperty(this, "_createdAt", {
|
|
54
78
|
enumerable: true,
|
|
55
79
|
configurable: true,
|
|
56
80
|
writable: true,
|
|
57
81
|
value: void 0
|
|
58
82
|
});
|
|
59
|
-
Object.defineProperty(this, "
|
|
83
|
+
Object.defineProperty(this, "_expiresAt", {
|
|
60
84
|
enumerable: true,
|
|
61
85
|
configurable: true,
|
|
62
86
|
writable: true,
|
|
63
87
|
value: void 0
|
|
64
88
|
});
|
|
65
|
-
Object.defineProperty(this, "
|
|
89
|
+
Object.defineProperty(this, "_usageCount", {
|
|
66
90
|
enumerable: true,
|
|
67
91
|
configurable: true,
|
|
68
92
|
writable: true,
|
|
69
93
|
value: void 0
|
|
70
94
|
});
|
|
71
|
-
Object.defineProperty(this, "
|
|
95
|
+
Object.defineProperty(this, "_maxUsageCount", {
|
|
72
96
|
enumerable: true,
|
|
73
97
|
configurable: true,
|
|
74
98
|
writable: true,
|
|
@@ -80,13 +104,13 @@ class Session {
|
|
|
80
104
|
writable: true,
|
|
81
105
|
value: void 0
|
|
82
106
|
});
|
|
83
|
-
Object.defineProperty(this, "
|
|
107
|
+
Object.defineProperty(this, "_errorScore", {
|
|
84
108
|
enumerable: true,
|
|
85
109
|
configurable: true,
|
|
86
110
|
writable: true,
|
|
87
111
|
value: void 0
|
|
88
112
|
});
|
|
89
|
-
Object.defineProperty(this, "
|
|
113
|
+
Object.defineProperty(this, "_cookieJar", {
|
|
90
114
|
enumerable: true,
|
|
91
115
|
configurable: true,
|
|
92
116
|
writable: true,
|
|
@@ -116,18 +140,18 @@ class Session {
|
|
|
116
140
|
const { sessionPool, id = `session_${(0, utilities_1.cryptoRandomObjectId)(10)}`, cookieJar = new tough_cookie_1.CookieJar(), maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = log_1.log, } = options;
|
|
117
141
|
const { expiresAt = (0, cookie_utils_1.getDefaultCookieExpirationDate)(maxAgeSecs) } = options;
|
|
118
142
|
this.log = log.child({ prefix: 'Session' });
|
|
119
|
-
this.
|
|
143
|
+
this._cookieJar = cookieJar.setCookie ? cookieJar : tough_cookie_1.CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
120
144
|
this.id = id;
|
|
121
145
|
this.maxAgeSecs = maxAgeSecs;
|
|
122
146
|
this.userData = userData;
|
|
123
|
-
this.
|
|
124
|
-
this.
|
|
147
|
+
this._maxErrorScore = maxErrorScore;
|
|
148
|
+
this._errorScoreDecrement = errorScoreDecrement;
|
|
125
149
|
// Internal
|
|
126
|
-
this.
|
|
127
|
-
this.
|
|
128
|
-
this.
|
|
129
|
-
this.
|
|
130
|
-
this.
|
|
150
|
+
this._expiresAt = expiresAt;
|
|
151
|
+
this._createdAt = createdAt;
|
|
152
|
+
this._usageCount = usageCount; // indicates how many times the session has been used
|
|
153
|
+
this._errorScore = errorScore; // indicates number of markBaded request with the session
|
|
154
|
+
this._maxUsageCount = maxUsageCount;
|
|
131
155
|
this.sessionPool = sessionPool;
|
|
132
156
|
}
|
|
133
157
|
/**
|
|
@@ -164,9 +188,9 @@ class Session {
|
|
|
164
188
|
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
165
189
|
*/
|
|
166
190
|
markGood() {
|
|
167
|
-
this.
|
|
168
|
-
if (this.
|
|
169
|
-
this.
|
|
191
|
+
this._usageCount += 1;
|
|
192
|
+
if (this._errorScore > 0) {
|
|
193
|
+
this._errorScore -= this._errorScoreDecrement;
|
|
170
194
|
}
|
|
171
195
|
this._maybeSelfRetire();
|
|
172
196
|
}
|
|
@@ -197,8 +221,8 @@ class Session {
|
|
|
197
221
|
*/
|
|
198
222
|
retire() {
|
|
199
223
|
// mark it as an invalid by increasing the error score count.
|
|
200
|
-
this.
|
|
201
|
-
this.
|
|
224
|
+
this._errorScore += this._maxErrorScore;
|
|
225
|
+
this._usageCount += 1;
|
|
202
226
|
// emit event so we can retire browser in puppeteer pool
|
|
203
227
|
this.sessionPool.emit(events_1.EVENT_SESSION_RETIRED, this);
|
|
204
228
|
}
|
|
@@ -207,8 +231,8 @@ class Session {
|
|
|
207
231
|
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
208
232
|
*/
|
|
209
233
|
markBad() {
|
|
210
|
-
this.
|
|
211
|
-
this.
|
|
234
|
+
this._errorScore += 1;
|
|
235
|
+
this._usageCount += 1;
|
|
212
236
|
this._maybeSelfRetire();
|
|
213
237
|
}
|
|
214
238
|
retireOnBlockedStatusCodes(statusCode, additionalBlockedStatusCodes = []) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AAAA,6CAA2C;AAG3C,
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AAAA,6CAA2C;AAG3C,oDAAoB;AAEpB,+CAAyC;AAGzC,gDAAwD;AAGxD,kDAKyB;AACzB,gCAA2C;AAC3C,qCAAiD;AA2EjD;;;;;GAKG;AACH,MAAa,OAAO;IAehB,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,OAAuB;QAjD1B;;;;;WAAW;QACZ;;;;;WAAmB;QAC3B;;;;;WAAqB;QACb;;;;;WAAuB;QACvB;;;;;WAA6B;QAC7B;;;;;WAAiB;QACjB;;;;;WAAiB;QACjB;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAAkD;QAClD;;;;;WAAoB;QACpB;;;;;WAAsB;QACtB;;;;;WAAS;QAsCb,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAY,CAAC;YAC/C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAA,6CAA8B,EAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IA2BD,0BAA0B,CAAC,UAAkB,EAAE,+BAAyC,EAAE;QACtF,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC;aACnD,MAAM,CAAC,4BAA4B,CAAC;aACpC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC1B,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAsB;QACzC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,IAAA,qCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;QACnG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,6CAA8B,EAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,6CAA8B,EAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ;AArUD,0BAqUC"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import type { Log } from '@apify/log';
|
|
3
|
-
import type { SessionOptions } from './session';
|
|
4
|
-
import { Session } from './session';
|
|
5
3
|
import { Configuration } from '../configuration';
|
|
6
4
|
import type { PersistenceOptions } from '../crawlers/statistics';
|
|
7
5
|
import type { EventManager } from '../events/event_manager';
|
|
8
6
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
7
|
+
import type { SessionOptions } from './session';
|
|
8
|
+
import { Session } from './session';
|
|
9
9
|
/**
|
|
10
10
|
* Factory user-function which creates customized {@apilink Session} instances.
|
|
11
11
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAuBrC,QAAQ,CAAC,MAAM;IAtBnB,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBAEC,OAAO,GAAE,kBAAuB,EACvB,MAAM,gBAAkC;IAsDrD;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAY/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO;IAUV;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC;CAKhG"}
|
|
@@ -5,11 +5,11 @@ const tslib_1 = require("tslib");
|
|
|
5
5
|
const node_events_1 = require("node:events");
|
|
6
6
|
const async_queue_1 = require("@sapphire/async-queue");
|
|
7
7
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
8
|
-
const consts_1 = require("./consts");
|
|
9
|
-
const session_1 = require("./session");
|
|
10
8
|
const configuration_1 = require("../configuration");
|
|
11
9
|
const log_1 = require("../log");
|
|
12
10
|
const key_value_store_1 = require("../storages/key_value_store");
|
|
11
|
+
const consts_1 = require("./consts");
|
|
12
|
+
const session_1 = require("./session");
|
|
13
13
|
/**
|
|
14
14
|
* Handles the rotation, creation and persistence of user-like sessions.
|
|
15
15
|
* Creates a pool of {@apilink Session} instances, that are randomly rotated.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AAAA,6CAA2C;
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AAAA,6CAA2C;AAG3C,uDAAmD;AACnD,oDAAoB;AAIpB,oDAAiD;AAIjD,gCAA2C;AAC3C,iEAA4D;AAC5D,qCAAkF;AAElF,uCAAoC;AAuDpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,WAAY,SAAQ,0BAAY;IAkBzC;;OAEG;IACH,YACI,UAA8B,EAAE,EACvB,SAAS,6BAAa,CAAC,eAAe,EAAE;QAEjD,KAAK,EAAE,CAAC;QAFR;;;;mBAAS,MAAM;WAAkC;QAtB3C;;;;;WAAS;QACT;;;;;WAAoB;QACpB;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;mBAAsB,EAAE;WAAC;QACzB;;;;mBAAa,IAAI,GAAG,EAAmB;WAAC;QACxC;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAwB;QACxB;;;;;WAAgC;QAChC;;;;;WAAqB;QACZ;;;;;WAA6B;QACtC;;;;;WAAuC;QACvC;;;;mBAAgB,KAAK;WAAC;QAExB;;;;mBAAQ,IAAI,wBAAU,EAAE;WAAC;QAW7B,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;YACvD,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,GAAG,sBAAa,EAC3B,2BAA2B,EAC3B,eAAe,GAAG,0BAAiB,EACnC,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,kBAAkB,GAAG,6BAAoB,EACzC,GAAG,GAAG,SAAU,EAChB,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,GACJ,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACzG,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GACZ,OAAO,YAAY,iBAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAExB,IAAI,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAE9B,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBAClD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC3B,OAAO,aAAa,CAAC;YACzB,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,6BAA6B,CACnC,WAAwB,EACxB,UAA+C,EAAE;QAEjD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACxC,OAAO,IAAI,iBAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YACrD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;YAEnG,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B,EAAE,MAAsB;QAClE,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AAjXD,kCAiXC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access_checking.d.ts","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAI7C;;GAEG;AACH,eAAO,MAAM,kBAAkB,wBAA4C,CAAC;AAE5E;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,GAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"access_checking.d.ts","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAI7C;;GAEG;AACH,eAAO,MAAM,kBAAkB,wBAA4C,CAAC;AAE5E;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,GAAU,CAAC,iBAAiB,MAAM,IAAI,YAAY,MAAM,SAAS,CAAC,CAAC,CAAC,eAC7D,CAAC"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.withCheckedStorageAccess = exports.checkStorageAccess = void 0;
|
|
4
|
-
const
|
|
5
|
-
const storage = new
|
|
4
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
5
|
+
const storage = new node_async_hooks_1.AsyncLocalStorage();
|
|
6
6
|
/**
|
|
7
7
|
* Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
|
|
8
8
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access_checking.js","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"access_checking.js","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":";;;AAAA,uDAAqD;AAIrD,MAAM,OAAO,GAAG,IAAI,oCAAiB,EAAiC,CAAC;AAEvE;;GAEG;AACI,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC;AAA/D,QAAA,kBAAkB,sBAA6C;AAE5E;;;;;GAKG;AACI,MAAM,wBAAwB,GAAG,KAAK,EAAK,aAAyB,EAAE,QAA4B,EAAE,EAAE,CACzG,OAAO,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,EAAE,QAAQ,CAAC,CAAC;AADhC,QAAA,wBAAwB,4BACQ"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { DatasetClient, DatasetInfo, Dictionary, StorageClient } from '@crawlee/types';
|
|
2
|
-
import type { StorageManagerOptions } from './storage_manager';
|
|
3
2
|
import { Configuration } from '../configuration';
|
|
4
3
|
import { type Log } from '../log';
|
|
5
4
|
import type { Awaitable } from '../typedefs';
|
|
5
|
+
import type { StorageManagerOptions } from './storage_manager';
|
|
6
6
|
/** @internal */
|
|
7
7
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
8
8
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,KAAK,GAAG,EAAO,MAAM,QAAQ,CAAC;AACvC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAG7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;CAAG;AAE7F,MAAM,WAAW,sBACb,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC3F,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAWjD,QAAQ,CAAC,MAAM;IAVnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,EAAE,GAAG,CAAoC;IAE5C;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IAOrD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAgB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAyBjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IA0BpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAMjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBzG;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAC/B,OAAO,GAAE,qBAA0B,GACpC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAsBzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
package/storages/dataset.js
CHANGED
|
@@ -4,15 +4,15 @@ exports.Dataset = exports.DATASET_ITERATORS_DEFAULT_LIMIT = void 0;
|
|
|
4
4
|
exports.checkAndSerialize = checkAndSerialize;
|
|
5
5
|
exports.chunkBySize = chunkBySize;
|
|
6
6
|
const tslib_1 = require("tslib");
|
|
7
|
-
const consts_1 = require("@apify/consts");
|
|
8
7
|
const sync_1 = require("csv-stringify/sync");
|
|
9
8
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
9
|
+
const consts_1 = require("@apify/consts");
|
|
10
|
+
const configuration_1 = require("../configuration");
|
|
11
|
+
const log_1 = require("../log");
|
|
10
12
|
const access_checking_1 = require("./access_checking");
|
|
11
13
|
const key_value_store_1 = require("./key_value_store");
|
|
12
14
|
const storage_manager_1 = require("./storage_manager");
|
|
13
15
|
const utils_1 = require("./utils");
|
|
14
|
-
const configuration_1 = require("../configuration");
|
|
15
|
-
const log_1 = require("../log");
|
|
16
16
|
/** @internal */
|
|
17
17
|
exports.DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
18
18
|
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
@@ -204,7 +204,8 @@ class Dataset {
|
|
|
204
204
|
// Handle singular Objects
|
|
205
205
|
if (!Array.isArray(data)) {
|
|
206
206
|
const payload = checkAndSerialize(data, limit);
|
|
207
|
-
|
|
207
|
+
await dispatch(payload);
|
|
208
|
+
return;
|
|
208
209
|
}
|
|
209
210
|
// Handle Arrays
|
|
210
211
|
const payloads = data.map((item, index) => checkAndSerialize(item, limit, index));
|
|
@@ -245,7 +246,7 @@ class Dataset {
|
|
|
245
246
|
}
|
|
246
247
|
items.push(...value.items);
|
|
247
248
|
if (value.total > offset + value.count) {
|
|
248
|
-
|
|
249
|
+
await fetchNextChunk(offset + value.count);
|
|
249
250
|
}
|
|
250
251
|
};
|
|
251
252
|
await fetchNextChunk();
|
|
@@ -381,7 +382,7 @@ class Dataset {
|
|
|
381
382
|
if (newOffset >= total)
|
|
382
383
|
return;
|
|
383
384
|
const newOpts = { ...options, offset: newOffset };
|
|
384
|
-
|
|
385
|
+
await this.forEach(iteratee, newOpts, index);
|
|
385
386
|
}
|
|
386
387
|
/**
|
|
387
388
|
* Produces a new array of values by mapping each value in list through a transformation function `iteratee()`.
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;AA0BA,8CAsBC;AAWD,kCA8BC;;AAxFD,6CAA+C;AAC/C,oDAAoB;AAEpB,0CAAuD;AAEvD,oDAAiD;AACjD,gCAAuC;AAEvC,uDAAuD;AACvD,uDAAkD;AAElD,uDAAmD;AACnD,mCAA+C;AAE/C,gBAAgB;AACH,QAAA,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AAqFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAa,OAAO;IAMhB;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,6BAAa,CAAC,eAAe,EAAE;QAAjD;;;;mBAAS,MAAM;WAAkC;QAVrD;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;;WAA4B;QAC5B;;;;mBAAW,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;WAAC;QASxC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAA,YAAE,EAAC,IAAI,EAAE,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,+BAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,+BAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,IAAA,gBAAS,EAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;QAE5D,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,IAAA,oCAAkB,GAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,uCAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,eAA+B,EAC/B,UAAiC,EAAE;QAEnC,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAA,YAAE,EAAC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;YACpD,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,6BAAa,CAAC,eAAe,EAAE,EAAC;QACnD,OAAO,CAAC,aAAa,KAArB,OAAO,CAAC,aAAa,GAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAC;QAE5D,MAAM,IAAA,4BAAoB,EAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAgB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ;AAvcD,0BAucC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Dictionary, StorageClient } from '@crawlee/types';
|
|
2
|
-
import type { StorageManagerOptions } from './storage_manager';
|
|
3
2
|
import { Configuration } from '../configuration';
|
|
4
3
|
import type { Awaitable } from '../typedefs';
|
|
4
|
+
import type { StorageManagerOptions } from './storage_manager';
|
|
5
5
|
/**
|
|
6
6
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
7
7
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAOrF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,SAAS,CAAC,WAAW;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAelB,QAAQ,CAAC,MAAM;IAdnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAQrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;IAc/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8D3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpF,WAAW;IAwBzB;;OAEG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAsB7G;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -4,14 +4,14 @@ exports.KeyValueStore = exports.maybeStringify = void 0;
|
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const promises_1 = require("node:fs/promises");
|
|
6
6
|
const node_path_1 = require("node:path");
|
|
7
|
-
const consts_1 = require("@apify/consts");
|
|
8
|
-
const utilities_1 = require("@apify/utilities");
|
|
9
7
|
const json5_1 = tslib_1.__importDefault(require("json5"));
|
|
10
8
|
const ow_1 = tslib_1.__importStar(require("ow"));
|
|
9
|
+
const consts_1 = require("@apify/consts");
|
|
10
|
+
const utilities_1 = require("@apify/utilities");
|
|
11
|
+
const configuration_1 = require("../configuration");
|
|
11
12
|
const access_checking_1 = require("./access_checking");
|
|
12
13
|
const storage_manager_1 = require("./storage_manager");
|
|
13
14
|
const utils_1 = require("./utils");
|
|
14
|
-
const configuration_1 = require("../configuration");
|
|
15
15
|
/**
|
|
16
16
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
17
17
|
*
|
|
@@ -28,7 +28,7 @@ const maybeStringify = (value, options) => {
|
|
|
28
28
|
catch (e) {
|
|
29
29
|
const error = e;
|
|
30
30
|
// Give more meaningful error message
|
|
31
|
-
if (error.message?.
|
|
31
|
+
if (error.message?.includes('Invalid string length')) {
|
|
32
32
|
error.message = 'Object is too large';
|
|
33
33
|
}
|
|
34
34
|
throw new Error(`The "value" parameter cannot be stringified to JSON: ${error.message}`);
|