@crawlee/core 3.0.0-beta.2 → 3.0.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -71
- package/autoscaling/autoscaled_pool.js +1 -1
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/crawlers/crawler_commons.d.ts +8 -3
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +13 -15
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +10 -12
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +2 -2
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +1 -2
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -4
- package/events/local_event_manager.js.map +1 -1
- package/package.json +21 -9
- package/request.d.ts +6 -6
- package/request.d.ts.map +1 -1
- package/request.js +4 -4
- package/request.js.map +1 -1
- package/serialization.js +6 -6
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +1 -1
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session_pool.d.ts +1 -1
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +2 -2
- package/session_pool/session_pool.js.map +1 -1
- package/session_pool/session_utils.d.ts +1 -1
- package/session_pool/session_utils.d.ts.map +1 -1
- package/storages/dataset.d.ts +28 -3
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +31 -3
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +105 -13
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +112 -12
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +5 -5
- package/storages/request_list.js +2 -2
- package/storages/request_queue.d.ts +5 -4
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +4 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +4 -4
- package/validators.d.ts.map +1 -1
- package/validators.js.map +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,6CAA2C;AAC3C,gDAA4F;AAC5F,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,0BAAY,EAAE;WAAC;QACtC;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;SACzC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;CACJ;AAnED,oCAmEC"}
|
|
@@ -2,8 +2,7 @@ import { EventManager } from './event_manager';
|
|
|
2
2
|
export declare class LocalEventManager extends EventManager {
|
|
3
3
|
private previousTicks;
|
|
4
4
|
/**
|
|
5
|
-
* Initializes
|
|
6
|
-
* This is an internal function that is automatically called by `Actor.main()`.
|
|
5
|
+
* Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
|
|
7
6
|
*/
|
|
8
7
|
init(): Promise<void>;
|
|
9
8
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;OAEG;IACY,IAAI;IAYnB;;;;OAIG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;YACW,gBAAgB;IAQ9B,OAAO,CAAC,aAAa;YAaP,gBAAgB;IAc9B;;OAEG;YACW,cAAc;CAG/B"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.LocalEventManager = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const
|
|
5
|
+
const node_os_1 = tslib_1.__importDefault(require("node:os"));
|
|
6
6
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
7
7
|
const utilities_1 = require("@apify/utilities");
|
|
8
8
|
const utils_1 = require("@crawlee/utils");
|
|
@@ -18,8 +18,7 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
18
18
|
});
|
|
19
19
|
}
|
|
20
20
|
/**
|
|
21
|
-
* Initializes
|
|
22
|
-
* This is an internal function that is automatically called by `Actor.main()`.
|
|
21
|
+
* Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
|
|
23
22
|
*/
|
|
24
23
|
async init() {
|
|
25
24
|
if (this.initialized) {
|
|
@@ -53,7 +52,7 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
53
52
|
intervalCallback();
|
|
54
53
|
}
|
|
55
54
|
getCurrentCpuTicks() {
|
|
56
|
-
const cpus =
|
|
55
|
+
const cpus = node_os_1.default.cpus();
|
|
57
56
|
return cpus.reduce((acc, cpu) => {
|
|
58
57
|
const cpuTimes = Object.values(cpu.times);
|
|
59
58
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AACzB,6DAA6B;AAC7B,gDAA0E;AAC1E,0CAA+C;AAE/C,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACI;;;;mBAAwB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;IAiGlD,CAAC;IA/FG;;OAEG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,iBAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9B,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;SACrB,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,OAAoC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI;YACA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;YAE1D,OAAO;gBACH,eAAe,EAAE,gBAAgB,GAAG,mBAAmB;aAC1D,CAAC;SACL;QAAC,OAAO,GAAG,EAAE;YACV,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;SACb;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AAlGD,8CAkGC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.20",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,20 +49,32 @@
|
|
|
49
49
|
"access": "public"
|
|
50
50
|
},
|
|
51
51
|
"dependencies": {
|
|
52
|
-
"@apify/consts": "^1.
|
|
52
|
+
"@apify/consts": "^1.10.0",
|
|
53
53
|
"@apify/datastructures": "^1.0.1",
|
|
54
|
-
"@apify/log": "^1.2.
|
|
55
|
-
"@apify/pseudo_url": "^1.1.
|
|
54
|
+
"@apify/log": "^1.2.3",
|
|
55
|
+
"@apify/pseudo_url": "^1.1.4",
|
|
56
56
|
"@apify/timeout": "^0.2.1",
|
|
57
|
-
"@
|
|
58
|
-
"@crawlee/
|
|
57
|
+
"@apify/utilities": "^1.2.12",
|
|
58
|
+
"@crawlee/browser-pool": "^3.0.0-beta.20",
|
|
59
|
+
"@crawlee/memory-storage": "^3.0.0-beta.20",
|
|
60
|
+
"@crawlee/types": "^3.0.0-beta.20",
|
|
61
|
+
"@crawlee/utils": "^3.0.0-beta.20",
|
|
59
62
|
"@types/tough-cookie": "^4.0.2",
|
|
60
63
|
"iconv-lite": "^0.6.3",
|
|
61
|
-
"minimatch": "^5.0
|
|
64
|
+
"minimatch": "^5.1.0",
|
|
62
65
|
"ow": "^0.28.1",
|
|
66
|
+
"stream-chain": "^2.2.5",
|
|
63
67
|
"stream-json": "^1.7.4",
|
|
64
|
-
"tldts": "^5.7.
|
|
68
|
+
"tldts": "^5.7.80",
|
|
65
69
|
"tough-cookie": "^4.0.0",
|
|
66
|
-
"ws": "^7.
|
|
70
|
+
"ws": "^8.7.0"
|
|
71
|
+
},
|
|
72
|
+
"peerDependencies": {
|
|
73
|
+
"puppeteer": ">= 9.x <= 14.x"
|
|
74
|
+
},
|
|
75
|
+
"peerDependenciesMeta": {
|
|
76
|
+
"puppeteer": {
|
|
77
|
+
"optional": true
|
|
78
|
+
}
|
|
67
79
|
}
|
|
68
80
|
}
|
package/request.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { BinaryLike } from 'crypto';
|
|
4
|
-
import { AllowedHttpMethods } from './typedefs';
|
|
3
|
+
import { BinaryLike } from 'node:crypto';
|
|
4
|
+
import { AllowedHttpMethods, Dictionary } from './typedefs';
|
|
5
5
|
/**
|
|
6
6
|
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
7
7
|
* The `Request` object also stores information about errors that occurred during processing of the request.
|
|
@@ -30,7 +30,7 @@ import { AllowedHttpMethods } from './typedefs';
|
|
|
30
30
|
* ```
|
|
31
31
|
* @category Sources
|
|
32
32
|
*/
|
|
33
|
-
export declare class Request {
|
|
33
|
+
export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
34
34
|
/** Request ID */
|
|
35
35
|
id?: string;
|
|
36
36
|
/** URL of the web page to crawl. */
|
|
@@ -64,7 +64,7 @@ export declare class Request {
|
|
|
64
64
|
/** Private store for the custom user data assigned to the request. */
|
|
65
65
|
private _userData;
|
|
66
66
|
/** Custom user data assigned to the request. */
|
|
67
|
-
userData:
|
|
67
|
+
userData: UserData;
|
|
68
68
|
/**
|
|
69
69
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
70
70
|
* Is `null` if the request has not been crawled yet.
|
|
@@ -101,7 +101,7 @@ export declare class Request {
|
|
|
101
101
|
/**
|
|
102
102
|
* Specifies required and optional fields for constructing a {@link Request}.
|
|
103
103
|
*/
|
|
104
|
-
export interface RequestOptions {
|
|
104
|
+
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
105
105
|
/** URL of the web page to crawl. It must be a non-empty string. */
|
|
106
106
|
url: string;
|
|
107
107
|
/**
|
|
@@ -140,7 +140,7 @@ export interface RequestOptions {
|
|
|
140
140
|
* Custom user data assigned to the request. Use this to save any request related data to the
|
|
141
141
|
* request's scope, keeping them accessible on retries, failures etc.
|
|
142
142
|
*/
|
|
143
|
-
userData?:
|
|
143
|
+
userData?: UserData;
|
|
144
144
|
/**
|
|
145
145
|
* If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
|
|
146
146
|
* For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;AACA,OAAe,EAAE,UAAU,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;AACA,OAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIjD,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAQ,MAAM,YAAY,CAAC;AAsBlE;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAqGnC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAG5B;IAED,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAKhC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,EAAE,uBAAuB;IAgBpH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAQtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IAEpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC"}
|
package/request.js
CHANGED
|
@@ -3,9 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Request = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const utilities_1 = require("@apify/utilities");
|
|
6
|
-
const
|
|
6
|
+
const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
|
|
7
7
|
const ow_1 = tslib_1.__importStar(require("ow"));
|
|
8
|
-
const
|
|
8
|
+
const node_util_1 = tslib_1.__importDefault(require("node:util"));
|
|
9
9
|
const log_1 = require("./log");
|
|
10
10
|
const typedefs_1 = require("./typedefs");
|
|
11
11
|
// new properties on the Request object breaks serialization
|
|
@@ -295,7 +295,7 @@ class Request {
|
|
|
295
295
|
}
|
|
296
296
|
else {
|
|
297
297
|
try {
|
|
298
|
-
message =
|
|
298
|
+
message = node_util_1.default.inspect(errorOrMessage);
|
|
299
299
|
}
|
|
300
300
|
catch (err) {
|
|
301
301
|
message = 'Unable to extract any message from the received object.';
|
|
@@ -326,7 +326,7 @@ class Request {
|
|
|
326
326
|
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
327
327
|
}
|
|
328
328
|
_hashPayload(payload) {
|
|
329
|
-
return
|
|
329
|
+
return node_crypto_1.default
|
|
330
330
|
.createHash('sha256')
|
|
331
331
|
.update(payload)
|
|
332
332
|
.digest('base64')
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,sEAAiD;AACjD,iDAAsD;AACtD,kEAA6B;AAC7B,+BAA0C;AAC1C,yCAAkE;AAElE,4DAA4D;AAC5D,MAAM,GAAG,GAAG,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,OAAO;IAqDhB;;OAEG;IACH,YAAY,OAAuB;QAvDnC,iBAAiB;QACjB;;;;;WAAY;QAEZ,oCAAoC;QACpC;;;;;WAAY;QAEZ;;;;;;;WAOG;QACH;;;;;WAAmB;QAEnB;;;WAGG;QACH;;;;;WAAkB;QAElB,yCAAyC;QACzC;;;;;WAA2B;QAE3B,oDAAoD;QACpD;;;;;WAAiB;QAEjB,8FAA8F;QAC9F;;;;;WAAiB;QAEjB,2FAA2F;QAC3F;;;;;WAAmB;QAEnB,0DAA0D;QAC1D;;;;;WAAwB;QAExB,wEAAwE;QACxE;;;;;WAAiC;QAEjC,sEAAsE;QACtE;;;;mBAAyC,EAAE;WAAC;QAE5C,gDAAgD;QAChD;;;;mBAAqB,EAAc;WAAC;QAEpC;;;WAGG;QACH;;;;;WAAmB;QAMf,IAAA,YAAE,EAAC,OAAO,EAAE,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE;gBACX,IAAA,YAAE,EAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;gBAChE,mDAAmD;aACtD;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE;gBACvB,MAAM,GAAG,GAAG,6BAA6B,IAAI,sBAAsB,KAAK,iCAAiC,CAAC;gBAC1G,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aAClD;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,GACjB,GAAG,OAA4H,CAAC;QAEjI,IAAI,EACA,MAAM,GAAG,KAAK,GACjB,GAAG,OAAO,CAAC;QAEZ,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAoB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEvG,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,gDAAgD;4BAChD,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,gDAAgD;4BAChD,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oCAClD,OAAO,CAAC;wCACJ,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC,CAAC;iCACN;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACrE,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,gDAAgD;QAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,IAAI,cAAc,CAAC,KAAc;QAC7B,gDAAgD;QAChD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAG,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAClG,gDAAgD;;YAC3C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;IACxD,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,IAAI,CAAC,cAAc,EAAE;gBACjB,OAAO,GAAG,MAAM,CAAC;aACpB;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE;gBACxC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,8BAA8B;oBAC9B,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;aAC9B;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE;gBACvD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;aAC5D;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE;gBACpE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;aACnD;iBAAM;gBACH,IAAI;oBACA,OAAO,GAAG,mBAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;iBAC1C;gBAAC,OAAO,GAAG,EAAE;oBACV,OAAO,GAAG,yDAAyD,CAAC;iBACvE;aACJ;SACJ;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE;YAC7B,OAAO,GAAG,WAAW,CAAC;SACzB;aAAM;YACH,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;SACnD;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAES,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAA2B;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE;YACvB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE;gBACvC,8EAA8E;gBAC9E,GAAG,CAAC,UAAU,CAAC,uBAAuB,gBAAgB,2BAA2B;sBAC3E,kFAAkF;sBAClF,iHAAiH,CAAC,CAAC;aAC5H;YACD,OAAO,aAAa,CAAC;SACxB;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAES,YAAY,CAAC,OAAmB;QACtC,OAAO,qBAAM;aACR,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,QAAQ,CAAC;aAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;CACJ;AAzPD,0BAyPC"}
|
package/serialization.js
CHANGED
|
@@ -5,9 +5,9 @@ const tslib_1 = require("tslib");
|
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
6
|
const stream_1 = require("stream");
|
|
7
7
|
const StreamArray_1 = tslib_1.__importDefault(require("stream-json/streamers/StreamArray"));
|
|
8
|
-
const
|
|
9
|
-
const
|
|
10
|
-
const pipeline =
|
|
8
|
+
const node_util_1 = tslib_1.__importDefault(require("node:util"));
|
|
9
|
+
const node_zlib_1 = tslib_1.__importDefault(require("node:zlib"));
|
|
10
|
+
const pipeline = node_util_1.default.promisify(stream_1.pipeline);
|
|
11
11
|
/**
|
|
12
12
|
* Transforms an array of items to a JSON in a streaming
|
|
13
13
|
* fashion to save memory. It operates in batches to speed
|
|
@@ -78,7 +78,7 @@ class ArrayToJson extends stream_1.Readable {
|
|
|
78
78
|
async function serializeArray(data) {
|
|
79
79
|
(0, ow_1.default)(data, ow_1.default.array);
|
|
80
80
|
const { chunks, collector } = createChunkCollector();
|
|
81
|
-
await pipeline(new ArrayToJson(data),
|
|
81
|
+
await pipeline(new ArrayToJson(data), node_zlib_1.default.createGzip(), collector);
|
|
82
82
|
return Buffer.concat(chunks);
|
|
83
83
|
}
|
|
84
84
|
exports.serializeArray = serializeArray;
|
|
@@ -94,7 +94,7 @@ exports.serializeArray = serializeArray;
|
|
|
94
94
|
async function deserializeArray(compressedData) {
|
|
95
95
|
(0, ow_1.default)(compressedData, ow_1.default.buffer);
|
|
96
96
|
const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
|
|
97
|
-
await pipeline(stream_1.Readable.from([compressedData]),
|
|
97
|
+
await pipeline(stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), StreamArray_1.default.withParser(), collector);
|
|
98
98
|
return chunks;
|
|
99
99
|
}
|
|
100
100
|
exports.deserializeArray = deserializeArray;
|
|
@@ -111,7 +111,7 @@ function createDeserialize(compressedData) {
|
|
|
111
111
|
(0, ow_1.default)(compressedData, ow_1.default.buffer);
|
|
112
112
|
const streamArray = StreamArray_1.default.withParser();
|
|
113
113
|
const destination = pluckValue(streamArray);
|
|
114
|
-
(0, stream_1.pipeline)(stream_1.Readable.from([compressedData]),
|
|
114
|
+
(0, stream_1.pipeline)(stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), destination,
|
|
115
115
|
// TODO: figure this out
|
|
116
116
|
// @ts-expect-error Somethings wrong here, the types are wrong but tests fail if we correct the code to make them right
|
|
117
117
|
(err) => destination.emit(err));
|
package/serialization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,
|
|
1
|
+
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,kEAA6B;AAC7B,kEAA6B;AAE7B,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,mBAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wBAAwB;IACxB,uHAAuH;IACvH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAfD,8CAeC;AAED,SAAS,oBAAoB,CAA4B,UAA0C,EAAE;IACjG,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,iBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { Log } from '@apify/log';
|
|
3
3
|
import { Cookie as BrowserPoolCookie } from '@crawlee/browser-pool';
|
|
4
|
-
import { IncomingMessage } from 'http';
|
|
4
|
+
import { IncomingMessage } from 'node:http';
|
|
5
5
|
// @ts-ignore optional peer dependency
|
|
6
6
|
import { HTTPResponse, Protocol } from 'puppeteer';
|
|
7
7
|
import { Cookie, CookieJar } from 'tough-cookie';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEjC,OAAO,EAAE,MAAM,IAAI,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEjC,OAAO,EAAE,MAAM,IAAI,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,oBAAY,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC;AAK1E;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAsDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWzH;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,MAAM;IAK3D;;;OAGG;IACH,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,EAAE;IAKnD;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;;OAGG;IACH,SAAS,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,GAAG,MAAM;IAkB3E;;;;OAIG;IACH,SAAS,CAAC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,eAAe;IAavE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;;;OAIG;IACH,SAAS,CAAC,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { Log } from '@apify/log';
|
|
3
|
-
import { EventEmitter } from 'events';
|
|
3
|
+
import { EventEmitter } from 'node:events';
|
|
4
4
|
import { Configuration } from '../configuration';
|
|
5
5
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
6
6
|
import { Session, SessionOptions } from './session';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAgBK,QAAQ,CAAC,MAAM;IAf7D,GAAG,EAAE,GAAG,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,aAAa,CAAC;IACrC,aAAa,EAAG,aAAa,CAAC;IAC9B,QAAQ,EAAE,OAAO,EAAE,CAAM;IACzB,UAAU,uBAA8B;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB,EAAW,MAAM,gBAAkC;IA6C/F;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8BrD;;;OAGG;IACH,QAAQ;;;;;IAQR;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO;IAU7H;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SessionPool = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const
|
|
5
|
+
const node_events_1 = require("node:events");
|
|
6
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
7
7
|
const configuration_1 = require("../configuration");
|
|
8
8
|
const log_1 = require("../log");
|
|
@@ -66,7 +66,7 @@ const session_1 = require("./session");
|
|
|
66
66
|
* ```
|
|
67
67
|
* @category Scaling
|
|
68
68
|
*/
|
|
69
|
-
class SessionPool extends
|
|
69
|
+
class SessionPool extends node_events_1.EventEmitter {
|
|
70
70
|
/**
|
|
71
71
|
* @internal
|
|
72
72
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,6CAA2C;AAC3C,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,iEAA4D;AAE5D,uCAAoD;AA4CpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,MAAa,WAAY,SAAQ,0BAAY;IAazC;;OAEG;IACH,YAAY,UAA8B,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;QAC3F,KAAK,EAAE,CAAC;;;;;mBAD2C;;QAfvD;;;;;WAAS;QACT;;;;;WAAoB;QACpB;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;mBAAsB,EAAE;WAAC;QACzB;;;;mBAAa,IAAI,GAAG,EAAmB;WAAC;QACxC;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAwB;QACxB;;;;;WAAwC;QACxC;;;;;WAA6B;QAQzB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,GAAG,IAAI,EAElB,2BAA2B,EAC3B,eAAe,GAAG,wBAAwB,EAE1C,qBAAqB,EACrB,cAAc,GAAG,EAAE,EAEnB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,aAAa,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzG,yDAAyD;QACzD,8DAA8D;QAC9D,2IAA2I;QAC3I,8HAA8H;QAC9H,IAAI;QAEJ,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE;YACJ,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE;gBACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;aAC1F;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACjC;QAED,MAAM,UAAU,GAAG,OAAO,YAAY,iBAAO;YACzC,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,SAAS,EAAE;YACX,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;gBAAE,OAAO,OAAO,CAAC;YAClD,OAAO,SAAS,CAAC;SACpB;QAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC5B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;SAChC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE;YAC1B,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,6BAA6B,CAAC,WAAwB,EAAE,UAA+C,EAAE;QAC/G,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACxC,OAAO,IAAI,iBAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE;YACpD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,IAAI,iBAAO,CAAC,aAAa,CAAC,CAAC;YAEpD,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE;gBAC7B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;aACtC;SACJ;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B;QAC1C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AA/TD,kCA+TC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAWtI"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -113,7 +113,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
|
|
|
113
113
|
*
|
|
114
114
|
* ```javascript
|
|
115
115
|
* // Write a single row to the default dataset
|
|
116
|
-
* await
|
|
116
|
+
* await Dataset.pushData({ col1: 123, col2: 'val2' });
|
|
117
117
|
*
|
|
118
118
|
* // Open a named dataset
|
|
119
119
|
* const dataset = await Dataset.open('some-name');
|
|
@@ -144,7 +144,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
144
144
|
* It has no result, but throws on invalid args or other errors.
|
|
145
145
|
*
|
|
146
146
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
147
|
-
* otherwise the
|
|
147
|
+
* otherwise the crawler process might finish before the data is stored!
|
|
148
148
|
*
|
|
149
149
|
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
150
150
|
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
@@ -258,10 +258,35 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
258
258
|
*
|
|
259
259
|
* @param [datasetIdOrName]
|
|
260
260
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
261
|
-
* the function returns the default dataset associated with the
|
|
261
|
+
* the function returns the default dataset associated with the crawler run.
|
|
262
262
|
* @param [options] Storage manager options.
|
|
263
263
|
*/
|
|
264
264
|
static open<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
|
|
265
|
+
/**
|
|
266
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
|
267
|
+
*
|
|
268
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
269
|
+
* For example, calling the following code:
|
|
270
|
+
* ```javascript
|
|
271
|
+
* await Dataset.pushData({ myValue: 123 });
|
|
272
|
+
* ```
|
|
273
|
+
*
|
|
274
|
+
* is equivalent to:
|
|
275
|
+
* ```javascript
|
|
276
|
+
* const dataset = await Dataset.open();
|
|
277
|
+
* await dataset.pushData({ myValue: 123 });
|
|
278
|
+
* ```
|
|
279
|
+
*
|
|
280
|
+
* For more information, see {@link Dataset.open} and {@link Dataset.pushData}
|
|
281
|
+
*
|
|
282
|
+
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
283
|
+
* otherwise the crawler process might finish before the data are stored!
|
|
284
|
+
*
|
|
285
|
+
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
286
|
+
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
287
|
+
* @ignore
|
|
288
|
+
*/
|
|
289
|
+
static pushData(item: Dictionary): Promise<void>;
|
|
265
290
|
}
|
|
266
291
|
/**
|
|
267
292
|
* User-function used in the `Dataset.forEach()` API.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/F,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC/H,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,+BAAoC;IAEvC;;OAEG;gBACS,OAAO,EAAE,cAAc;IAMnC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;;;;;OAMG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAY7E;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAIjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAWlG;;;;;;;;;;;;;;OAcG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,CAAC;IAoB7G;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/F,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC/H,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,+BAAoC;IAEvC;;OAEG;gBACS,OAAO,EAAE,cAAc;IAMnC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;;;;;OAMG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAY7E;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAIjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAWlG;;;;;;;;;;;;;;OAcG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,CAAC;IAoB7G;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAUrJ;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;CAIzD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IAEjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAEhD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAElC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAE7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IAEnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAEtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAGD,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB"}
|
package/storages/dataset.js
CHANGED
|
@@ -110,7 +110,7 @@ exports.chunkBySize = chunkBySize;
|
|
|
110
110
|
*
|
|
111
111
|
* ```javascript
|
|
112
112
|
* // Write a single row to the default dataset
|
|
113
|
-
* await
|
|
113
|
+
* await Dataset.pushData({ col1: 123, col2: 'val2' });
|
|
114
114
|
*
|
|
115
115
|
* // Open a named dataset
|
|
116
116
|
* const dataset = await Dataset.open('some-name');
|
|
@@ -165,7 +165,7 @@ class Dataset {
|
|
|
165
165
|
* It has no result, but throws on invalid args or other errors.
|
|
166
166
|
*
|
|
167
167
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
168
|
-
* otherwise the
|
|
168
|
+
* otherwise the crawler process might finish before the data is stored!
|
|
169
169
|
*
|
|
170
170
|
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
171
171
|
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
@@ -351,7 +351,7 @@ class Dataset {
|
|
|
351
351
|
*
|
|
352
352
|
* @param [datasetIdOrName]
|
|
353
353
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
354
|
-
* the function returns the default dataset associated with the
|
|
354
|
+
* the function returns the default dataset associated with the crawler run.
|
|
355
355
|
* @param [options] Storage manager options.
|
|
356
356
|
*/
|
|
357
357
|
static async open(datasetIdOrName, options = {}) {
|
|
@@ -362,6 +362,34 @@ class Dataset {
|
|
|
362
362
|
const manager = new storage_manager_1.StorageManager(Dataset, options.config);
|
|
363
363
|
return manager.openStorage(datasetIdOrName);
|
|
364
364
|
}
|
|
365
|
+
/**
|
|
366
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
|
367
|
+
*
|
|
368
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
369
|
+
* For example, calling the following code:
|
|
370
|
+
* ```javascript
|
|
371
|
+
* await Dataset.pushData({ myValue: 123 });
|
|
372
|
+
* ```
|
|
373
|
+
*
|
|
374
|
+
* is equivalent to:
|
|
375
|
+
* ```javascript
|
|
376
|
+
* const dataset = await Dataset.open();
|
|
377
|
+
* await dataset.pushData({ myValue: 123 });
|
|
378
|
+
* ```
|
|
379
|
+
*
|
|
380
|
+
* For more information, see {@link Dataset.open} and {@link Dataset.pushData}
|
|
381
|
+
*
|
|
382
|
+
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
383
|
+
* otherwise the crawler process might finish before the data are stored!
|
|
384
|
+
*
|
|
385
|
+
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
386
|
+
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
387
|
+
* @ignore
|
|
388
|
+
*/
|
|
389
|
+
static async pushData(item) {
|
|
390
|
+
const dataset = await this.open();
|
|
391
|
+
return dataset.pushData(item);
|
|
392
|
+
}
|
|
365
393
|
}
|
|
366
394
|
exports.Dataset = Dataset;
|
|
367
395
|
//# sourceMappingURL=dataset.js.map
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;;AAAA,0CAAuD;AACvD,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA6B;AAE7B,uDAA0E;AAE1E,gBAAgB;AACH,QAAA,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;KAC/F;IAED,IAAI,OAAO,CAAC;IACZ,IAAI;QACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAClC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACvF;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;KACnG;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAtBD,8CAsBC;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE;YACjD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;SAC1B;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE;YAC7C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnB;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;SAChE;aAAM;YACH,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;SAC7D;KACJ;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA9BD,kCA8BC;AA6ED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,OAAO;IAMhB;;OAEG;IACH,YAAY,OAAuB;QARnC;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;;WAA4B;QAC5B;;;;mBAAM,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;WAAC;QAMnC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,IAAA,YAAE,EAAC,IAAI,EAAE,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,+BAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,+BAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC;SAC5B;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;SACzB;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;SAC/C;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE;gBAC9D,MAAM,IAAI,KAAK,CAAC,4GAA4G,CAAC,CAAC;aACjI;YACD,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/H,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,uCAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACjC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,MAAM,CAAI,QAAiC,EAAE,IAAO,EAAE,UAAkC,EAAE;QAC5F,IAAI,WAAW,GAAM,IAAI,CAAC;QAE1B,MAAM,WAAW,GAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACvD,OAAO,OAAO;iBACT,OAAO,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,OAAO,CAAC,KAAK,IAAI,WAAW,KAAK,SAAS;oBACtC,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7C,CAAC,CAAC;iBACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBACd,WAAW,GAAG,OAAY,CAAC;YAC/B,CAAC,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAuC,eAA+B,EAAE,UAAiC,EAAE;QACxH,IAAA,YAAE,EAAC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAgB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChD,CAAC;CACJ;
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;;AAAA,0CAAuD;AACvD,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA6B;AAE7B,uDAA0E;AAE1E,gBAAgB;AACH,QAAA,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;KAC/F;IAED,IAAI,OAAO,CAAC;IACZ,IAAI;QACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAClC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACvF;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;KACnG;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAtBD,8CAsBC;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE;YACjD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;SAC1B;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE;YAC7C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnB;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;SAChE;aAAM;YACH,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;SAC7D;KACJ;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA9BD,kCA8BC;AA6ED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,OAAO;IAMhB;;OAEG;IACH,YAAY,OAAuB;QARnC;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;;WAA4B;QAC5B;;;;mBAAM,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;WAAC;QAMnC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,IAAA,YAAE,EAAC,IAAI,EAAE,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,+BAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,+BAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC;SAC5B;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;SACzB;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;SAC/C;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE;gBAC9D,MAAM,IAAI,KAAK,CAAC,4GAA4G,CAAC,CAAC;aACjI;YACD,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/H,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,uCAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACjC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,MAAM,CAAI,QAAiC,EAAE,IAAO,EAAE,UAAkC,EAAE;QAC5F,IAAI,WAAW,GAAM,IAAI,CAAC;QAE1B,MAAM,WAAW,GAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACvD,OAAO,OAAO;iBACT,OAAO,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,OAAO,CAAC,KAAK,IAAI,WAAW,KAAK,SAAS;oBACtC,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7C,CAAC,CAAC;iBACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBACd,WAAW,GAAG,OAAY,CAAC;YAC/B,CAAC,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAuC,eAA+B,EAAE,UAAiC,EAAE;QACxH,IAAA,YAAE,EAAC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAgB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;CACJ;AArQD,0BAqQC"}
|