@crawlee/core 3.0.0-beta.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -133
- package/autoscaling/autoscaled_pool.d.ts +5 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +16 -16
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +6 -6
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +12 -8
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +1 -1
- package/autoscaling/system_status.d.ts.map +1 -1
- package/configuration.d.ts +77 -62
- package/configuration.d.ts.map +1 -1
- package/configuration.js +85 -87
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +7 -12
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_utils.d.ts +1 -1
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +25 -23
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +23 -23
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +11 -6
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +7 -2
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +16 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +22 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +12 -8
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +17 -6
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +3 -5
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -5
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -0
- package/index.d.ts.map +1 -1
- package/index.js +5 -0
- package/index.js.map +1 -1
- package/index.mjs +6 -0
- package/package.json +22 -27
- package/proxy_configuration.d.ts +7 -14
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -15
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +11 -6
- package/request.d.ts.map +1 -1
- package/request.js +5 -1
- package/request.js.map +1 -1
- package/router.d.ts +93 -0
- package/router.d.ts.map +1 -0
- package/router.js +137 -0
- package/router.js.map +1 -0
- package/serialization.d.ts.map +1 -1
- package/serialization.js +1 -2
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +10 -15
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +18 -18
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +3 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +4 -5
- package/session_pool/session_pool.js.map +1 -1
- package/session_pool/session_utils.d.ts +6 -5
- package/session_pool/session_utils.d.ts.map +1 -1
- package/session_pool/session_utils.js +3 -0
- package/session_pool/session_utils.js.map +1 -1
- package/storages/dataset.d.ts +37 -15
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +44 -12
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +1 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +1 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +121 -23
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +178 -24
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +32 -35
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +32 -24
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +12 -8
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +18 -10
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +9 -5
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +28 -18
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts +16 -0
- package/storages/utils.d.ts.map +1 -0
- package/storages/utils.js +26 -0
- package/storages/utils.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts +0 -2
- package/typedefs.d.ts.map +1 -1
- package/typedefs.js.map +1 -1
- package/validators.d.ts +5 -5
- package/validators.d.ts.map +1 -1
- package/validators.js.map +1 -1
package/router.js
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Router = void 0;
|
|
4
|
+
const errors_1 = require("./errors");
|
|
5
|
+
const defaultRoute = Symbol('default-route');
|
|
6
|
+
/**
|
|
7
|
+
* Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
11
|
+
*
|
|
12
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
13
|
+
*
|
|
14
|
+
* // we can also use factory methods for specific crawling contexts, the above equals to:
|
|
15
|
+
* // import { createCheerioRouter } from 'crawlee';
|
|
16
|
+
* // const router = createCheerioRouter();
|
|
17
|
+
*
|
|
18
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
19
|
+
* ctx.log.info('...');
|
|
20
|
+
* });
|
|
21
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
22
|
+
* ctx.log.info('...');
|
|
23
|
+
* });
|
|
24
|
+
*
|
|
25
|
+
* const crawler = new CheerioCrawler({
|
|
26
|
+
* requestHandler: router,
|
|
27
|
+
* });
|
|
28
|
+
* await crawler.run();
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* Alternatively we can use the default router instance from crawler object:
|
|
32
|
+
*
|
|
33
|
+
* ```ts
|
|
34
|
+
* import { CheerioCrawler } from 'crawlee';
|
|
35
|
+
*
|
|
36
|
+
* const crawler = new CheerioCrawler();
|
|
37
|
+
*
|
|
38
|
+
* crawler.router.addHandler('label-a', async (ctx) => {
|
|
39
|
+
* ctx.log.info('...');
|
|
40
|
+
* });
|
|
41
|
+
* crawler.router.addDefaultHandler(async (ctx) => {
|
|
42
|
+
* ctx.log.info('...');
|
|
43
|
+
* });
|
|
44
|
+
*
|
|
45
|
+
* await crawler.run();
|
|
46
|
+
* ```
|
|
47
|
+
*/
|
|
48
|
+
class Router {
|
|
49
|
+
/**
|
|
50
|
+
* use Router.create() instead!
|
|
51
|
+
* @ignore
|
|
52
|
+
*/
|
|
53
|
+
constructor() {
|
|
54
|
+
Object.defineProperty(this, "routes", {
|
|
55
|
+
enumerable: true,
|
|
56
|
+
configurable: true,
|
|
57
|
+
writable: true,
|
|
58
|
+
value: new Map()
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Registers new route handler for given label.
|
|
63
|
+
*/
|
|
64
|
+
addHandler(label, handler) {
|
|
65
|
+
this.validate(label);
|
|
66
|
+
this.routes.set(label, handler);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Registers default route handler.
|
|
70
|
+
*/
|
|
71
|
+
addDefaultHandler(handler) {
|
|
72
|
+
this.validate(defaultRoute);
|
|
73
|
+
this.routes.set(defaultRoute, handler);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
77
|
+
*/
|
|
78
|
+
getHandler(label) {
|
|
79
|
+
if (label && this.routes.has(label)) {
|
|
80
|
+
return this.routes.get(label);
|
|
81
|
+
}
|
|
82
|
+
if (this.routes.has(defaultRoute)) {
|
|
83
|
+
return this.routes.get(defaultRoute);
|
|
84
|
+
}
|
|
85
|
+
if (!label) {
|
|
86
|
+
throw new errors_1.MissingRouteError(`No default route set up!`);
|
|
87
|
+
}
|
|
88
|
+
throw new errors_1.MissingRouteError(`Route not found for label '${String(label)}' and no default route set up!`);
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Throws when the label already exists in our registry.
|
|
92
|
+
*/
|
|
93
|
+
validate(label) {
|
|
94
|
+
if (this.routes.has(label)) {
|
|
95
|
+
const message = label === defaultRoute
|
|
96
|
+
? `Default route is already defined!`
|
|
97
|
+
: `Route for label '${String(label)}' is already defined!`;
|
|
98
|
+
throw new Error(message);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
|
|
103
|
+
*
|
|
104
|
+
* ```ts
|
|
105
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
106
|
+
*
|
|
107
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
108
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
109
|
+
* ctx.log.info('...');
|
|
110
|
+
* });
|
|
111
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
112
|
+
* ctx.log.info('...');
|
|
113
|
+
* });
|
|
114
|
+
*
|
|
115
|
+
* const crawler = new CheerioCrawler({
|
|
116
|
+
* requestHandler: router,
|
|
117
|
+
* });
|
|
118
|
+
* await crawler.run();
|
|
119
|
+
* ```
|
|
120
|
+
*/
|
|
121
|
+
static create() {
|
|
122
|
+
const router = new Router();
|
|
123
|
+
const obj = Object.create(Function.prototype);
|
|
124
|
+
obj.addHandler = router.addHandler.bind(router);
|
|
125
|
+
obj.addDefaultHandler = router.addDefaultHandler.bind(router);
|
|
126
|
+
obj.getHandler = router.getHandler.bind(router);
|
|
127
|
+
const func = function (context) {
|
|
128
|
+
const { url, loadedUrl, label } = context.request;
|
|
129
|
+
context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
|
|
130
|
+
return router.getHandler(label)(context);
|
|
131
|
+
};
|
|
132
|
+
Object.setPrototypeOf(func, obj);
|
|
133
|
+
return func;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
exports.Router = Router;
|
|
137
|
+
//# sourceMappingURL=router.js.map
|
package/router.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":";;;AAEA,qCAA6C;AAE7C,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAM7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAa,MAAM;IAGf;;;OAGG;IACH;QANA;;;;mBAAmF,IAAI,GAAG,EAAE;WAAC;IAMpE,CAAC;IAE1B;;OAEG;IACH,UAAU,CAAC,KAAsB,EAAE,OAA0C;QACzE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAA0C;QACxD,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAuB;QAC9B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;SAClC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;YAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAE,CAAC;SACzC;QAED,IAAI,CAAC,KAAK,EAAE;YACR,MAAM,IAAI,0BAAiB,CAAC,0BAA0B,CAAC,CAAC;SAC3D;QAED,MAAM,IAAI,0BAAiB,CAAC,8BAA8B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC7G,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,KAAsB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACxB,MAAM,OAAO,GAAG,KAAK,KAAK,YAAY;gBAClC,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,oBAAoB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;SAC5B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM;QACT,MAAM,MAAM,GAAG,IAAI,MAAM,EAAW,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE9C,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAG,UAAU,OAAgB;YACnC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC,CAAC;YACpE,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC;QAEF,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEjC,OAAO,IAAyC,CAAC;IACrD,CAAC;CACJ;AA9FD,wBA8FC"}
|
package/serialization.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;AACA,OAAO,EAA8B,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAkDxE;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAUlE;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAWtG;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,QAAQ,
|
|
1
|
+
{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;AACA,OAAO,EAA8B,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAkDxE;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAUlE;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAWtG;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,QAAQ,CAclE"}
|
package/serialization.js
CHANGED
|
@@ -112,8 +112,7 @@ function createDeserialize(compressedData) {
|
|
|
112
112
|
const streamArray = StreamArray_1.default.withParser();
|
|
113
113
|
const destination = pluckValue(streamArray);
|
|
114
114
|
(0, stream_1.pipeline)(stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), destination,
|
|
115
|
-
//
|
|
116
|
-
// @ts-expect-error Somethings wrong here, the types are wrong but tests fail if we correct the code to make them right
|
|
115
|
+
// @ts-expect-error Something's wrong here, the types are wrong but tests fail if we correct the code to make them right
|
|
117
116
|
(err) => destination.emit(err));
|
|
118
117
|
return destination;
|
|
119
118
|
}
|
package/serialization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,kEAA6B;AAC7B,kEAA6B;AAE7B,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,mBAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,
|
|
1
|
+
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,kEAA6B;AAC7B,kEAA6B;AAE7B,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,mBAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wHAAwH;IACxH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAdD,8CAcC;AAED,SAAS,oBAAoB,CAA4B,UAA0C,EAAE;IACjG,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,iBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { Log } from '@apify/log';
|
|
3
|
-
import { Cookie as
|
|
4
|
-
import { IncomingMessage } from 'node:http';
|
|
5
|
-
// @ts-ignore optional peer dependency
|
|
6
|
-
import { HTTPResponse, Protocol } from 'puppeteer';
|
|
2
|
+
import type { Log } from '@apify/log';
|
|
3
|
+
import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@crawlee/types';
|
|
4
|
+
import type { IncomingMessage } from 'node:http';
|
|
7
5
|
import { Cookie, CookieJar } from 'tough-cookie';
|
|
8
|
-
import { Dictionary } from '../typedefs';
|
|
9
|
-
import { SessionPool } from './session_pool';
|
|
10
|
-
export declare type PuppeteerCookie = Protocol.Network.Cookie | BrowserPoolCookie;
|
|
11
6
|
/**
|
|
12
7
|
* Persistable {@link Session} state.
|
|
13
8
|
*/
|
|
@@ -64,7 +59,7 @@ export interface SessionOptions {
|
|
|
64
59
|
*/
|
|
65
60
|
maxUsageCount?: number;
|
|
66
61
|
/** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
|
|
67
|
-
sessionPool?: SessionPool;
|
|
62
|
+
sessionPool?: import('./session_pool').SessionPool;
|
|
68
63
|
log?: Log;
|
|
69
64
|
errorScore?: number;
|
|
70
65
|
cookieJar?: CookieJar;
|
|
@@ -154,7 +149,7 @@ export declare class Session {
|
|
|
154
149
|
*
|
|
155
150
|
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
156
151
|
*/
|
|
157
|
-
setCookiesFromResponse(response: IncomingMessage |
|
|
152
|
+
setCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | {
|
|
158
153
|
headers: Dictionary<string | string[]>;
|
|
159
154
|
url: string;
|
|
160
155
|
}): void;
|
|
@@ -171,12 +166,12 @@ export declare class Session {
|
|
|
171
166
|
* ]
|
|
172
167
|
* ```
|
|
173
168
|
*/
|
|
174
|
-
|
|
169
|
+
setCookies(cookies: CookieObject[], url: string): void;
|
|
175
170
|
/**
|
|
176
171
|
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
177
172
|
* @param url website url. Only cookies stored for this url will be returned
|
|
178
173
|
*/
|
|
179
|
-
|
|
174
|
+
getCookies(url: string): CookieObject[];
|
|
180
175
|
/**
|
|
181
176
|
* Returns cookies saved with the session in the typical
|
|
182
177
|
* key1=value1; key2=value2 format, ready to be used in
|
|
@@ -190,15 +185,15 @@ export declare class Session {
|
|
|
190
185
|
setCookie(rawCookie: string, url: string): void;
|
|
191
186
|
/**
|
|
192
187
|
* Transforms puppeteer cookie to tough-cookie.
|
|
193
|
-
* @param
|
|
188
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
194
189
|
*/
|
|
195
|
-
protected
|
|
190
|
+
protected _cookieObjectToTough(cookieObject: CookieObject): Cookie;
|
|
196
191
|
/**
|
|
197
192
|
* Transforms tough-cookie to puppeteer cookie.
|
|
198
193
|
* @param toughCookie Cookie from CookieJar
|
|
199
194
|
* @return Cookie from Puppeteer
|
|
200
195
|
*/
|
|
201
|
-
protected _toughCookieToPuppeteer(toughCookie: Cookie):
|
|
196
|
+
protected _toughCookieToPuppeteer(toughCookie: Cookie): CookieObject;
|
|
202
197
|
/**
|
|
203
198
|
* Sets cookies.
|
|
204
199
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAGjD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AASjD;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,gBAAgB,EAAE,WAAW,CAAC;IAEnD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAsDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWhI;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;;OAGG;IACH,SAAS,CAAC,oBAAoB,CAAC,YAAY,EAAE,YAAY,GAAG,MAAM;IAkBlE;;;;OAIG;IACH,SAAS,CAAC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY;IAapE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;;;OAIG;IACH,SAAS,CAAC,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -3,12 +3,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Session = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const utilities_1 = require("@apify/utilities");
|
|
6
|
+
const node_events_1 = require("node:events");
|
|
6
7
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
7
8
|
const tough_cookie_1 = require("tough-cookie");
|
|
8
9
|
const constants_1 = require("../constants");
|
|
9
10
|
const log_1 = require("../log");
|
|
10
11
|
const events_1 = require("./events");
|
|
11
|
-
const session_pool_1 = require("./session_pool");
|
|
12
12
|
const session_utils_1 = require("./session_utils");
|
|
13
13
|
// CONSTANTS
|
|
14
14
|
const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
|
|
@@ -102,7 +102,7 @@ class Session {
|
|
|
102
102
|
value: void 0
|
|
103
103
|
});
|
|
104
104
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
105
|
-
sessionPool: ow_1.default.object.instanceOf(
|
|
105
|
+
sessionPool: ow_1.default.object.instanceOf(node_events_1.EventEmitter),
|
|
106
106
|
id: ow_1.default.optional.string,
|
|
107
107
|
cookieJar: ow_1.default.optional.object,
|
|
108
108
|
maxAgeSecs: ow_1.default.optional.number,
|
|
@@ -132,7 +132,7 @@ class Session {
|
|
|
132
132
|
this.errorScore = errorScore; // indicates number of markBaded request with the session
|
|
133
133
|
this.maxUsageCount = maxUsageCount;
|
|
134
134
|
this.sessionPool = sessionPool;
|
|
135
|
-
this.
|
|
135
|
+
this._cookieObjectToTough = this._cookieObjectToTough.bind(this);
|
|
136
136
|
}
|
|
137
137
|
/**
|
|
138
138
|
* Indicates whether the session is blocked.
|
|
@@ -262,15 +262,15 @@ class Session {
|
|
|
262
262
|
* ]
|
|
263
263
|
* ```
|
|
264
264
|
*/
|
|
265
|
-
|
|
266
|
-
const normalizedCookies = cookies.map(this.
|
|
265
|
+
setCookies(cookies, url) {
|
|
266
|
+
const normalizedCookies = cookies.map(this._cookieObjectToTough);
|
|
267
267
|
this._setCookies(normalizedCookies, url);
|
|
268
268
|
}
|
|
269
269
|
/**
|
|
270
270
|
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
271
271
|
* @param url website url. Only cookies stored for this url will be returned
|
|
272
272
|
*/
|
|
273
|
-
|
|
273
|
+
getCookies(url) {
|
|
274
274
|
const cookies = this.cookieJar.getCookiesSync(url);
|
|
275
275
|
return cookies.map(this._toughCookieToPuppeteer);
|
|
276
276
|
}
|
|
@@ -291,22 +291,22 @@ class Session {
|
|
|
291
291
|
}
|
|
292
292
|
/**
|
|
293
293
|
* Transforms puppeteer cookie to tough-cookie.
|
|
294
|
-
* @param
|
|
294
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
295
295
|
*/
|
|
296
|
-
|
|
297
|
-
const isExpiresValid =
|
|
298
|
-
const expires = isExpiresValid ? new Date(
|
|
299
|
-
const domain = typeof
|
|
300
|
-
?
|
|
301
|
-
:
|
|
296
|
+
_cookieObjectToTough(cookieObject) {
|
|
297
|
+
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
|
|
298
|
+
const expires = isExpiresValid ? new Date(cookieObject.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
|
|
299
|
+
const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
|
|
300
|
+
? cookieObject.domain.slice(1)
|
|
301
|
+
: cookieObject.domain;
|
|
302
302
|
return new tough_cookie_1.Cookie({
|
|
303
|
-
key:
|
|
304
|
-
value:
|
|
303
|
+
key: cookieObject.name,
|
|
304
|
+
value: cookieObject.value,
|
|
305
305
|
expires,
|
|
306
306
|
domain,
|
|
307
|
-
path:
|
|
308
|
-
secure:
|
|
309
|
-
httpOnly:
|
|
307
|
+
path: cookieObject.path,
|
|
308
|
+
secure: cookieObject.secure,
|
|
309
|
+
httpOnly: cookieObject.httpOnly,
|
|
310
310
|
});
|
|
311
311
|
}
|
|
312
312
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,oDAAoB;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,6CAA2C;AAC3C,oDAAoB;AACpB,+CAAiD;AACjD,4CAAoD;AACpD,gCAA2C;AAC3C,qCAAiD;AACjD,mDAAyD;AAEzD,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAA0D;QAC1D;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAY,CAAC;YAC/C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAEjF,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAyG;QAC5H,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,sCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACjE,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACO,oBAAoB,CAAC,YAA0B;QACrD,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;QACpH,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChI,MAAM,MAAM,GAAG,OAAO,YAAY,CAAC,MAAM,KAAK,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YACzF,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;QAE1B,OAAO,IAAI,qBAAM,CAAC;YACd,GAAG,EAAE,YAAY,CAAC,IAAI;YACtB,KAAK,EAAE,YAAY,CAAC,KAAK;YACzB,OAAO;YACP,MAAM;YACN,IAAI,EAAE,YAAY,CAAC,IAAI;YACvB,MAAM,EAAE,YAAY,CAAC,MAAM;YAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;SAClC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACO,uBAAuB,CAAC,WAAmB;QACjD,OAAO;YACH,IAAI,EAAE,WAAW,CAAC,GAAG;YACrB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,iFAAiF;YACjF,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;YACvD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;SACjB,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;;;OAIG;IACO,+BAA+B,CAAC,UAAkB;QACxD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AAlUD,0BAkUC"}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { Log } from '@apify/log';
|
|
2
|
+
import type { Log } from '@apify/log';
|
|
3
3
|
import { EventEmitter } from 'node:events';
|
|
4
4
|
import { Configuration } from '../configuration';
|
|
5
5
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
6
|
-
import {
|
|
6
|
+
import type { SessionOptions } from './session';
|
|
7
|
+
import { Session } from './session';
|
|
7
8
|
/**
|
|
8
9
|
* Factory user-function which creates customized {@link Session} instances.
|
|
9
10
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAgBK,QAAQ,CAAC,MAAM;IAf7D,GAAG,EAAE,GAAG,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,aAAa,CAAC;IACrC,aAAa,EAAG,aAAa,CAAC;IAC9B,QAAQ,EAAE,OAAO,EAAE,CAAM;IACzB,UAAU,uBAA8B;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB,EAAW,MAAM,gBAAkC;IA6C/F;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8BrD;;;OAGG;IACH,QAAQ;;;;;IAQR;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO;IAU7H;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}
|
|
@@ -188,11 +188,10 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
188
188
|
*/
|
|
189
189
|
async initialize() {
|
|
190
190
|
this.keyValueStore = await key_value_store_1.KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
// }
|
|
191
|
+
if (!this.persistStateKeyValueStoreId) {
|
|
192
|
+
// eslint-disable-next-line max-len
|
|
193
|
+
this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
|
|
194
|
+
}
|
|
196
195
|
// in case of migration happened and SessionPool state should be restored from the keyValueStore.
|
|
197
196
|
await this._maybeLoadSessionPool();
|
|
198
197
|
this._listener = this.persistState.bind(this);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,6CAA2C;
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,6CAA2C;AAE3C,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,iEAA4D;AAE5D,uCAAoC;AA6CpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,MAAa,WAAY,SAAQ,0BAAY;IAazC;;OAEG;IACH,YAAY,UAA8B,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;QAC3F,KAAK,EAAE,CAAC;;;;;mBAD2C;;QAfvD;;;;;WAAS;QACT;;;;;WAAoB;QACpB;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;mBAAsB,EAAE;WAAC;QACzB;;;;mBAAa,IAAI,GAAG,EAAmB;WAAC;QACxC;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAwB;QACxB;;;;;WAAwC;QACxC;;;;;WAA6B;QAQzB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,GAAG,IAAI,EAElB,2BAA2B,EAC3B,eAAe,GAAG,wBAAwB,EAE1C,qBAAqB,EACrB,cAAc,GAAG,EAAE,EAEnB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,aAAa,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzG,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACnC,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;SAC5K;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE;YACJ,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE;gBACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;aAC1F;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACjC;QAED,MAAM,UAAU,GAAG,OAAO,YAAY,iBAAO;YACzC,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,SAAS,EAAE;YACX,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;gBAAE,OAAO,OAAO,CAAC;YAClD,OAAO,SAAS,CAAC;SACpB;QAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC5B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;SAChC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE;YAC1B,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,6BAA6B,CAAC,WAAwB,EAAE,UAA+C,EAAE;QAC/G,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACxC,OAAO,IAAI,iBAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE;YACpD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,IAAI,iBAAO,CAAC,aAAa,CAAC,CAAC;YAEpD,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE;gBAC7B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;aACtC;SACJ;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B;QAC1C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AA9TD,kCA8TC"}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
import { HTTPResponse } from 'puppeteer';
|
|
2
|
+
import type { BrowserLikeResponse, Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { IncomingMessage } from 'node:http';
|
|
5
4
|
import { Cookie } from 'tough-cookie';
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
/**
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | {
|
|
8
9
|
headers: Dictionary<string | string[]>;
|
|
9
10
|
}): Cookie[];
|
|
10
11
|
//# sourceMappingURL=session_utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAW7I"}
|
|
@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.getCookiesFromResponse = void 0;
|
|
4
4
|
const tough_cookie_1 = require("tough-cookie");
|
|
5
5
|
const errors_1 = require("./errors");
|
|
6
|
+
/**
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
6
9
|
function getCookiesFromResponse(response) {
|
|
7
10
|
const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
|
|
8
11
|
const cookieHeader = headers['set-cookie'] || '';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;AAEA,+CAAsC;
|
|
1
|
+
{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;AAEA,+CAAsC;AACtC,qCAA4C;AAE5C;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAA4F;IAC/H,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI;QACA,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;KACvC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;KAC5C;AACL,CAAC;AAXD,wDAWC"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type { DatasetClient, DatasetInfo, PaginatedList, StorageClient } from '@crawlee/types';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import type { DatasetClient, DatasetInfo, PaginatedList, StorageClient, Dictionary } from '@crawlee/types';
|
|
2
|
+
import { Configuration } from '../configuration';
|
|
3
|
+
import type { Awaitable } from '../typedefs';
|
|
4
|
+
import type { StorageManagerOptions } from './storage_manager';
|
|
4
5
|
/** @internal */
|
|
5
6
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
6
7
|
/**
|
|
@@ -113,7 +114,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
|
|
|
113
114
|
*
|
|
114
115
|
* ```javascript
|
|
115
116
|
* // Write a single row to the default dataset
|
|
116
|
-
* await
|
|
117
|
+
* await Dataset.pushData({ col1: 123, col2: 'val2' });
|
|
117
118
|
*
|
|
118
119
|
* // Open a named dataset
|
|
119
120
|
* const dataset = await Dataset.open('some-name');
|
|
@@ -130,6 +131,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
|
|
|
130
131
|
* @category Result Stores
|
|
131
132
|
*/
|
|
132
133
|
export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
134
|
+
readonly config: Configuration;
|
|
133
135
|
id: string;
|
|
134
136
|
name?: string;
|
|
135
137
|
client: DatasetClient<Data>;
|
|
@@ -137,14 +139,14 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
137
139
|
/**
|
|
138
140
|
* @internal
|
|
139
141
|
*/
|
|
140
|
-
constructor(options: DatasetOptions);
|
|
142
|
+
constructor(options: DatasetOptions, config?: Configuration);
|
|
141
143
|
/**
|
|
142
144
|
* Stores an object or an array of objects to the dataset.
|
|
143
145
|
* The function returns a promise that resolves when the operation finishes.
|
|
144
146
|
* It has no result, but throws on invalid args or other errors.
|
|
145
147
|
*
|
|
146
148
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
147
|
-
* otherwise the
|
|
149
|
+
* otherwise the crawler process might finish before the data is stored!
|
|
148
150
|
*
|
|
149
151
|
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
150
152
|
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
@@ -165,10 +167,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
165
167
|
pushData(data: Data | Data[]): Promise<void>;
|
|
166
168
|
/**
|
|
167
169
|
* Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
|
|
168
|
-
*
|
|
169
|
-
* If you need to get data in an unparsed format, use the {@link Actor.newClient} function to get a new
|
|
170
|
-
* `apify-client` instance and call
|
|
171
|
-
* [`datasetClient.downloadItems()`](https://github.com/apify/apify-client-js#DatasetClient+downloadItems)
|
|
172
170
|
*/
|
|
173
171
|
getData(options?: DatasetDataOptions): Promise<PaginatedList<Data>>;
|
|
174
172
|
/**
|
|
@@ -190,7 +188,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
190
188
|
* modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
|
|
191
189
|
* accessedAt: new Date("2015-12-14T08:36:13.202Z"),
|
|
192
190
|
* itemCount: 14,
|
|
193
|
-
* cleanItemCount: 10
|
|
194
191
|
* }
|
|
195
192
|
* ```
|
|
196
193
|
*/
|
|
@@ -258,10 +255,35 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
258
255
|
*
|
|
259
256
|
* @param [datasetIdOrName]
|
|
260
257
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
261
|
-
* the function returns the default dataset associated with the
|
|
258
|
+
* the function returns the default dataset associated with the crawler run.
|
|
262
259
|
* @param [options] Storage manager options.
|
|
263
260
|
*/
|
|
264
261
|
static open<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
|
|
262
|
+
/**
|
|
263
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
|
264
|
+
*
|
|
265
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
266
|
+
* For example, calling the following code:
|
|
267
|
+
* ```javascript
|
|
268
|
+
* await Dataset.pushData({ myValue: 123 });
|
|
269
|
+
* ```
|
|
270
|
+
*
|
|
271
|
+
* is equivalent to:
|
|
272
|
+
* ```javascript
|
|
273
|
+
* const dataset = await Dataset.open();
|
|
274
|
+
* await dataset.pushData({ myValue: 123 });
|
|
275
|
+
* ```
|
|
276
|
+
*
|
|
277
|
+
* For more information, see {@link Dataset.open} and {@link Dataset.pushData}
|
|
278
|
+
*
|
|
279
|
+
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
280
|
+
* otherwise the crawler process might finish before the data are stored!
|
|
281
|
+
*
|
|
282
|
+
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
283
|
+
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
284
|
+
* @ignore
|
|
285
|
+
*/
|
|
286
|
+
static pushData<Data extends Dictionary = Dictionary>(item: Data | Data[]): Promise<void>;
|
|
265
287
|
}
|
|
266
288
|
/**
|
|
267
289
|
* User-function used in the `Dataset.forEach()` API.
|
|
@@ -269,7 +291,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
269
291
|
export interface DatasetConsumer<Data> {
|
|
270
292
|
/**
|
|
271
293
|
* @param item Current {@link Dataset} entry being processed.
|
|
272
|
-
* @param index Position of current {Dataset} entry.
|
|
294
|
+
* @param index Position of current {@link Dataset} entry.
|
|
273
295
|
*/
|
|
274
296
|
(item: Data, index: number): Awaitable<void>;
|
|
275
297
|
}
|
|
@@ -280,7 +302,7 @@ export interface DatasetMapper<Data, R> {
|
|
|
280
302
|
/**
|
|
281
303
|
* User-function used in the `Dataset.map()` API.
|
|
282
304
|
* @param item Current {@link Dataset} entry being processed.
|
|
283
|
-
* @param index Position of current {Dataset} entry.
|
|
305
|
+
* @param index Position of current {@link Dataset} entry.
|
|
284
306
|
*/
|
|
285
307
|
(item: Data, index: number): Awaitable<R>;
|
|
286
308
|
}
|
|
@@ -291,7 +313,7 @@ export interface DatasetReducer<T, Data> {
|
|
|
291
313
|
/**
|
|
292
314
|
* @param memo Previous state of the reduction.
|
|
293
315
|
* @param item Current {@link Dataset} entry being processed.
|
|
294
|
-
* @param index Position of current {Dataset} entry.
|
|
316
|
+
* @param index Position of current {@link Dataset} entry.
|
|
295
317
|
*/
|
|
296
318
|
(memo: T, item: Data, index: number): Awaitable<T>;
|
|
297
319
|
}
|