@crawlee/core 3.0.0-beta.5 → 3.0.0-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -71
- package/autoscaling/autoscaled_pool.d.ts +3 -3
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +1 -1
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +5 -5
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +2 -2
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +1 -1
- package/autoscaling/system_status.d.ts.map +1 -1
- package/configuration.d.ts +12 -11
- package/configuration.d.ts.map +1 -1
- package/configuration.js +10 -8
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +12 -7
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_utils.d.ts +1 -1
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +20 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +13 -13
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +11 -6
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +7 -2
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +16 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +22 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +6 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -4
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +3 -5
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -5
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +3 -0
- package/index.d.ts.map +1 -1
- package/index.js +2 -0
- package/index.js.map +1 -1
- package/index.mjs +5 -0
- package/package.json +17 -14
- package/proxy_configuration.d.ts +7 -14
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -15
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +11 -6
- package/request.d.ts.map +1 -1
- package/request.js +5 -1
- package/request.js.map +1 -1
- package/router.d.ts +93 -0
- package/router.d.ts.map +1 -0
- package/router.js +137 -0
- package/router.js.map +1 -0
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +5 -5
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +3 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +4 -5
- package/session_pool/session_pool.js.map +1 -1
- package/session_pool/session_utils.d.ts +6 -3
- package/session_pool/session_utils.d.ts.map +1 -1
- package/session_pool/session_utils.js +3 -0
- package/session_pool/session_utils.js.map +1 -1
- package/storages/dataset.d.ts +34 -12
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +44 -12
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +1 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +1 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +120 -22
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +178 -24
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +8 -7
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +5 -3
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +12 -8
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +18 -6
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +9 -5
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +28 -9
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts +16 -0
- package/storages/utils.d.ts.map +1 -0
- package/storages/utils.js +26 -0
- package/storages/utils.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts +0 -2
- package/typedefs.d.ts.map +1 -1
- package/typedefs.js.map +1 -1
- package/validators.d.ts +5 -5
- package/validators.d.ts.map +1 -1
- package/validators.js.map +1 -1
package/router.js
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Router = void 0;
|
|
4
|
+
const errors_1 = require("./errors");
|
|
5
|
+
const defaultRoute = Symbol('default-route');
|
|
6
|
+
/**
|
|
7
|
+
* Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from '@crawlee/cheerio';
|
|
11
|
+
*
|
|
12
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
13
|
+
*
|
|
14
|
+
* // we can also use factory methods for specific crawling contexts, the above equals to:
|
|
15
|
+
* // import { createCheerioRouter } from '@crawlee/cheerio';
|
|
16
|
+
* // const router = createCheerioRouter();
|
|
17
|
+
*
|
|
18
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
19
|
+
* ctx.log.info('...');
|
|
20
|
+
* });
|
|
21
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
22
|
+
* ctx.log.info('...');
|
|
23
|
+
* });
|
|
24
|
+
*
|
|
25
|
+
* const crawler = new CheerioCrawler({
|
|
26
|
+
* requestHandler: router,
|
|
27
|
+
* });
|
|
28
|
+
* await crawler.run();
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* Alternatively we can use the default router instance from crawler object:
|
|
32
|
+
*
|
|
33
|
+
* ```ts
|
|
34
|
+
* import { CheerioCrawler } from '@crawlee/cheerio';
|
|
35
|
+
*
|
|
36
|
+
* const crawler = new CheerioCrawler();
|
|
37
|
+
*
|
|
38
|
+
* crawler.router.addHandler('label-a', async (ctx) => {
|
|
39
|
+
* ctx.log.info('...');
|
|
40
|
+
* });
|
|
41
|
+
* crawler.router.addDefaultHandler(async (ctx) => {
|
|
42
|
+
* ctx.log.info('...');
|
|
43
|
+
* });
|
|
44
|
+
*
|
|
45
|
+
* await crawler.run();
|
|
46
|
+
* ```
|
|
47
|
+
*/
|
|
48
|
+
class Router {
|
|
49
|
+
/**
|
|
50
|
+
* use Router.create() instead!
|
|
51
|
+
* @ignore
|
|
52
|
+
*/
|
|
53
|
+
constructor() {
|
|
54
|
+
Object.defineProperty(this, "routes", {
|
|
55
|
+
enumerable: true,
|
|
56
|
+
configurable: true,
|
|
57
|
+
writable: true,
|
|
58
|
+
value: new Map()
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Registers new route handler for given label.
|
|
63
|
+
*/
|
|
64
|
+
addHandler(label, handler) {
|
|
65
|
+
this.validate(label);
|
|
66
|
+
this.routes.set(label, handler);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Registers default route handler.
|
|
70
|
+
*/
|
|
71
|
+
addDefaultHandler(handler) {
|
|
72
|
+
this.validate(defaultRoute);
|
|
73
|
+
this.routes.set(defaultRoute, handler);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
77
|
+
*/
|
|
78
|
+
getHandler(label) {
|
|
79
|
+
if (label && this.routes.has(label)) {
|
|
80
|
+
return this.routes.get(label);
|
|
81
|
+
}
|
|
82
|
+
if (this.routes.has(defaultRoute)) {
|
|
83
|
+
return this.routes.get(defaultRoute);
|
|
84
|
+
}
|
|
85
|
+
if (!label) {
|
|
86
|
+
throw new errors_1.MissingRouteError(`No default route set up!`);
|
|
87
|
+
}
|
|
88
|
+
throw new errors_1.MissingRouteError(`Route not found for label '${String(label)}' and no default route set up!`);
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Throws when the label already exists in our registry.
|
|
92
|
+
*/
|
|
93
|
+
validate(label) {
|
|
94
|
+
if (this.routes.has(label)) {
|
|
95
|
+
const message = label === defaultRoute
|
|
96
|
+
? `Default route is already defined!`
|
|
97
|
+
: `Route for label '${String(label)}' is already defined!`;
|
|
98
|
+
throw new Error(message);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
|
|
103
|
+
*
|
|
104
|
+
* ```ts
|
|
105
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from '@crawlee/cheerio';
|
|
106
|
+
*
|
|
107
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
108
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
109
|
+
* ctx.log.info('...');
|
|
110
|
+
* });
|
|
111
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
112
|
+
* ctx.log.info('...');
|
|
113
|
+
* });
|
|
114
|
+
*
|
|
115
|
+
* const crawler = new CheerioCrawler({
|
|
116
|
+
* requestHandler: router,
|
|
117
|
+
* });
|
|
118
|
+
* await crawler.run();
|
|
119
|
+
* ```
|
|
120
|
+
*/
|
|
121
|
+
static create() {
|
|
122
|
+
const router = new Router();
|
|
123
|
+
const obj = Object.create(Function.prototype);
|
|
124
|
+
obj.addHandler = router.addHandler.bind(router);
|
|
125
|
+
obj.addDefaultHandler = router.addDefaultHandler.bind(router);
|
|
126
|
+
obj.getHandler = router.getHandler.bind(router);
|
|
127
|
+
const func = function (context) {
|
|
128
|
+
const { url, loadedUrl, label } = context.request;
|
|
129
|
+
context.log.info('Page opened.', { label, url: loadedUrl ?? url });
|
|
130
|
+
return router.getHandler(label)(context);
|
|
131
|
+
};
|
|
132
|
+
Object.setPrototypeOf(func, obj);
|
|
133
|
+
return func;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
exports.Router = Router;
|
|
137
|
+
//# sourceMappingURL=router.js.map
|
package/router.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":";;;AAEA,qCAA6C;AAE7C,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAM7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAa,MAAM;IAGf;;;OAGG;IACH;QANA;;;;mBAAmF,IAAI,GAAG,EAAE;WAAC;IAMpE,CAAC;IAE1B;;OAEG;IACH,UAAU,CAAC,KAAsB,EAAE,OAA0C;QACzE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAA0C;QACxD,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAuB;QAC9B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;SAClC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;YAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAE,CAAC;SACzC;QAED,IAAI,CAAC,KAAK,EAAE;YACR,MAAM,IAAI,0BAAiB,CAAC,0BAA0B,CAAC,CAAC;SAC3D;QAED,MAAM,IAAI,0BAAiB,CAAC,8BAA8B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC7G,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,KAAsB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACxB,MAAM,OAAO,GAAG,KAAK,KAAK,YAAY;gBAClC,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,oBAAoB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;SAC5B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM;QACT,MAAM,MAAM,GAAG,IAAI,MAAM,EAAW,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE9C,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAG,UAAU,OAAgB;YACnC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC;QAEF,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEjC,OAAO,IAAyC,CAAC;IACrD,CAAC;CACJ;AA9FD,wBA8FC"}
|
package/serialization.js
CHANGED
|
@@ -113,7 +113,7 @@ function createDeserialize(compressedData) {
|
|
|
113
113
|
const destination = pluckValue(streamArray);
|
|
114
114
|
(0, stream_1.pipeline)(stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), destination,
|
|
115
115
|
// TODO: figure this out
|
|
116
|
-
// @ts-expect-error
|
|
116
|
+
// @ts-expect-error Something's wrong here, the types are wrong but tests fail if we correct the code to make them right
|
|
117
117
|
(err) => destination.emit(err));
|
|
118
118
|
return destination;
|
|
119
119
|
}
|
package/serialization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,kEAA6B;AAC7B,kEAA6B;AAE7B,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,mBAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wBAAwB;IACxB,
|
|
1
|
+
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,kEAA6B;AAC7B,kEAA6B;AAE7B,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,mBAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wBAAwB;IACxB,wHAAwH;IACxH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAfD,8CAeC;AAED,SAAS,oBAAoB,CAA4B,UAA0C,EAAE;IACjG,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,iBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { Log } from '@apify/log';
|
|
3
|
-
import { Cookie as BrowserPoolCookie } from '@crawlee/browser-pool';
|
|
4
|
-
import {
|
|
2
|
+
import type { Log } from '@apify/log';
|
|
3
|
+
import type { Cookie as BrowserPoolCookie } from '@crawlee/browser-pool';
|
|
4
|
+
import type { Dictionary } from '@crawlee/types';
|
|
5
|
+
import type { IncomingMessage } from 'node:http';
|
|
5
6
|
// @ts-ignore optional peer dependency
|
|
6
|
-
import { HTTPResponse, Protocol } from 'puppeteer';
|
|
7
|
+
import type { HTTPResponse, Protocol } from 'puppeteer';
|
|
7
8
|
import { Cookie, CookieJar } from 'tough-cookie';
|
|
8
|
-
import { Dictionary } from '../typedefs';
|
|
9
9
|
import { SessionPool } from './session_pool';
|
|
10
10
|
export declare type PuppeteerCookie = Protocol.Network.Cookie | BrowserPoolCookie;
|
|
11
11
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,MAAM,IAAI,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACzE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,oBAAY,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC;AAK1E;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAsDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWzH;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,MAAM;IAK3D;;;OAGG;IACH,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,EAAE;IAKnD;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;;OAGG;IACH,SAAS,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,GAAG,MAAM;IAkB3E;;;;OAIG;IACH,SAAS,CAAC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,eAAe;IAavE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;;;OAIG;IACH,SAAS,CAAC,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAIxD,oDAAoB;AAEpB,+CAAiD;AACjD,4CAAoD;AACpD,gCAA2C;AAC3C,qCAAiD;AACjD,iDAA6C;AAC7C,mDAAyD;AAIzD,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAAiC;QACjC;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAW,CAAC;YAC9C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAEjF,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAkG;QACrH,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,sCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAA0B,EAAE,GAAW;QACvD,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpE,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,GAAW;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACO,uBAAuB,CAAC,eAAgC;QAC9D,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,IAAI,OAAO,eAAe,CAAC,OAAO,KAAK,QAAQ,IAAI,eAAe,CAAC,OAAO,GAAG,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnI,MAAM,MAAM,GAAG,OAAO,eAAe,CAAC,MAAM,KAAK,QAAQ,IAAI,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAC/F,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QAE7B,OAAO,IAAI,qBAAM,CAAC;YACd,GAAG,EAAE,eAAe,CAAC,IAAI;YACzB,KAAK,EAAE,eAAe,CAAC,KAAK;YAC5B,OAAO;YACP,MAAM;YACN,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,MAAM,EAAE,eAAe,CAAC,MAAM;YAC9B,QAAQ,EAAE,eAAe,CAAC,QAAQ;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACO,uBAAuB,CAAC,WAAmB;QACjD,OAAO;YACH,IAAI,EAAE,WAAW,CAAC,GAAG;YACrB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,iFAAiF;YACjF,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;YACvD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;SACd,CAAC;IACzB,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;;;OAIG;IACO,+BAA+B,CAAC,UAAkB;QACxD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AAlUD,0BAkUC"}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { Log } from '@apify/log';
|
|
2
|
+
import type { Log } from '@apify/log';
|
|
3
3
|
import { EventEmitter } from 'node:events';
|
|
4
4
|
import { Configuration } from '../configuration';
|
|
5
5
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
6
|
-
import {
|
|
6
|
+
import type { SessionOptions } from './session';
|
|
7
|
+
import { Session } from './session';
|
|
7
8
|
/**
|
|
8
9
|
* Factory user-function which creates customized {@link Session} instances.
|
|
9
10
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAgBK,QAAQ,CAAC,MAAM;IAf7D,GAAG,EAAE,GAAG,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,aAAa,CAAC;IACrC,aAAa,EAAG,aAAa,CAAC;IAC9B,QAAQ,EAAE,OAAO,EAAE,CAAM;IACzB,UAAU,uBAA8B;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB,EAAW,MAAM,gBAAkC;IA6C/F;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8BrD;;;OAGG;IACH,QAAQ;;;;;IAQR;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO;IAU7H;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}
|
|
@@ -188,11 +188,10 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
188
188
|
*/
|
|
189
189
|
async initialize() {
|
|
190
190
|
this.keyValueStore = await key_value_store_1.KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
// }
|
|
191
|
+
if (!this.persistStateKeyValueStoreId) {
|
|
192
|
+
// eslint-disable-next-line max-len
|
|
193
|
+
this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
|
|
194
|
+
}
|
|
196
195
|
// in case of migration happened and SessionPool state should be restored from the keyValueStore.
|
|
197
196
|
await this._maybeLoadSessionPool();
|
|
198
197
|
this._listener = this.persistState.bind(this);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,6CAA2C;
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,6CAA2C;AAE3C,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,iEAA4D;AAE5D,uCAAoC;AA6CpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,MAAa,WAAY,SAAQ,0BAAY;IAazC;;OAEG;IACH,YAAY,UAA8B,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;QAC3F,KAAK,EAAE,CAAC;;;;;mBAD2C;;QAfvD;;;;;WAAS;QACT;;;;;WAAoB;QACpB;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;mBAAsB,EAAE;WAAC;QACzB;;;;mBAAa,IAAI,GAAG,EAAmB;WAAC;QACxC;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAwB;QACxB;;;;;WAAwC;QACxC;;;;;WAA6B;QAQzB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,GAAG,IAAI,EAElB,2BAA2B,EAC3B,eAAe,GAAG,wBAAwB,EAE1C,qBAAqB,EACrB,cAAc,GAAG,EAAE,EAEnB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,aAAa,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzG,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACnC,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;SAC5K;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE;YACJ,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE;gBACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;aAC1F;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACjC;QAED,MAAM,UAAU,GAAG,OAAO,YAAY,iBAAO;YACzC,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,SAAS,EAAE;YACX,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;gBAAE,OAAO,OAAO,CAAC;YAClD,OAAO,SAAS,CAAC;SACpB;QAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC5B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;SAChC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE;YAC1B,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,6BAA6B,CAAC,WAAwB,EAAE,UAA+C,EAAE;QAC/G,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACxC,OAAO,IAAI,iBAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE;YACpD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,IAAI,iBAAO,CAAC,aAAa,CAAC,CAAC;YAEpD,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE;gBAC7B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;aACtC;SACJ;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B;QAC1C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AA9TD,kCA8TC"}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
2
|
+
import type { Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { IncomingMessage } from 'node:http';
|
|
3
4
|
// @ts-ignore optional peer dependency
|
|
4
|
-
import { HTTPResponse } from 'puppeteer';
|
|
5
|
+
import type { HTTPResponse } from 'puppeteer';
|
|
5
6
|
import { Cookie } from 'tough-cookie';
|
|
6
|
-
|
|
7
|
+
/**
|
|
8
|
+
* @internal
|
|
9
|
+
*/
|
|
7
10
|
export declare function getCookiesFromResponse(response: IncomingMessage | HTTPResponse | {
|
|
8
11
|
headers: Dictionary<string | string[]>;
|
|
9
12
|
}): Cookie[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAWtI"}
|
|
@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.getCookiesFromResponse = void 0;
|
|
4
4
|
const tough_cookie_1 = require("tough-cookie");
|
|
5
5
|
const errors_1 = require("./errors");
|
|
6
|
+
/**
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
6
9
|
function getCookiesFromResponse(response) {
|
|
7
10
|
const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
|
|
8
11
|
const cookieHeader = headers['set-cookie'] || '';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;AAGA,+CAAsC;AACtC,qCAA4C;AAE5C;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAAqF;IACxH,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI;QACA,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;KACvC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;KAC5C;AACL,CAAC;AAXD,wDAWC"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type { DatasetClient, DatasetInfo, PaginatedList, StorageClient } from '@crawlee/types';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import type { DatasetClient, DatasetInfo, PaginatedList, StorageClient, Dictionary } from '@crawlee/types';
|
|
2
|
+
import { Configuration } from '../configuration';
|
|
3
|
+
import type { Awaitable } from '../typedefs';
|
|
4
|
+
import type { StorageManagerOptions } from './storage_manager';
|
|
4
5
|
/** @internal */
|
|
5
6
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
6
7
|
/**
|
|
@@ -113,7 +114,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
|
|
|
113
114
|
*
|
|
114
115
|
* ```javascript
|
|
115
116
|
* // Write a single row to the default dataset
|
|
116
|
-
* await
|
|
117
|
+
* await Dataset.pushData({ col1: 123, col2: 'val2' });
|
|
117
118
|
*
|
|
118
119
|
* // Open a named dataset
|
|
119
120
|
* const dataset = await Dataset.open('some-name');
|
|
@@ -130,6 +131,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
|
|
|
130
131
|
* @category Result Stores
|
|
131
132
|
*/
|
|
132
133
|
export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
134
|
+
readonly config: Configuration;
|
|
133
135
|
id: string;
|
|
134
136
|
name?: string;
|
|
135
137
|
client: DatasetClient<Data>;
|
|
@@ -137,14 +139,14 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
137
139
|
/**
|
|
138
140
|
* @internal
|
|
139
141
|
*/
|
|
140
|
-
constructor(options: DatasetOptions);
|
|
142
|
+
constructor(options: DatasetOptions, config?: Configuration);
|
|
141
143
|
/**
|
|
142
144
|
* Stores an object or an array of objects to the dataset.
|
|
143
145
|
* The function returns a promise that resolves when the operation finishes.
|
|
144
146
|
* It has no result, but throws on invalid args or other errors.
|
|
145
147
|
*
|
|
146
148
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
147
|
-
* otherwise the
|
|
149
|
+
* otherwise the crawler process might finish before the data is stored!
|
|
148
150
|
*
|
|
149
151
|
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
150
152
|
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
@@ -165,10 +167,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
165
167
|
pushData(data: Data | Data[]): Promise<void>;
|
|
166
168
|
/**
|
|
167
169
|
* Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
|
|
168
|
-
*
|
|
169
|
-
* If you need to get data in an unparsed format, use the {@link Actor.newClient} function to get a new
|
|
170
|
-
* `apify-client` instance and call
|
|
171
|
-
* [`datasetClient.downloadItems()`](https://github.com/apify/apify-client-js#DatasetClient+downloadItems)
|
|
172
170
|
*/
|
|
173
171
|
getData(options?: DatasetDataOptions): Promise<PaginatedList<Data>>;
|
|
174
172
|
/**
|
|
@@ -190,7 +188,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
190
188
|
* modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
|
|
191
189
|
* accessedAt: new Date("2015-12-14T08:36:13.202Z"),
|
|
192
190
|
* itemCount: 14,
|
|
193
|
-
* cleanItemCount: 10
|
|
194
191
|
* }
|
|
195
192
|
* ```
|
|
196
193
|
*/
|
|
@@ -258,10 +255,35 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
258
255
|
*
|
|
259
256
|
* @param [datasetIdOrName]
|
|
260
257
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
261
|
-
* the function returns the default dataset associated with the
|
|
258
|
+
* the function returns the default dataset associated with the crawler run.
|
|
262
259
|
* @param [options] Storage manager options.
|
|
263
260
|
*/
|
|
264
261
|
static open<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
|
|
262
|
+
/**
|
|
263
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
|
264
|
+
*
|
|
265
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
266
|
+
* For example, calling the following code:
|
|
267
|
+
* ```javascript
|
|
268
|
+
* await Dataset.pushData({ myValue: 123 });
|
|
269
|
+
* ```
|
|
270
|
+
*
|
|
271
|
+
* is equivalent to:
|
|
272
|
+
* ```javascript
|
|
273
|
+
* const dataset = await Dataset.open();
|
|
274
|
+
* await dataset.pushData({ myValue: 123 });
|
|
275
|
+
* ```
|
|
276
|
+
*
|
|
277
|
+
* For more information, see {@link Dataset.open} and {@link Dataset.pushData}
|
|
278
|
+
*
|
|
279
|
+
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
280
|
+
* otherwise the crawler process might finish before the data are stored!
|
|
281
|
+
*
|
|
282
|
+
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
283
|
+
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
284
|
+
* @ignore
|
|
285
|
+
*/
|
|
286
|
+
static pushData<Data extends Dictionary = Dictionary>(item: Data | Data[]): Promise<void>;
|
|
265
287
|
}
|
|
266
288
|
/**
|
|
267
289
|
* User-function used in the `Dataset.forEach()` API.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC3G,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAI/D,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC/H,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAShB,QAAQ,CAAC,MAAM;IARpD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,+BAAoC;IAEvC;;OAEG;gBACS,OAAO,EAAE,cAAc,EAAW,MAAM,gBAAkC;IAMtF;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAY7E;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAIjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAWlG;;;;;;;;;;;;;;OAcG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,CAAC;IAoB7G;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAYrJ;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAIlG;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IAEjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAEhD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAElC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAE7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IAEnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAEtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB"}
|
package/storages/dataset.js
CHANGED
|
@@ -7,6 +7,7 @@ const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
|
7
7
|
const configuration_1 = require("../configuration");
|
|
8
8
|
const log_1 = require("../log");
|
|
9
9
|
const storage_manager_1 = require("./storage_manager");
|
|
10
|
+
const utils_1 = require("./utils");
|
|
10
11
|
/** @internal */
|
|
11
12
|
exports.DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
12
13
|
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
@@ -110,7 +111,7 @@ exports.chunkBySize = chunkBySize;
|
|
|
110
111
|
*
|
|
111
112
|
* ```javascript
|
|
112
113
|
* // Write a single row to the default dataset
|
|
113
|
-
* await
|
|
114
|
+
* await Dataset.pushData({ col1: 123, col2: 'val2' });
|
|
114
115
|
*
|
|
115
116
|
* // Open a named dataset
|
|
116
117
|
* const dataset = await Dataset.open('some-name');
|
|
@@ -130,7 +131,13 @@ class Dataset {
|
|
|
130
131
|
/**
|
|
131
132
|
* @internal
|
|
132
133
|
*/
|
|
133
|
-
constructor(options) {
|
|
134
|
+
constructor(options, config = configuration_1.Configuration.getGlobalConfig()) {
|
|
135
|
+
Object.defineProperty(this, "config", {
|
|
136
|
+
enumerable: true,
|
|
137
|
+
configurable: true,
|
|
138
|
+
writable: true,
|
|
139
|
+
value: config
|
|
140
|
+
});
|
|
134
141
|
Object.defineProperty(this, "id", {
|
|
135
142
|
enumerable: true,
|
|
136
143
|
configurable: true,
|
|
@@ -165,7 +172,7 @@ class Dataset {
|
|
|
165
172
|
* It has no result, but throws on invalid args or other errors.
|
|
166
173
|
*
|
|
167
174
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
168
|
-
* otherwise the
|
|
175
|
+
* otherwise the crawler process might finish before the data is stored!
|
|
169
176
|
*
|
|
170
177
|
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
171
178
|
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
@@ -202,10 +209,6 @@ class Dataset {
|
|
|
202
209
|
}
|
|
203
210
|
/**
|
|
204
211
|
* Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
|
|
205
|
-
*
|
|
206
|
-
* If you need to get data in an unparsed format, use the {@link Actor.newClient} function to get a new
|
|
207
|
-
* `apify-client` instance and call
|
|
208
|
-
* [`datasetClient.downloadItems()`](https://github.com/apify/apify-client-js#DatasetClient+downloadItems)
|
|
209
212
|
*/
|
|
210
213
|
async getData(options = {}) {
|
|
211
214
|
try {
|
|
@@ -238,7 +241,6 @@ class Dataset {
|
|
|
238
241
|
* modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
|
|
239
242
|
* accessedAt: new Date("2015-12-14T08:36:13.202Z"),
|
|
240
243
|
* itemCount: 14,
|
|
241
|
-
* cleanItemCount: 10
|
|
242
244
|
* }
|
|
243
245
|
* ```
|
|
244
246
|
*/
|
|
@@ -337,7 +339,7 @@ class Dataset {
|
|
|
337
339
|
*/
|
|
338
340
|
async drop() {
|
|
339
341
|
await this.client.delete();
|
|
340
|
-
const manager =
|
|
342
|
+
const manager = storage_manager_1.StorageManager.getManager(Dataset, this.config);
|
|
341
343
|
manager.closeStorage(this);
|
|
342
344
|
}
|
|
343
345
|
/**
|
|
@@ -351,7 +353,7 @@ class Dataset {
|
|
|
351
353
|
*
|
|
352
354
|
* @param [datasetIdOrName]
|
|
353
355
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
354
|
-
* the function returns the default dataset associated with the
|
|
356
|
+
* the function returns the default dataset associated with the crawler run.
|
|
355
357
|
* @param [options] Storage manager options.
|
|
356
358
|
*/
|
|
357
359
|
static async open(datasetIdOrName, options = {}) {
|
|
@@ -359,8 +361,38 @@ class Dataset {
|
|
|
359
361
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
360
362
|
config: ow_1.default.optional.object.instanceOf(configuration_1.Configuration),
|
|
361
363
|
}));
|
|
362
|
-
|
|
363
|
-
|
|
364
|
+
options.config ?? (options.config = configuration_1.Configuration.getGlobalConfig());
|
|
365
|
+
await (0, utils_1.purgeDefaultStorages)();
|
|
366
|
+
const manager = storage_manager_1.StorageManager.getManager(Dataset, options.config);
|
|
367
|
+
return manager.openStorage(datasetIdOrName, options.config.getStorageClient());
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
|
371
|
+
*
|
|
372
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
373
|
+
* For example, calling the following code:
|
|
374
|
+
* ```javascript
|
|
375
|
+
* await Dataset.pushData({ myValue: 123 });
|
|
376
|
+
* ```
|
|
377
|
+
*
|
|
378
|
+
* is equivalent to:
|
|
379
|
+
* ```javascript
|
|
380
|
+
* const dataset = await Dataset.open();
|
|
381
|
+
* await dataset.pushData({ myValue: 123 });
|
|
382
|
+
* ```
|
|
383
|
+
*
|
|
384
|
+
* For more information, see {@link Dataset.open} and {@link Dataset.pushData}
|
|
385
|
+
*
|
|
386
|
+
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
387
|
+
* otherwise the crawler process might finish before the data are stored!
|
|
388
|
+
*
|
|
389
|
+
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
390
|
+
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
391
|
+
* @ignore
|
|
392
|
+
*/
|
|
393
|
+
static async pushData(item) {
|
|
394
|
+
const dataset = await this.open();
|
|
395
|
+
return dataset.pushData(item);
|
|
364
396
|
}
|
|
365
397
|
}
|
|
366
398
|
exports.Dataset = Dataset;
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;;AAAA,0CAAuD;AACvD,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA6B;
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;;AAAA,0CAAuD;AACvD,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA6B;AAG7B,uDAAmD;AACnD,mCAA+C;AAE/C,gBAAgB;AACH,QAAA,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;KAC/F;IAED,IAAI,OAAO,CAAC;IACZ,IAAI;QACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAClC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACvF;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;KACnG;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAtBD,8CAsBC;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE;YACjD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;SAC1B;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE;YAC7C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnB;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;SAChE;aAAM;YACH,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;SAC7D;KACJ;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA9BD,kCA8BC;AA6ED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,OAAO;IAMhB;;OAEG;IACH,YAAY,OAAuB,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAR9C;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;;WAA4B;QAC5B;;;;mBAAM,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;WAAC;QAMnC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,IAAA,YAAE,EAAC,IAAI,EAAE,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,+BAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,+BAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC;SAC5B;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;SACzB;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;SAC/C;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE;gBAC9D,MAAM,IAAI,KAAK,CAAC,4GAA4G,CAAC,CAAC;aACjI;YACD,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/H,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,uCAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACjC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,MAAM,CAAI,QAAiC,EAAE,IAAO,EAAE,UAAkC,EAAE;QAC5F,IAAI,WAAW,GAAM,IAAI,CAAC;QAE1B,MAAM,WAAW,GAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACvD,OAAO,OAAO;iBACT,OAAO,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,OAAO,CAAC,KAAK,IAAI,WAAW,KAAK,SAAS;oBACtC,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7C,CAAC,CAAC;iBACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBACd,WAAW,GAAG,OAAY,CAAC;YAC/B,CAAC,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAuC,eAA+B,EAAE,UAAiC,EAAE;QACxH,IAAA,YAAE,EAAC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,6BAAa,CAAC,eAAe,EAAE,EAAC;QACnD,MAAM,IAAA,4BAAoB,GAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAgB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAElF,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;CACJ;AAlQD,0BAkQC"}
|