@crawlee/core 3.0.0-beta.56 → 3.0.0-beta.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/enqueue_links/enqueue_links.d.ts +6 -6
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +11 -11
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/package.json +5 -13
- package/session_pool/session.d.ts +7 -11
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +16 -16
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_utils.d.ts +2 -4
- package/session_pool/session_utils.d.ts.map +1 -1
- package/session_pool/session_utils.js.map +1 -1
- package/storages/request_list.d.ts +24 -28
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +26 -20
- package/storages/request_list.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -90,9 +90,9 @@ export interface EnqueueLinksOptions {
|
|
|
90
90
|
transformRequestFunction?: RequestTransform;
|
|
91
91
|
/**
|
|
92
92
|
* The strategy to use when enqueueing the urls.
|
|
93
|
-
* @default EnqueueStrategy.
|
|
93
|
+
* @default EnqueueStrategy.SameHostname
|
|
94
94
|
*/
|
|
95
|
-
strategy?: EnqueueStrategy | 'all' | 'same-
|
|
95
|
+
strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname';
|
|
96
96
|
}
|
|
97
97
|
export declare enum EnqueueStrategy {
|
|
98
98
|
/**
|
|
@@ -100,17 +100,17 @@ export declare enum EnqueueStrategy {
|
|
|
100
100
|
*/
|
|
101
101
|
All = "all",
|
|
102
102
|
/**
|
|
103
|
-
* Matches any URLs that have the same
|
|
103
|
+
* Matches any URLs that have the same hostname.
|
|
104
104
|
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://example.com/`, but
|
|
105
105
|
* `https://example.com/hello` will not be matched.
|
|
106
106
|
*/
|
|
107
|
-
|
|
107
|
+
SameHostname = "same-hostname",
|
|
108
108
|
/**
|
|
109
|
-
* Matches any URLs that have the same
|
|
109
|
+
* Matches any URLs that have the same (sub-)domain as the base URL.
|
|
110
110
|
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
111
111
|
* `https://example.com`.
|
|
112
112
|
*/
|
|
113
|
-
|
|
113
|
+
SameDomain = "same-domain"
|
|
114
114
|
}
|
|
115
115
|
/**
|
|
116
116
|
* This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA4BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -13,17 +13,17 @@ var EnqueueStrategy;
|
|
|
13
13
|
*/
|
|
14
14
|
EnqueueStrategy["All"] = "all";
|
|
15
15
|
/**
|
|
16
|
-
* Matches any URLs that have the same
|
|
16
|
+
* Matches any URLs that have the same hostname.
|
|
17
17
|
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://example.com/`, but
|
|
18
18
|
* `https://example.com/hello` will not be matched.
|
|
19
19
|
*/
|
|
20
|
-
EnqueueStrategy["
|
|
20
|
+
EnqueueStrategy["SameHostname"] = "same-hostname";
|
|
21
21
|
/**
|
|
22
|
-
* Matches any URLs that have the same
|
|
22
|
+
* Matches any URLs that have the same (sub-)domain as the base URL.
|
|
23
23
|
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
24
24
|
* `https://example.com`.
|
|
25
25
|
*/
|
|
26
|
-
EnqueueStrategy["
|
|
26
|
+
EnqueueStrategy["SameDomain"] = "same-domain";
|
|
27
27
|
})(EnqueueStrategy = exports.EnqueueStrategy || (exports.EnqueueStrategy = {}));
|
|
28
28
|
/**
|
|
29
29
|
* This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
@@ -64,8 +64,8 @@ async function enqueueLinks(options) {
|
|
|
64
64
|
transformRequestFunction: ow_1.default.optional.function,
|
|
65
65
|
strategy: ow_1.default.optional.string.oneOf([
|
|
66
66
|
EnqueueStrategy.All,
|
|
67
|
-
EnqueueStrategy.SameSubdomain,
|
|
68
67
|
EnqueueStrategy.SameHostname,
|
|
68
|
+
EnqueueStrategy.SameDomain,
|
|
69
69
|
]),
|
|
70
70
|
}));
|
|
71
71
|
const { requestQueue, limit, urls, pseudoUrls, globs, regexps, transformRequestFunction, } = options;
|
|
@@ -81,18 +81,18 @@ async function enqueueLinks(options) {
|
|
|
81
81
|
urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
|
|
82
82
|
}
|
|
83
83
|
if (!urlPatternObjects.length) {
|
|
84
|
-
options.strategy ?? (options.strategy = EnqueueStrategy.
|
|
84
|
+
options.strategy ?? (options.strategy = EnqueueStrategy.SameHostname);
|
|
85
85
|
}
|
|
86
86
|
if (options.baseUrl) {
|
|
87
|
+
const url = new URL(options.baseUrl);
|
|
87
88
|
switch (options.strategy) {
|
|
88
|
-
case EnqueueStrategy.
|
|
89
|
+
case EnqueueStrategy.SameHostname:
|
|
89
90
|
// We need to get the origin of the passed in domain in the event someone sets baseUrl
|
|
90
91
|
// to an url like https://example.com/deep/default/path and one of the found urls is an
|
|
91
92
|
// absolute relative path (/path/to/page)
|
|
92
|
-
urlPatternObjects.push({ glob: `${
|
|
93
|
+
urlPatternObjects.push({ glob: `${url.origin}/**` });
|
|
93
94
|
break;
|
|
94
|
-
case EnqueueStrategy.
|
|
95
|
-
const url = new URL(options.baseUrl);
|
|
95
|
+
case EnqueueStrategy.SameDomain: {
|
|
96
96
|
// Get the actual hostname from the base url
|
|
97
97
|
const baseUrlHostname = (0, tldts_1.getDomain)(url.hostname, { mixedInputs: false });
|
|
98
98
|
if (baseUrlHostname) {
|
|
@@ -137,7 +137,7 @@ function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl,
|
|
|
137
137
|
return finalUrlOrigin ?? originalUrlOrigin;
|
|
138
138
|
}
|
|
139
139
|
// Ensure links use the same hostname
|
|
140
|
-
if (enqueueStrategy === EnqueueStrategy.
|
|
140
|
+
if (enqueueStrategy === EnqueueStrategy.SameDomain) {
|
|
141
141
|
const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
|
|
142
142
|
const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
|
|
143
143
|
if (originalHostname === finalHostname) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,IAAI,iBAAiB,CAAC;KAC9C;IAED,qCAAqC;IACrC,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,oDAAoD;IACpD,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAjCD,wCAiCC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.59",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -58,10 +58,10 @@
|
|
|
58
58
|
"@apify/pseudo_url": "^2.0.0",
|
|
59
59
|
"@apify/timeout": "^0.3.0",
|
|
60
60
|
"@apify/utilities": "^2.0.0",
|
|
61
|
-
"@crawlee/browser-pool": "^3.0.0-beta.
|
|
62
|
-
"@crawlee/memory-storage": "^3.0.0-beta.
|
|
63
|
-
"@crawlee/types": "^3.0.0-beta.
|
|
64
|
-
"@crawlee/utils": "^3.0.0-beta.
|
|
61
|
+
"@crawlee/browser-pool": "^3.0.0-beta.59",
|
|
62
|
+
"@crawlee/memory-storage": "^3.0.0-beta.59",
|
|
63
|
+
"@crawlee/types": "^3.0.0-beta.59",
|
|
64
|
+
"@crawlee/utils": "^3.0.0-beta.59",
|
|
65
65
|
"@types/tough-cookie": "^4.0.2",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
67
67
|
"fs-extra": "^10.1.0",
|
|
@@ -71,13 +71,5 @@
|
|
|
71
71
|
"stream-json": "^1.7.4",
|
|
72
72
|
"tldts": "^5.7.80",
|
|
73
73
|
"tough-cookie": "^4.0.0"
|
|
74
|
-
},
|
|
75
|
-
"peerDependencies": {
|
|
76
|
-
"puppeteer": ">= 9.x <= 14.x"
|
|
77
|
-
},
|
|
78
|
-
"peerDependenciesMeta": {
|
|
79
|
-
"puppeteer": {
|
|
80
|
-
"optional": true
|
|
81
|
-
}
|
|
82
74
|
}
|
|
83
75
|
}
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import type { Log } from '@apify/log';
|
|
3
|
-
import type { Cookie as
|
|
4
|
-
import type { Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@crawlee/types';
|
|
5
4
|
import type { IncomingMessage } from 'node:http';
|
|
6
|
-
// @ts-ignore optional peer dependency
|
|
7
|
-
import type { HTTPResponse, Protocol } from 'puppeteer';
|
|
8
5
|
import { Cookie, CookieJar } from 'tough-cookie';
|
|
9
6
|
import { SessionPool } from './session_pool';
|
|
10
|
-
export declare type PuppeteerCookie = Protocol.Network.Cookie | BrowserPoolCookie;
|
|
11
7
|
/**
|
|
12
8
|
* Persistable {@link Session} state.
|
|
13
9
|
*/
|
|
@@ -154,7 +150,7 @@ export declare class Session {
|
|
|
154
150
|
*
|
|
155
151
|
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
156
152
|
*/
|
|
157
|
-
setCookiesFromResponse(response: IncomingMessage |
|
|
153
|
+
setCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | {
|
|
158
154
|
headers: Dictionary<string | string[]>;
|
|
159
155
|
url: string;
|
|
160
156
|
}): void;
|
|
@@ -171,12 +167,12 @@ export declare class Session {
|
|
|
171
167
|
* ]
|
|
172
168
|
* ```
|
|
173
169
|
*/
|
|
174
|
-
|
|
170
|
+
setCookies(cookies: CookieObject[], url: string): void;
|
|
175
171
|
/**
|
|
176
172
|
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
177
173
|
* @param url website url. Only cookies stored for this url will be returned
|
|
178
174
|
*/
|
|
179
|
-
|
|
175
|
+
getCookies(url: string): CookieObject[];
|
|
180
176
|
/**
|
|
181
177
|
* Returns cookies saved with the session in the typical
|
|
182
178
|
* key1=value1; key2=value2 format, ready to be used in
|
|
@@ -190,15 +186,15 @@ export declare class Session {
|
|
|
190
186
|
setCookie(rawCookie: string, url: string): void;
|
|
191
187
|
/**
|
|
192
188
|
* Transforms puppeteer cookie to tough-cookie.
|
|
193
|
-
* @param
|
|
189
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
194
190
|
*/
|
|
195
|
-
protected
|
|
191
|
+
protected _cookieObjectToTough(cookieObject: CookieObject): Cookie;
|
|
196
192
|
/**
|
|
197
193
|
* Transforms tough-cookie to puppeteer cookie.
|
|
198
194
|
* @param toughCookie Cookie from CookieJar
|
|
199
195
|
* @return Cookie from Puppeteer
|
|
200
196
|
*/
|
|
201
|
-
protected _toughCookieToPuppeteer(toughCookie: Cookie):
|
|
197
|
+
protected _toughCookieToPuppeteer(toughCookie: Cookie): CookieObject;
|
|
202
198
|
/**
|
|
203
199
|
* Sets cookies.
|
|
204
200
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAsDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWhI;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;;OAGG;IACH,SAAS,CAAC,oBAAoB,CAAC,YAAY,EAAE,YAAY,GAAG,MAAM;IAkBlE;;;;OAIG;IACH,SAAS,CAAC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY;IAapE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;;;OAIG;IACH,SAAS,CAAC,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -132,7 +132,7 @@ class Session {
|
|
|
132
132
|
this.errorScore = errorScore; // indicates number of markBaded request with the session
|
|
133
133
|
this.maxUsageCount = maxUsageCount;
|
|
134
134
|
this.sessionPool = sessionPool;
|
|
135
|
-
this.
|
|
135
|
+
this._cookieObjectToTough = this._cookieObjectToTough.bind(this);
|
|
136
136
|
}
|
|
137
137
|
/**
|
|
138
138
|
* Indicates whether the session is blocked.
|
|
@@ -262,15 +262,15 @@ class Session {
|
|
|
262
262
|
* ]
|
|
263
263
|
* ```
|
|
264
264
|
*/
|
|
265
|
-
|
|
266
|
-
const normalizedCookies = cookies.map(this.
|
|
265
|
+
setCookies(cookies, url) {
|
|
266
|
+
const normalizedCookies = cookies.map(this._cookieObjectToTough);
|
|
267
267
|
this._setCookies(normalizedCookies, url);
|
|
268
268
|
}
|
|
269
269
|
/**
|
|
270
270
|
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
271
271
|
* @param url website url. Only cookies stored for this url will be returned
|
|
272
272
|
*/
|
|
273
|
-
|
|
273
|
+
getCookies(url) {
|
|
274
274
|
const cookies = this.cookieJar.getCookiesSync(url);
|
|
275
275
|
return cookies.map(this._toughCookieToPuppeteer);
|
|
276
276
|
}
|
|
@@ -291,22 +291,22 @@ class Session {
|
|
|
291
291
|
}
|
|
292
292
|
/**
|
|
293
293
|
* Transforms puppeteer cookie to tough-cookie.
|
|
294
|
-
* @param
|
|
294
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
295
295
|
*/
|
|
296
|
-
|
|
297
|
-
const isExpiresValid =
|
|
298
|
-
const expires = isExpiresValid ? new Date(
|
|
299
|
-
const domain = typeof
|
|
300
|
-
?
|
|
301
|
-
:
|
|
296
|
+
_cookieObjectToTough(cookieObject) {
|
|
297
|
+
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
|
|
298
|
+
const expires = isExpiresValid ? new Date(cookieObject.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
|
|
299
|
+
const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
|
|
300
|
+
? cookieObject.domain.slice(1)
|
|
301
|
+
: cookieObject.domain;
|
|
302
302
|
return new tough_cookie_1.Cookie({
|
|
303
|
-
key:
|
|
304
|
-
value:
|
|
303
|
+
key: cookieObject.name,
|
|
304
|
+
value: cookieObject.value,
|
|
305
305
|
expires,
|
|
306
306
|
domain,
|
|
307
|
-
path:
|
|
308
|
-
secure:
|
|
309
|
-
httpOnly:
|
|
307
|
+
path: cookieObject.path,
|
|
308
|
+
secure: cookieObject.secure,
|
|
309
|
+
httpOnly: cookieObject.httpOnly,
|
|
310
310
|
});
|
|
311
311
|
}
|
|
312
312
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,oDAAoB;AACpB,+CAAiD;AACjD,4CAAoD;AACpD,gCAA2C;AAC3C,qCAAiD;AACjD,iDAA6C;AAC7C,mDAAyD;AAEzD,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAAiC;QACjC;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAW,CAAC;YAC9C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAEjF,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAyG;QAC5H,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,sCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACjE,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACO,oBAAoB,CAAC,YAA0B;QACrD,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;QACpH,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChI,MAAM,MAAM,GAAG,OAAO,YAAY,CAAC,MAAM,KAAK,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YACzF,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;QAE1B,OAAO,IAAI,qBAAM,CAAC;YACd,GAAG,EAAE,YAAY,CAAC,IAAI;YACtB,KAAK,EAAE,YAAY,CAAC,KAAK;YACzB,OAAO;YACP,MAAM;YACN,IAAI,EAAE,YAAY,CAAC,IAAI;YACvB,MAAM,EAAE,YAAY,CAAC,MAAM;YAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;SAClC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACO,uBAAuB,CAAC,WAAmB;QACjD,OAAO;YACH,IAAI,EAAE,WAAW,CAAC,GAAG;YACrB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,iFAAiF;YACjF,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;YACvD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;SACjB,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;;;OAIG;IACO,+BAA+B,CAAC,UAAkB;QACxD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AAlUD,0BAkUC"}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import type { Dictionary } from '@crawlee/types';
|
|
2
|
+
import type { BrowserLikeResponse, Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { IncomingMessage } from 'node:http';
|
|
4
|
-
// @ts-ignore optional peer dependency
|
|
5
|
-
import type { HTTPResponse } from 'puppeteer';
|
|
6
4
|
import { Cookie } from 'tough-cookie';
|
|
7
5
|
/**
|
|
8
6
|
* @internal
|
|
9
7
|
*/
|
|
10
|
-
export declare function getCookiesFromResponse(response: IncomingMessage |
|
|
8
|
+
export declare function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | {
|
|
11
9
|
headers: Dictionary<string | string[]>;
|
|
12
10
|
}): Cookie[];
|
|
13
11
|
//# sourceMappingURL=session_utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAW7I"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;AAEA,+CAAsC;AACtC,qCAA4C;AAE5C;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAA4F;IAC/H,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI;QACA,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;KACvC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;KAC5C;AACL,CAAC;AAXD,wDAWC"}
|
|
@@ -64,22 +64,20 @@ export interface RequestListOptions {
|
|
|
64
64
|
* // and big websites typically have multiple sitemaps.
|
|
65
65
|
* const sitemaps = await downloadHugeSitemaps();
|
|
66
66
|
* return parseUrlsFromSitemaps(sitemaps);
|
|
67
|
-
* }
|
|
67
|
+
* };
|
|
68
68
|
*
|
|
69
69
|
* // Sitemaps can change in real-time, so it's important to persist
|
|
70
70
|
* // the URLs we collected. Otherwise we might lose our scraping
|
|
71
71
|
* // state in case of an crawler migration / failure / time-out.
|
|
72
|
-
* const requestList =
|
|
72
|
+
* const requestList = await RequestList.open(null, [], {
|
|
73
|
+
* // The sourcesFunction is called now and the Requests are persisted.
|
|
74
|
+
* // If something goes wrong and we need to start again, RequestList
|
|
75
|
+
* // will load the persisted Requests from storage and will NOT
|
|
76
|
+
* // call the sourcesFunction again, saving time and resources.
|
|
73
77
|
* sourcesFunction,
|
|
74
78
|
* persistStateKey: 'state-key',
|
|
75
79
|
* persistRequestsKey: 'requests-key',
|
|
76
80
|
* })
|
|
77
|
-
*
|
|
78
|
-
* // The sourcesFunction is called now and the Requests are persisted.
|
|
79
|
-
* // If something goes wrong and we need to start again, RequestList
|
|
80
|
-
* // will load the persisted Requests from storage and will NOT
|
|
81
|
-
* // call the sourcesFunction again, saving time and resources.
|
|
82
|
-
* await requestList.initialize();
|
|
83
81
|
* ```
|
|
84
82
|
*/
|
|
85
83
|
sourcesFunction?: RequestListSourcesFunction;
|
|
@@ -197,23 +195,19 @@ export interface RequestListOptions {
|
|
|
197
195
|
* **Advanced usage:**
|
|
198
196
|
* ```javascript
|
|
199
197
|
* // Use the constructor to get more control over the initialization.
|
|
200
|
-
* const requestList =
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
* { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
|
|
205
|
-
*
|
|
206
|
-
* // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
|
|
207
|
-
* // Note that all URLs must start with http:// or https://
|
|
208
|
-
* { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
|
|
209
|
-
* ],
|
|
198
|
+
* const requestList = await RequestList.open(null, [
|
|
199
|
+
* // Separate requests
|
|
200
|
+
* { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
|
|
201
|
+
* { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
|
|
210
202
|
*
|
|
203
|
+
* // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
|
|
204
|
+
* // Note that all URLs must start with http:// or https://
|
|
205
|
+
* { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
|
|
206
|
+
* ], {
|
|
211
207
|
* // Persist the state to avoid re-crawling which can lead to data duplications.
|
|
212
208
|
* // Keep in mind that the sources have to be immutable or this will throw an error.
|
|
213
209
|
* persistStateKey: 'my-state',
|
|
214
210
|
* });
|
|
215
|
-
*
|
|
216
|
-
* await requestList.initialize();
|
|
217
211
|
* ```
|
|
218
212
|
* @category Sources
|
|
219
213
|
*/
|
|
@@ -261,14 +255,16 @@ export declare class RequestList {
|
|
|
261
255
|
private proxyConfiguration?;
|
|
262
256
|
private events;
|
|
263
257
|
/**
|
|
258
|
+
* To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
|
|
264
259
|
* @param options All `RequestList` configuration options
|
|
260
|
+
* @internal
|
|
265
261
|
*/
|
|
266
|
-
constructor(
|
|
262
|
+
private constructor();
|
|
267
263
|
/**
|
|
268
264
|
* Loads all remote sources of URLs and potentially starts periodic state persistence.
|
|
269
265
|
* This function must be called before you can start using the instance in a meaningful way.
|
|
270
266
|
*/
|
|
271
|
-
initialize
|
|
267
|
+
private initialize;
|
|
272
268
|
/**
|
|
273
269
|
* Adds previously persisted Requests, as retrieved from the key-value store.
|
|
274
270
|
* This needs to be done in a memory efficient way. We should update the input
|
|
@@ -397,10 +393,10 @@ export declare class RequestList {
|
|
|
397
393
|
* const requestList = await RequestList.open('my-name', sources);
|
|
398
394
|
* ```
|
|
399
395
|
*
|
|
400
|
-
* @param
|
|
401
|
-
* Name of the request list to be opened. Setting a name enables the `RequestList`'s
|
|
402
|
-
* in the key-value store. This is useful in case of a restart or migration. Since `RequestList`
|
|
403
|
-
* stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
|
|
396
|
+
* @param listNameOrOptions
|
|
397
|
+
* Name of the request list to be opened, or the options object. Setting a name enables the `RequestList`'s
|
|
398
|
+
* state to be persisted in the key-value store. This is useful in case of a restart or migration. Since `RequestList`
|
|
399
|
+
* is only stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
|
|
404
400
|
* state to survive those situations and continue where it left off.
|
|
405
401
|
*
|
|
406
402
|
* The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE`
|
|
@@ -408,7 +404,7 @@ export declare class RequestList {
|
|
|
408
404
|
*
|
|
409
405
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
410
406
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
411
|
-
* @param sources
|
|
407
|
+
* @param [sources]
|
|
412
408
|
* An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
|
|
413
409
|
* plain objects that define at least the `url` property, or an array of {@link Request} instances.
|
|
414
410
|
*
|
|
@@ -427,7 +423,7 @@ export declare class RequestList {
|
|
|
427
423
|
* the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
|
|
428
424
|
* options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
|
|
429
425
|
*/
|
|
430
|
-
static open(
|
|
426
|
+
static open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options?: RequestListOptions): Promise<RequestList>;
|
|
431
427
|
/**
|
|
432
428
|
* @internal wraps public utility for mocking purposes
|
|
433
429
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB
|
|
1
|
+
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;MAIE;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,MAAM,CAAe;IAE7B;;;;OAIG;IACH,OAAO;IA6CP;;;OAGG;YACW,UAAU;IA4BxB;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,GAAG,kBAAkB,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0BpJ;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,oBAAY,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AACjH,aAAK,cAAc,GAAG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAClE,oBAAY,0BAA0B,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}
|
package/storages/request_list.js
CHANGED
|
@@ -63,29 +63,27 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
|
|
|
63
63
|
* **Advanced usage:**
|
|
64
64
|
* ```javascript
|
|
65
65
|
* // Use the constructor to get more control over the initialization.
|
|
66
|
-
* const requestList =
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
* { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
|
|
71
|
-
*
|
|
72
|
-
* // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
|
|
73
|
-
* // Note that all URLs must start with http:// or https://
|
|
74
|
-
* { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
|
|
75
|
-
* ],
|
|
66
|
+
* const requestList = await RequestList.open(null, [
|
|
67
|
+
* // Separate requests
|
|
68
|
+
* { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } },
|
|
69
|
+
* { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
|
|
76
70
|
*
|
|
71
|
+
* // Bulk load of URLs from file `http://www.example.com/my-url-list.txt`
|
|
72
|
+
* // Note that all URLs must start with http:// or https://
|
|
73
|
+
* { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
|
|
74
|
+
* ], {
|
|
77
75
|
* // Persist the state to avoid re-crawling which can lead to data duplications.
|
|
78
76
|
* // Keep in mind that the sources have to be immutable or this will throw an error.
|
|
79
77
|
* persistStateKey: 'my-state',
|
|
80
78
|
* });
|
|
81
|
-
*
|
|
82
|
-
* await requestList.initialize();
|
|
83
79
|
* ```
|
|
84
80
|
* @category Sources
|
|
85
81
|
*/
|
|
86
82
|
class RequestList {
|
|
87
83
|
/**
|
|
84
|
+
* To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
|
|
88
85
|
* @param options All `RequestList` configuration options
|
|
86
|
+
* @internal
|
|
89
87
|
*/
|
|
90
88
|
constructor(options = {}) {
|
|
91
89
|
Object.defineProperty(this, "log", {
|
|
@@ -249,7 +247,7 @@ class RequestList {
|
|
|
249
247
|
// If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
|
|
250
248
|
this.keepDuplicateUrls = keepDuplicateUrls;
|
|
251
249
|
// Will be empty after initialization to save memory.
|
|
252
|
-
this.sources = sources
|
|
250
|
+
this.sources = sources ? [...sources] : [];
|
|
253
251
|
this.sourcesFunction = sourcesFunction;
|
|
254
252
|
// The proxy configuration used for `requestsFromUrls` requests.
|
|
255
253
|
this.proxyConfiguration = proxyConfiguration;
|
|
@@ -280,6 +278,7 @@ class RequestList {
|
|
|
280
278
|
if (this.persistStateKey) {
|
|
281
279
|
this.events.on("persistState" /* EventType.PERSIST_STATE */, this.persistState.bind(this));
|
|
282
280
|
}
|
|
281
|
+
return this;
|
|
283
282
|
}
|
|
284
283
|
/**
|
|
285
284
|
* Adds previously persisted Requests, as retrieved from the key-value store.
|
|
@@ -384,7 +383,7 @@ class RequestList {
|
|
|
384
383
|
throw new Error('The state object is invalid: nextIndex must be a non-negative number.');
|
|
385
384
|
}
|
|
386
385
|
if (state.nextIndex > this.requests.length) {
|
|
387
|
-
throw new Error('The state object is not consistent with RequestList too few requests loaded.');
|
|
386
|
+
throw new Error('The state object is not consistent with RequestList, too few requests loaded.');
|
|
388
387
|
}
|
|
389
388
|
if (state.nextIndex < this.requests.length
|
|
390
389
|
&& this.requests[state.nextIndex].uniqueKey !== state.nextUniqueKey) {
|
|
@@ -672,10 +671,10 @@ class RequestList {
|
|
|
672
671
|
* const requestList = await RequestList.open('my-name', sources);
|
|
673
672
|
* ```
|
|
674
673
|
*
|
|
675
|
-
* @param
|
|
676
|
-
* Name of the request list to be opened. Setting a name enables the `RequestList`'s
|
|
677
|
-
* in the key-value store. This is useful in case of a restart or migration. Since `RequestList`
|
|
678
|
-
* stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
|
|
674
|
+
* @param listNameOrOptions
|
|
675
|
+
* Name of the request list to be opened, or the options object. Setting a name enables the `RequestList`'s
|
|
676
|
+
* state to be persisted in the key-value store. This is useful in case of a restart or migration. Since `RequestList`
|
|
677
|
+
* is only stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
|
|
679
678
|
* state to survive those situations and continue where it left off.
|
|
680
679
|
*
|
|
681
680
|
* The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE`
|
|
@@ -683,7 +682,7 @@ class RequestList {
|
|
|
683
682
|
*
|
|
684
683
|
* If `null`, the list will not be persisted and will only be stored in memory. Process restart
|
|
685
684
|
* will then cause the list to be crawled again from the beginning. We suggest always using a name.
|
|
686
|
-
* @param sources
|
|
685
|
+
* @param [sources]
|
|
687
686
|
* An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
|
|
688
687
|
* plain objects that define at least the `url` property, or an array of {@link Request} instances.
|
|
689
688
|
*
|
|
@@ -702,7 +701,14 @@ class RequestList {
|
|
|
702
701
|
* the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
|
|
703
702
|
* options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
|
|
704
703
|
*/
|
|
705
|
-
static async open(
|
|
704
|
+
static async open(listNameOrOptions, sources, options = {}) {
|
|
705
|
+
if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
|
|
706
|
+
options = { ...listNameOrOptions, ...options };
|
|
707
|
+
const rl = new RequestList(options);
|
|
708
|
+
await rl.initialize();
|
|
709
|
+
return rl;
|
|
710
|
+
}
|
|
711
|
+
const listName = listNameOrOptions;
|
|
706
712
|
(0, ow_1.default)(listName, ow_1.default.optional.any(ow_1.default.string, ow_1.default.null));
|
|
707
713
|
(0, ow_1.default)(sources, ow_1.default.array);
|
|
708
714
|
(0, ow_1.default)(options, ow_1.default.object.is((v) => !Array.isArray(v)));
|