@crawlee/core 3.0.3-beta.7 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/autoscaling/autoscaled_pool.d.ts +16 -16
- package/autoscaling/autoscaled_pool.js +13 -13
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/system_status.d.ts +12 -12
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +33 -13
- package/configuration.d.ts.map +1 -1
- package/configuration.js +27 -7
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -13
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +6 -5
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +2 -2
- package/errors.d.ts +6 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +10 -1
- package/errors.js.map +1 -1
- package/index.mjs +1 -0
- package/package.json +4 -4
- package/proxy_configuration.d.ts +11 -11
- package/proxy_configuration.js +8 -8
- package/request.d.ts +3 -3
- package/request.js +2 -2
- package/session_pool/session.d.ts +1 -1
- package/session_pool/session_pool.d.ts +12 -12
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +10 -10
- package/storages/dataset.d.ts +14 -14
- package/storages/dataset.js +8 -8
- package/storages/key_value_store.d.ts +30 -49
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +51 -46
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +33 -33
- package/storages/request_list.js +23 -23
- package/storages/request_queue.d.ts +25 -25
- package/storages/request_queue.js +23 -23
- package/storages/utils.d.ts +1 -1
- package/storages/utils.js +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -10,9 +10,9 @@ export interface EnqueueLinksOptions {
|
|
|
10
10
|
requestQueue: RequestQueue;
|
|
11
11
|
/** A CSS selector matching links to be enqueued. */
|
|
12
12
|
selector?: string;
|
|
13
|
-
/** Sets {@
|
|
13
|
+
/** Sets {@apilink Request.userData} for newly enqueued requests. */
|
|
14
14
|
userData?: Dictionary;
|
|
15
|
-
/** Sets {@
|
|
15
|
+
/** Sets {@apilink Request.label} for newly enqueued requests. */
|
|
16
16
|
label?: string;
|
|
17
17
|
/**
|
|
18
18
|
* A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
|
|
@@ -24,7 +24,7 @@ export interface EnqueueLinksOptions {
|
|
|
24
24
|
* containing glob pattern strings matching the URLs to be enqueued.
|
|
25
25
|
*
|
|
26
26
|
* The plain objects must include at least the `glob` property, which holds the glob pattern string.
|
|
27
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@
|
|
27
|
+
* All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
|
|
28
28
|
*
|
|
29
29
|
* The matching is always case-insensitive.
|
|
30
30
|
* If you need case-sensitive matching, use `regexps` property directly.
|
|
@@ -38,7 +38,7 @@ export interface EnqueueLinksOptions {
|
|
|
38
38
|
* containing regular expressions matching the URLs to be enqueued.
|
|
39
39
|
*
|
|
40
40
|
* The plain objects must include at least the `regexp` property, which holds the regular expression.
|
|
41
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@
|
|
41
|
+
* All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
|
|
42
42
|
*
|
|
43
43
|
* If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
|
|
44
44
|
* enqueues the links with the same subdomain.
|
|
@@ -48,11 +48,11 @@ export interface EnqueueLinksOptions {
|
|
|
48
48
|
* *NOTE:* In future versions of SDK the options will be removed.
|
|
49
49
|
* Please use `globs` or `regexps` instead.
|
|
50
50
|
*
|
|
51
|
-
* An array of {@
|
|
52
|
-
* containing {@
|
|
51
|
+
* An array of {@apilink PseudoUrl} strings or plain objects
|
|
52
|
+
* containing {@apilink PseudoUrl} strings matching the URLs to be enqueued.
|
|
53
53
|
*
|
|
54
54
|
* The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
|
|
55
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@
|
|
55
|
+
* All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
|
|
56
56
|
*
|
|
57
57
|
* With a pseudo-URL string, the matching is always case-insensitive.
|
|
58
58
|
* If you need case-sensitive matching, use `regexps` property directly.
|
|
@@ -64,7 +64,7 @@ export interface EnqueueLinksOptions {
|
|
|
64
64
|
*/
|
|
65
65
|
pseudoUrls?: PseudoUrlInput[];
|
|
66
66
|
/**
|
|
67
|
-
* Just before a new {@
|
|
67
|
+
* Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
|
|
68
68
|
* to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
|
|
69
69
|
* when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
|
|
70
70
|
* or to dynamically update or create `userData`.
|
|
@@ -113,11 +113,11 @@ export declare enum EnqueueStrategy {
|
|
|
113
113
|
SameDomain = "same-domain"
|
|
114
114
|
}
|
|
115
115
|
/**
|
|
116
|
-
* This function enqueues the urls provided to the {@
|
|
116
|
+
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
117
117
|
* you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
|
|
118
118
|
*
|
|
119
119
|
* Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
|
|
120
|
-
* and override settings of the enqueued {@
|
|
120
|
+
* and override settings of the enqueued {@apilink Request} objects.
|
|
121
121
|
*
|
|
122
122
|
* **Example usage**
|
|
123
123
|
*
|
|
@@ -134,14 +134,15 @@ export declare enum EnqueueStrategy {
|
|
|
134
134
|
* ```
|
|
135
135
|
*
|
|
136
136
|
* @param options All `enqueueLinks()` parameters are passed via an options object.
|
|
137
|
-
* @returns Promise that resolves to {@
|
|
137
|
+
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
|
|
138
138
|
*/
|
|
139
139
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
140
140
|
/**
|
|
141
141
|
* @internal
|
|
142
|
-
* This method helps resolve the baseUrl that will be used for filtering in {@
|
|
142
|
+
* This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
|
|
143
143
|
* - If a user provides a base url, we always return it
|
|
144
|
-
* - If a user specifies {@
|
|
144
|
+
* - If a user specifies {@apilink EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
|
|
145
|
+
* request domain, or a redirected one
|
|
145
146
|
* - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
|
|
146
147
|
*/
|
|
147
148
|
export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -26,11 +26,11 @@ var EnqueueStrategy;
|
|
|
26
26
|
EnqueueStrategy["SameDomain"] = "same-domain";
|
|
27
27
|
})(EnqueueStrategy = exports.EnqueueStrategy || (exports.EnqueueStrategy = {}));
|
|
28
28
|
/**
|
|
29
|
-
* This function enqueues the urls provided to the {@
|
|
29
|
+
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
30
30
|
* you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
|
|
31
31
|
*
|
|
32
32
|
* Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
|
|
33
|
-
* and override settings of the enqueued {@
|
|
33
|
+
* and override settings of the enqueued {@apilink Request} objects.
|
|
34
34
|
*
|
|
35
35
|
* **Example usage**
|
|
36
36
|
*
|
|
@@ -47,7 +47,7 @@ var EnqueueStrategy;
|
|
|
47
47
|
* ```
|
|
48
48
|
*
|
|
49
49
|
* @param options All `enqueueLinks()` parameters are passed via an options object.
|
|
50
|
-
* @returns Promise that resolves to {@
|
|
50
|
+
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
|
|
51
51
|
*/
|
|
52
52
|
async function enqueueLinks(options) {
|
|
53
53
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
@@ -124,9 +124,10 @@ async function enqueueLinks(options) {
|
|
|
124
124
|
exports.enqueueLinks = enqueueLinks;
|
|
125
125
|
/**
|
|
126
126
|
* @internal
|
|
127
|
-
* This method helps resolve the baseUrl that will be used for filtering in {@
|
|
127
|
+
* This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
|
|
128
128
|
* - If a user provides a base url, we always return it
|
|
129
|
-
* - If a user specifies {@
|
|
129
|
+
* - If a user specifies {@apilink EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
|
|
130
|
+
* request domain, or a redirected one
|
|
130
131
|
* - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
|
|
131
132
|
*/
|
|
132
133
|
function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC"}
|
|
@@ -52,8 +52,8 @@ export declare function createRequests(requestOptions: (string | RequestOptions)
|
|
|
52
52
|
*/
|
|
53
53
|
export declare function createRequestOptions(sources: (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData'>): RequestOptions[];
|
|
54
54
|
/**
|
|
55
|
-
* Takes an Apify {@
|
|
56
|
-
* {@
|
|
55
|
+
* Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
|
|
56
|
+
* {@apilink enqueueLinks} to modify requests before enqueuing them.
|
|
57
57
|
*/
|
|
58
58
|
export interface RequestTransform {
|
|
59
59
|
/**
|
package/errors.d.ts
CHANGED
|
@@ -13,4 +13,10 @@ export declare class CriticalError extends NonRetryableError {
|
|
|
13
13
|
*/
|
|
14
14
|
export declare class MissingRouteError extends CriticalError {
|
|
15
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* Indicates that the request should be retried (while still respecting the maximum number of retries).
|
|
18
|
+
*/
|
|
19
|
+
export declare class RetryRequestError extends Error {
|
|
20
|
+
constructor(message?: string);
|
|
21
|
+
}
|
|
16
22
|
//# sourceMappingURL=errors.d.ts.map
|
package/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;GAEG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG"}
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;GAEG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B"}
|
package/errors.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
|
|
3
|
+
exports.RetryRequestError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
|
|
4
4
|
/**
|
|
5
5
|
* Errors of `NonRetryableError` type will never be retried by the crawler.
|
|
6
6
|
*/
|
|
@@ -19,4 +19,13 @@ exports.CriticalError = CriticalError;
|
|
|
19
19
|
class MissingRouteError extends CriticalError {
|
|
20
20
|
}
|
|
21
21
|
exports.MissingRouteError = MissingRouteError;
|
|
22
|
+
/**
|
|
23
|
+
* Indicates that the request should be retried (while still respecting the maximum number of retries).
|
|
24
|
+
*/
|
|
25
|
+
class RetryRequestError extends Error {
|
|
26
|
+
constructor(message) {
|
|
27
|
+
super(message ?? "Request is being retried at the user's request");
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
exports.RetryRequestError = RetryRequestError;
|
|
22
31
|
//# sourceMappingURL=errors.js.map
|
package/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,MAAa,iBAAkB,SAAQ,KAAK;CAAG;AAA/C,8CAA+C;AAE/C;;GAEG;AACH,MAAa,aAAc,SAAQ,iBAAiB;CAAG;AAAvD,sCAAuD;AAEvD;;GAEG;AACH,MAAa,iBAAkB,SAAQ,aAAa;CAAG;AAAvD,8CAAuD"}
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,MAAa,iBAAkB,SAAQ,KAAK;CAAG;AAA/C,8CAA+C;AAE/C;;GAEG;AACH,MAAa,aAAc,SAAQ,iBAAiB;CAAG;AAAvD,sCAAuD;AAEvD;;GAEG;AACH,MAAa,iBAAkB,SAAQ,aAAa;CAAG;AAAvD,8CAAuD;AAEvD;;GAEG;AACH,MAAa,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAJD,8CAIC"}
|
package/index.mjs
CHANGED
|
@@ -31,6 +31,7 @@ export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
|
31
31
|
export const Request = mod.Request;
|
|
32
32
|
export const RequestList = mod.RequestList;
|
|
33
33
|
export const RequestQueue = mod.RequestQueue;
|
|
34
|
+
export const RetryRequestError = mod.RetryRequestError;
|
|
34
35
|
export const Router = mod.Router;
|
|
35
36
|
export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
|
|
36
37
|
export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.3
|
|
3
|
+
"version": "3.0.3",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -58,9 +58,9 @@
|
|
|
58
58
|
"@apify/pseudo_url": "^2.0.0",
|
|
59
59
|
"@apify/timeout": "^0.3.0",
|
|
60
60
|
"@apify/utilities": "^2.0.0",
|
|
61
|
-
"@crawlee/memory-storage": "^3.0.3
|
|
62
|
-
"@crawlee/types": "^3.0.3
|
|
63
|
-
"@crawlee/utils": "^3.0.3
|
|
61
|
+
"@crawlee/memory-storage": "^3.0.3",
|
|
62
|
+
"@crawlee/types": "^3.0.3",
|
|
63
|
+
"@crawlee/utils": "^3.0.3",
|
|
64
64
|
"@types/tough-cookie": "^4.0.2",
|
|
65
65
|
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
66
66
|
"fs-extra": "^10.1.0",
|
package/proxy_configuration.d.ts
CHANGED
|
@@ -11,14 +11,14 @@ export interface ProxyConfigurationOptions {
|
|
|
11
11
|
/**
|
|
12
12
|
* Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
|
|
13
13
|
* and should always return stringified proxy URL. Can be asynchronous.
|
|
14
|
-
* This function is used to generate the URL when {@
|
|
14
|
+
* This function is used to generate the URL when {@apilink ProxyConfiguration.newUrl} or {@apilink ProxyConfiguration.newProxyInfo} is called.
|
|
15
15
|
*/
|
|
16
16
|
newUrlFunction?: ProxyConfigurationFunction;
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
19
|
* The main purpose of the ProxyInfo object is to provide information
|
|
20
20
|
* about the current proxy connection used by the crawler for the request.
|
|
21
|
-
* Outside of crawlers, you can get this object by calling {@
|
|
21
|
+
* Outside of crawlers, you can get this object by calling {@apilink ProxyConfiguration.newProxyInfo}.
|
|
22
22
|
*
|
|
23
23
|
* **Example usage:**
|
|
24
24
|
*
|
|
@@ -47,7 +47,7 @@ export interface ProxyConfigurationOptions {
|
|
|
47
47
|
*/
|
|
48
48
|
export interface ProxyInfo {
|
|
49
49
|
/**
|
|
50
|
-
* The identifier of used {@
|
|
50
|
+
* The identifier of used {@apilink Session}, if used.
|
|
51
51
|
*/
|
|
52
52
|
sessionId?: string;
|
|
53
53
|
/**
|
|
@@ -75,9 +75,9 @@ export interface ProxyInfo {
|
|
|
75
75
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
76
76
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
77
77
|
* them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
|
|
78
|
-
* the {@
|
|
78
|
+
* the {@apilink ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
|
|
79
79
|
*
|
|
80
|
-
* If you want to use your own proxies, use the {@
|
|
80
|
+
* If you want to use your own proxies, use the {@apilink ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
|
|
81
81
|
* be rotated by the configuration if this option is provided.
|
|
82
82
|
*
|
|
83
83
|
* **Example usage:**
|
|
@@ -107,7 +107,7 @@ export declare class ProxyConfiguration {
|
|
|
107
107
|
protected newUrlFunction?: ProxyConfigurationFunction;
|
|
108
108
|
protected log: import("@apify/log").Log;
|
|
109
109
|
/**
|
|
110
|
-
* Creates a {@
|
|
110
|
+
* Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
111
111
|
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
112
112
|
* them to use the selected proxies for all connections.
|
|
113
113
|
*
|
|
@@ -128,16 +128,16 @@ export declare class ProxyConfiguration {
|
|
|
128
128
|
*/
|
|
129
129
|
constructor(options?: ProxyConfigurationOptions);
|
|
130
130
|
/**
|
|
131
|
-
* This function creates a new {@
|
|
131
|
+
* This function creates a new {@apilink ProxyInfo} info object.
|
|
132
132
|
* It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
|
|
133
133
|
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
134
134
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
135
|
-
* If you need the URL string only, use {@
|
|
135
|
+
* If you need the URL string only, use {@apilink ProxyConfiguration.newUrl}.
|
|
136
136
|
* @param [sessionId]
|
|
137
|
-
* Represents the identifier of user {@
|
|
137
|
+
* Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or
|
|
138
138
|
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
139
139
|
* When the provided sessionId is a number, it's converted to a string. Property sessionId of
|
|
140
|
-
* {@
|
|
140
|
+
* {@apilink ProxyInfo} is always returned as a type string.
|
|
141
141
|
*
|
|
142
142
|
* All the HTTP requests going through the proxy with the same session identifier
|
|
143
143
|
* will use the same target proxy server (i.e. the same IP address).
|
|
@@ -148,7 +148,7 @@ export declare class ProxyConfiguration {
|
|
|
148
148
|
/**
|
|
149
149
|
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
150
150
|
* @param [sessionId]
|
|
151
|
-
* Represents the identifier of user {@
|
|
151
|
+
* Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or
|
|
152
152
|
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
153
153
|
* When the provided sessionId is a number, it's converted to a string.
|
|
154
154
|
*
|
package/proxy_configuration.js
CHANGED
|
@@ -8,9 +8,9 @@ const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
|
8
8
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
9
9
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
10
10
|
* them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
|
|
11
|
-
* the {@
|
|
11
|
+
* the {@apilink ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
|
|
12
12
|
*
|
|
13
|
-
* If you want to use your own proxies, use the {@
|
|
13
|
+
* If you want to use your own proxies, use the {@apilink ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
|
|
14
14
|
* be rotated by the configuration if this option is provided.
|
|
15
15
|
*
|
|
16
16
|
* **Example usage:**
|
|
@@ -34,7 +34,7 @@ const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
|
34
34
|
*/
|
|
35
35
|
class ProxyConfiguration {
|
|
36
36
|
/**
|
|
37
|
-
* Creates a {@
|
|
37
|
+
* Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
38
38
|
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
39
39
|
* them to use the selected proxies for all connections.
|
|
40
40
|
*
|
|
@@ -104,16 +104,16 @@ class ProxyConfiguration {
|
|
|
104
104
|
this.newUrlFunction = newUrlFunction;
|
|
105
105
|
}
|
|
106
106
|
/**
|
|
107
|
-
* This function creates a new {@
|
|
107
|
+
* This function creates a new {@apilink ProxyInfo} info object.
|
|
108
108
|
* It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
|
|
109
109
|
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
110
110
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
111
|
-
* If you need the URL string only, use {@
|
|
111
|
+
* If you need the URL string only, use {@apilink ProxyConfiguration.newUrl}.
|
|
112
112
|
* @param [sessionId]
|
|
113
|
-
* Represents the identifier of user {@
|
|
113
|
+
* Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or
|
|
114
114
|
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
115
115
|
* When the provided sessionId is a number, it's converted to a string. Property sessionId of
|
|
116
|
-
* {@
|
|
116
|
+
* {@apilink ProxyInfo} is always returned as a type string.
|
|
117
117
|
*
|
|
118
118
|
* All the HTTP requests going through the proxy with the same session identifier
|
|
119
119
|
* will use the same target proxy server (i.e. the same IP address).
|
|
@@ -137,7 +137,7 @@ class ProxyConfiguration {
|
|
|
137
137
|
/**
|
|
138
138
|
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
139
139
|
* @param [sessionId]
|
|
140
|
-
* Represents the identifier of user {@
|
|
140
|
+
* Represents the identifier of user {@apilink Session} that can be managed by the {@apilink SessionPool} or
|
|
141
141
|
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
142
142
|
* When the provided sessionId is a number, it's converted to a string.
|
|
143
143
|
*
|
package/request.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { AllowedHttpMethods } from './typedefs';
|
|
|
10
10
|
* Each `Request` instance has the `uniqueKey` property, which can be either specified
|
|
11
11
|
* manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
|
|
12
12
|
* are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
|
|
13
|
-
* such as {@
|
|
13
|
+
* such as {@apilink RequestList}, {@apilink RequestQueue}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}.
|
|
14
14
|
*
|
|
15
15
|
* Example use:
|
|
16
16
|
*
|
|
@@ -40,7 +40,7 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
40
40
|
* An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
|
|
41
41
|
* to be included.
|
|
42
42
|
*
|
|
43
|
-
* When using {@
|
|
43
|
+
* When using {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}, meta tag and JavaScript redirects may,
|
|
44
44
|
* or may not be included, depending on their nature. This generally means that redirects,
|
|
45
45
|
* which happen immediately will most likely be included, but delayed redirects will not.
|
|
46
46
|
*/
|
|
@@ -100,7 +100,7 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
100
100
|
protected _hashPayload(payload: BinaryLike): string;
|
|
101
101
|
}
|
|
102
102
|
/**
|
|
103
|
-
* Specifies required and optional fields for constructing a {@
|
|
103
|
+
* Specifies required and optional fields for constructing a {@apilink Request}.
|
|
104
104
|
*/
|
|
105
105
|
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
106
106
|
/** URL of the web page to crawl. It must be a non-empty string. */
|
package/request.js
CHANGED
|
@@ -34,7 +34,7 @@ const requestOptionalPredicates = {
|
|
|
34
34
|
* Each `Request` instance has the `uniqueKey` property, which can be either specified
|
|
35
35
|
* manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
|
|
36
36
|
* are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
|
|
37
|
-
* such as {@
|
|
37
|
+
* such as {@apilink RequestList}, {@apilink RequestQueue}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}.
|
|
38
38
|
*
|
|
39
39
|
* Example use:
|
|
40
40
|
*
|
|
@@ -78,7 +78,7 @@ class Request {
|
|
|
78
78
|
* An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
|
|
79
79
|
* to be included.
|
|
80
80
|
*
|
|
81
|
-
* When using {@
|
|
81
|
+
* When using {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}, meta tag and JavaScript redirects may,
|
|
82
82
|
* or may not be included, depending on their nature. This generally means that redirects,
|
|
83
83
|
* which happen immediately will most likely be included, but delayed redirects will not.
|
|
84
84
|
*/
|
|
@@ -5,7 +5,7 @@ import type { IncomingMessage } from 'node:http';
|
|
|
5
5
|
import type { Cookie } from 'tough-cookie';
|
|
6
6
|
import { CookieJar } from 'tough-cookie';
|
|
7
7
|
/**
|
|
8
|
-
* Persistable {@
|
|
8
|
+
* Persistable {@apilink Session} state.
|
|
9
9
|
*/
|
|
10
10
|
export interface SessionState {
|
|
11
11
|
id: string;
|
|
@@ -7,7 +7,7 @@ import type { SessionOptions } from './session';
|
|
|
7
7
|
import { Session } from './session';
|
|
8
8
|
import type { EventManager } from '../events/event_manager';
|
|
9
9
|
/**
|
|
10
|
-
* Factory user-function which creates customized {@
|
|
10
|
+
* Factory user-function which creates customized {@apilink Session} instances.
|
|
11
11
|
*/
|
|
12
12
|
export interface CreateSession {
|
|
13
13
|
/**
|
|
@@ -24,7 +24,7 @@ export interface SessionPoolOptions {
|
|
|
24
24
|
* @default 1000
|
|
25
25
|
*/
|
|
26
26
|
maxPoolSize?: number;
|
|
27
|
-
/** The configuration options for {@
|
|
27
|
+
/** The configuration options for {@apilink Session} instances. */
|
|
28
28
|
sessionOptions?: SessionOptions;
|
|
29
29
|
/** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
|
|
30
30
|
persistStateKeyValueStoreId?: string;
|
|
@@ -50,11 +50,11 @@ export interface SessionPoolOptions {
|
|
|
50
50
|
}
|
|
51
51
|
/**
|
|
52
52
|
* Handles the rotation, creation and persistence of user-like sessions.
|
|
53
|
-
* Creates a pool of {@
|
|
53
|
+
* Creates a pool of {@apilink Session} instances, that are randomly rotated.
|
|
54
54
|
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
55
55
|
* Learn more in the [Session management guide](../../../docs/guides/session-management).
|
|
56
56
|
*
|
|
57
|
-
* You can create one by calling the {@
|
|
57
|
+
* You can create one by calling the {@apilink SessionPool.open} function.
|
|
58
58
|
*
|
|
59
59
|
* Session pool is already integrated into crawlers, and it can significantly improve your scraper
|
|
60
60
|
* performance with just 2 lines of code.
|
|
@@ -69,10 +69,10 @@ export interface SessionPoolOptions {
|
|
|
69
69
|
* })
|
|
70
70
|
* ```
|
|
71
71
|
*
|
|
72
|
-
* You can configure the pool with many options. See the {@
|
|
73
|
-
* Session pool is by default persisted in default {@
|
|
72
|
+
* You can configure the pool with many options. See the {@apilink SessionPoolOptions}.
|
|
73
|
+
* Session pool is by default persisted in default {@apilink KeyValueStore}.
|
|
74
74
|
* If you want to have one pool for all runs you have to specify
|
|
75
|
-
* {@
|
|
75
|
+
* {@apilink SessionPoolOptions.persistStateKeyValueStoreId}.
|
|
76
76
|
*
|
|
77
77
|
* **Advanced usage:**
|
|
78
78
|
*
|
|
@@ -133,8 +133,8 @@ export declare class SessionPool extends EventEmitter {
|
|
|
133
133
|
*/
|
|
134
134
|
get retiredSessionsCount(): number;
|
|
135
135
|
/**
|
|
136
|
-
* Starts periodic state persistence and potentially loads SessionPool state from {@
|
|
137
|
-
* It is called automatically by the {@
|
|
136
|
+
* Starts periodic state persistence and potentially loads SessionPool state from {@apilink KeyValueStore}.
|
|
137
|
+
* It is called automatically by the {@apilink SessionPool.open} function.
|
|
138
138
|
*/
|
|
139
139
|
initialize(): Promise<void>;
|
|
140
140
|
/**
|
|
@@ -165,7 +165,7 @@ export declare class SessionPool extends EventEmitter {
|
|
|
165
165
|
sessions: import("./session").SessionState[];
|
|
166
166
|
};
|
|
167
167
|
/**
|
|
168
|
-
* Persists the current state of the `SessionPool` into the default {@
|
|
168
|
+
* Persists the current state of the `SessionPool` into the default {@apilink KeyValueStore}.
|
|
169
169
|
* The state is persisted automatically in regular intervals.
|
|
170
170
|
*/
|
|
171
171
|
persistState(): Promise<void>;
|
|
@@ -222,9 +222,9 @@ export declare class SessionPool extends EventEmitter {
|
|
|
222
222
|
protected _maybeLoadSessionPool(): Promise<void>;
|
|
223
223
|
/**
|
|
224
224
|
* Opens a SessionPool and returns a promise resolving to an instance
|
|
225
|
-
* of the {@
|
|
225
|
+
* of the {@apilink SessionPool} class that is already initialized.
|
|
226
226
|
*
|
|
227
|
-
* For more details and code examples, see the {@
|
|
227
|
+
* For more details and code examples, see the {@apilink SessionPool} class.
|
|
228
228
|
*/
|
|
229
229
|
static open(options?: SessionPoolOptions): Promise<SessionPool>;
|
|
230
230
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAiBK,QAAQ,CAAC,MAAM;IAhB7D,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAEhD;;OAEG;gBACS,OAAO,GAAE,kBAAuB,EAAW,MAAM,gBAAkC;IA6C/F;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8BrD;;;OAGG;IACH,QAAQ;;;;;IAQR;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO;IAU7H;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}
|
|
@@ -10,11 +10,11 @@ const key_value_store_1 = require("../storages/key_value_store");
|
|
|
10
10
|
const session_1 = require("./session");
|
|
11
11
|
/**
|
|
12
12
|
* Handles the rotation, creation and persistence of user-like sessions.
|
|
13
|
-
* Creates a pool of {@
|
|
13
|
+
* Creates a pool of {@apilink Session} instances, that are randomly rotated.
|
|
14
14
|
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
15
15
|
* Learn more in the [Session management guide](../../../docs/guides/session-management).
|
|
16
16
|
*
|
|
17
|
-
* You can create one by calling the {@
|
|
17
|
+
* You can create one by calling the {@apilink SessionPool.open} function.
|
|
18
18
|
*
|
|
19
19
|
* Session pool is already integrated into crawlers, and it can significantly improve your scraper
|
|
20
20
|
* performance with just 2 lines of code.
|
|
@@ -29,10 +29,10 @@ const session_1 = require("./session");
|
|
|
29
29
|
* })
|
|
30
30
|
* ```
|
|
31
31
|
*
|
|
32
|
-
* You can configure the pool with many options. See the {@
|
|
33
|
-
* Session pool is by default persisted in default {@
|
|
32
|
+
* You can configure the pool with many options. See the {@apilink SessionPoolOptions}.
|
|
33
|
+
* Session pool is by default persisted in default {@apilink KeyValueStore}.
|
|
34
34
|
* If you want to have one pool for all runs you have to specify
|
|
35
|
-
* {@
|
|
35
|
+
* {@apilink SessionPoolOptions.persistStateKeyValueStoreId}.
|
|
36
36
|
*
|
|
37
37
|
* **Advanced usage:**
|
|
38
38
|
*
|
|
@@ -191,8 +191,8 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
191
191
|
return this.sessions.filter((session) => !session.isUsable()).length;
|
|
192
192
|
}
|
|
193
193
|
/**
|
|
194
|
-
* Starts periodic state persistence and potentially loads SessionPool state from {@
|
|
195
|
-
* It is called automatically by the {@
|
|
194
|
+
* Starts periodic state persistence and potentially loads SessionPool state from {@apilink KeyValueStore}.
|
|
195
|
+
* It is called automatically by the {@apilink SessionPool.open} function.
|
|
196
196
|
*/
|
|
197
197
|
async initialize() {
|
|
198
198
|
this.keyValueStore = await key_value_store_1.KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
|
|
@@ -266,7 +266,7 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
266
266
|
};
|
|
267
267
|
}
|
|
268
268
|
/**
|
|
269
|
-
* Persists the current state of the `SessionPool` into the default {@
|
|
269
|
+
* Persists the current state of the `SessionPool` into the default {@apilink KeyValueStore}.
|
|
270
270
|
* The state is persisted automatically in regular intervals.
|
|
271
271
|
*/
|
|
272
272
|
async persistState() {
|
|
@@ -382,9 +382,9 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
382
382
|
}
|
|
383
383
|
/**
|
|
384
384
|
* Opens a SessionPool and returns a promise resolving to an instance
|
|
385
|
-
* of the {@
|
|
385
|
+
* of the {@apilink SessionPool} class that is already initialized.
|
|
386
386
|
*
|
|
387
|
-
* For more details and code examples, see the {@
|
|
387
|
+
* For more details and code examples, see the {@apilink SessionPool} class.
|
|
388
388
|
*/
|
|
389
389
|
static async open(options) {
|
|
390
390
|
const sessionPool = new SessionPool(options);
|