@crawlee/core 3.1.5-beta.2 → 3.1.5-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/crawler_commons.d.ts +1 -1
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +51 -4
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +50 -10
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +7 -7
- package/enqueue_links/shared.d.ts.map +1 -1
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/package.json +11 -9
- package/router.d.ts +3 -4
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +2 -2
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_queue.d.ts +11 -13
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts +2 -2
- package/typedefs.d.ts.map +1 -1
- package/except.d.ts +0 -7
- package/except.d.ts.map +0 -1
- package/except.js +0 -4
- package/except.js.map +0 -1
|
@@ -5,7 +5,7 @@ import type { Log } from '../log';
|
|
|
5
5
|
import type { ProxyInfo } from '../proxy_configuration';
|
|
6
6
|
import type { Request } from '../request';
|
|
7
7
|
import type { Session } from '../session_pool/session';
|
|
8
|
-
export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<
|
|
8
|
+
export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
|
|
9
9
|
id: string;
|
|
10
10
|
/**
|
|
11
11
|
* The original {@apilink Request} object.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,IAAI,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,IAAI,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGvD,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,OAAO,CAAC;IAC9H,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAE7E;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,QAAQ,GAAG,MAAM,EAAE,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1G"}
|
|
@@ -91,10 +91,45 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
91
91
|
transformRequestFunction?: RequestTransform;
|
|
92
92
|
/**
|
|
93
93
|
* The strategy to use when enqueueing the urls.
|
|
94
|
+
*
|
|
95
|
+
* Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
|
|
96
|
+
*
|
|
97
|
+
* ```md
|
|
98
|
+
* Protocol Domain
|
|
99
|
+
* ┌────┐ ┌─────────┐
|
|
100
|
+
* https://example.crawlee.dev/...
|
|
101
|
+
* │ └─────────────────┤
|
|
102
|
+
* │ Hostname │
|
|
103
|
+
* │ │
|
|
104
|
+
* └─────────────────────────┘
|
|
105
|
+
* Origin
|
|
106
|
+
*```
|
|
107
|
+
*
|
|
94
108
|
* @default EnqueueStrategy.SameHostname
|
|
95
109
|
*/
|
|
96
|
-
strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname';
|
|
110
|
+
strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
|
|
97
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* The different enqueueing strategies available.
|
|
114
|
+
*
|
|
115
|
+
* Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
|
|
116
|
+
*
|
|
117
|
+
* ```md
|
|
118
|
+
* Protocol Domain
|
|
119
|
+
* ┌────┐ ┌─────────┐
|
|
120
|
+
* https://example.crawlee.dev/...
|
|
121
|
+
* │ └─────────────────┤
|
|
122
|
+
* │ Hostname │
|
|
123
|
+
* │ │
|
|
124
|
+
* └─────────────────────────┘
|
|
125
|
+
* Origin
|
|
126
|
+
*```
|
|
127
|
+
*
|
|
128
|
+
* - The `Protocol` is usually `http` or `https`
|
|
129
|
+
* - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
|
|
130
|
+
* - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
|
|
131
|
+
* - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
|
|
132
|
+
*/
|
|
98
133
|
export declare enum EnqueueStrategy {
|
|
99
134
|
/**
|
|
100
135
|
* Matches any URLs found
|
|
@@ -102,16 +137,28 @@ export declare enum EnqueueStrategy {
|
|
|
102
137
|
All = "all",
|
|
103
138
|
/**
|
|
104
139
|
* Matches any URLs that have the same hostname.
|
|
105
|
-
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://example.com/`, but
|
|
140
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
106
141
|
* `https://example.com/hello` will not be matched.
|
|
142
|
+
*
|
|
143
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
107
144
|
*/
|
|
108
145
|
SameHostname = "same-hostname",
|
|
109
146
|
/**
|
|
110
|
-
* Matches any URLs that have the same
|
|
147
|
+
* Matches any URLs that have the same domain as the base URL.
|
|
111
148
|
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
112
149
|
* `https://example.com`.
|
|
150
|
+
*
|
|
151
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
152
|
+
*/
|
|
153
|
+
SameDomain = "same-domain",
|
|
154
|
+
/**
|
|
155
|
+
* Matches any URLs that have the same hostname and protocol.
|
|
156
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
157
|
+
* `http://wow.example.com/hello` will not be matched.
|
|
158
|
+
*
|
|
159
|
+
* > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
|
|
113
160
|
*/
|
|
114
|
-
|
|
161
|
+
SameOrigin = "same-origin"
|
|
115
162
|
}
|
|
116
163
|
/**
|
|
117
164
|
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAS3G,OAAO,KAAK,EAAE,YAAY,EAAE,4BAA4B,EAAE,MAAM,2BAA2B,CAAC;AAG5F,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAS3G,OAAO,KAAK,EAAE,YAAY,EAAE,4BAA4B,EAAE,MAAM,2BAA2B,CAAC;AAG5F,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;CACxF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAgItI;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -6,6 +6,27 @@ const tldts_1 = require("tldts");
|
|
|
6
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
7
7
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
8
8
|
const shared_1 = require("./shared");
|
|
9
|
+
/**
|
|
10
|
+
* The different enqueueing strategies available.
|
|
11
|
+
*
|
|
12
|
+
* Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
|
|
13
|
+
*
|
|
14
|
+
* ```md
|
|
15
|
+
* Protocol Domain
|
|
16
|
+
* ┌────┐ ┌─────────┐
|
|
17
|
+
* https://example.crawlee.dev/...
|
|
18
|
+
* │ └─────────────────┤
|
|
19
|
+
* │ Hostname │
|
|
20
|
+
* │ │
|
|
21
|
+
* └─────────────────────────┘
|
|
22
|
+
* Origin
|
|
23
|
+
*```
|
|
24
|
+
*
|
|
25
|
+
* - The `Protocol` is usually `http` or `https`
|
|
26
|
+
* - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
|
|
27
|
+
* - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
|
|
28
|
+
* - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
|
|
29
|
+
*/
|
|
9
30
|
var EnqueueStrategy;
|
|
10
31
|
(function (EnqueueStrategy) {
|
|
11
32
|
/**
|
|
@@ -14,16 +35,28 @@ var EnqueueStrategy;
|
|
|
14
35
|
EnqueueStrategy["All"] = "all";
|
|
15
36
|
/**
|
|
16
37
|
* Matches any URLs that have the same hostname.
|
|
17
|
-
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://example.com/`, but
|
|
38
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
18
39
|
* `https://example.com/hello` will not be matched.
|
|
40
|
+
*
|
|
41
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
19
42
|
*/
|
|
20
43
|
EnqueueStrategy["SameHostname"] = "same-hostname";
|
|
21
44
|
/**
|
|
22
|
-
* Matches any URLs that have the same
|
|
45
|
+
* Matches any URLs that have the same domain as the base URL.
|
|
23
46
|
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
24
47
|
* `https://example.com`.
|
|
48
|
+
*
|
|
49
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
25
50
|
*/
|
|
26
51
|
EnqueueStrategy["SameDomain"] = "same-domain";
|
|
52
|
+
/**
|
|
53
|
+
* Matches any URLs that have the same hostname and protocol.
|
|
54
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
55
|
+
* `http://wow.example.com/hello` will not be matched.
|
|
56
|
+
*
|
|
57
|
+
* > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
|
|
58
|
+
*/
|
|
59
|
+
EnqueueStrategy["SameOrigin"] = "same-origin";
|
|
27
60
|
})(EnqueueStrategy = exports.EnqueueStrategy || (exports.EnqueueStrategy = {}));
|
|
28
61
|
/**
|
|
29
62
|
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
|
|
@@ -70,11 +103,7 @@ async function enqueueLinks(options) {
|
|
|
70
103
|
globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
|
|
71
104
|
regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
|
|
72
105
|
transformRequestFunction: ow_1.default.optional.function,
|
|
73
|
-
strategy: ow_1.default.optional.string.oneOf(
|
|
74
|
-
EnqueueStrategy.All,
|
|
75
|
-
EnqueueStrategy.SameHostname,
|
|
76
|
-
EnqueueStrategy.SameDomain,
|
|
77
|
-
]),
|
|
106
|
+
strategy: ow_1.default.optional.string.oneOf(Object.values(EnqueueStrategy)),
|
|
78
107
|
}));
|
|
79
108
|
const { requestQueue, limit, urls, pseudoUrls, globs, regexps, transformRequestFunction, forefront, } = options;
|
|
80
109
|
const urlPatternObjects = [];
|
|
@@ -99,7 +128,7 @@ async function enqueueLinks(options) {
|
|
|
99
128
|
// We need to get the origin of the passed in domain in the event someone sets baseUrl
|
|
100
129
|
// to an url like https://example.com/deep/default/path and one of the found urls is an
|
|
101
130
|
// absolute relative path (/path/to/page)
|
|
102
|
-
enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
|
|
131
|
+
enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
|
|
103
132
|
break;
|
|
104
133
|
case EnqueueStrategy.SameDomain: {
|
|
105
134
|
// Get the actual hostname from the base url
|
|
@@ -107,15 +136,20 @@ async function enqueueLinks(options) {
|
|
|
107
136
|
if (baseUrlHostname) {
|
|
108
137
|
// We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
|
|
109
138
|
url.hostname = baseUrlHostname;
|
|
110
|
-
enqueueStrategyPatterns.push({ glob: `${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**` }, { glob: `${url.origin}/**` });
|
|
139
|
+
enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) }, { glob: ignoreHttpSchema(`${url.origin}/**`) });
|
|
111
140
|
}
|
|
112
141
|
else {
|
|
113
142
|
// We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
|
|
114
143
|
// as SameDomainAndSubdomain
|
|
115
|
-
enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
|
|
144
|
+
enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
|
|
116
145
|
}
|
|
117
146
|
break;
|
|
118
147
|
}
|
|
148
|
+
case EnqueueStrategy.SameOrigin: {
|
|
149
|
+
// The same behavior as SameHostname, but respecting the protocol of the URL
|
|
150
|
+
enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
119
153
|
case EnqueueStrategy.All:
|
|
120
154
|
default:
|
|
121
155
|
break;
|
|
@@ -176,4 +210,10 @@ function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestU
|
|
|
176
210
|
return originalUrlOrigin;
|
|
177
211
|
}
|
|
178
212
|
exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
|
|
213
|
+
/**
|
|
214
|
+
* Internal function that changes the enqueue globs to match both http and https
|
|
215
|
+
*/
|
|
216
|
+
function ignoreHttpSchema(pattern) {
|
|
217
|
+
return pattern.replace(/^(https?):\/\//, 'http{s,}://');
|
|
218
|
+
}
|
|
179
219
|
//# sourceMappingURL=enqueue_links.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAI7B,qCAOkB;
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAI7B,qCAOkB;AA2HlB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAAkE;IACjG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/C,MAAM,IAAI,UAAU,CAAC;YACjB,mCAAmC;YACnC,4JAA4J;YAC5J,+GAA+G;SAClH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;KACjB;IAED,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;KACrE,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,GACZ,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;iBAChF;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,OAAO,IAAA,uBAAc,EAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;SAClE;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAC3F,+EAA+E;QAC/E,OAAO,IAAA,iCAAwB,EAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;AAC7D,CAAC;AAhID,oCAgIC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
import type { RequestOptions } from '../request';
|
|
2
2
|
import { Request } from '../request';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links';
|
|
4
|
-
export
|
|
4
|
+
export type UrlPatternObject = {
|
|
5
5
|
glob?: string;
|
|
6
6
|
regexp?: RegExp;
|
|
7
7
|
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
|
|
8
|
-
export
|
|
8
|
+
export type PseudoUrlObject = {
|
|
9
9
|
purl: string;
|
|
10
10
|
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
|
|
11
|
-
export
|
|
12
|
-
export
|
|
11
|
+
export type PseudoUrlInput = string | PseudoUrlObject;
|
|
12
|
+
export type GlobObject = {
|
|
13
13
|
glob: string;
|
|
14
14
|
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
|
|
15
|
-
export
|
|
16
|
-
export
|
|
15
|
+
export type GlobInput = string | GlobObject;
|
|
16
|
+
export type RegExpObject = {
|
|
17
17
|
regexp: RegExp;
|
|
18
18
|
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
|
|
19
|
-
export
|
|
19
|
+
export type RegExpInput = RegExp | RegExpObject;
|
|
20
20
|
/**
|
|
21
21
|
* @ignore
|
|
22
22
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAY3D,
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAY3D,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE/H,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE1H,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE9H,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,CAiBjG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAiB9E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBxF;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CA6B7H;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAuBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EAC7C,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,CAAM,GAC9D,cAAc,EAAE,CAqBlB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
|
|
@@ -8,7 +8,7 @@ export declare const enum EventType {
|
|
|
8
8
|
ABORTING = "aborting",
|
|
9
9
|
EXIT = "exit"
|
|
10
10
|
}
|
|
11
|
-
export
|
|
11
|
+
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
|
|
12
12
|
interface Intervals {
|
|
13
13
|
persistState?: BetterIntervalID;
|
|
14
14
|
systemInfo?: BetterIntervalID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,kIAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAI7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACH,6BAA6B;CAGhC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.1.5-beta.
|
|
3
|
+
"version": "3.1.5-beta.21",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
},
|
|
45
45
|
"homepage": "https://crawlee.dev",
|
|
46
46
|
"scripts": {
|
|
47
|
-
"build": "
|
|
47
|
+
"build": "yarn clean && yarn compile && yarn copy",
|
|
48
48
|
"clean": "rimraf ./dist",
|
|
49
49
|
"compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
|
|
50
50
|
"copy": "ts-node -T ../../scripts/copy.ts"
|
|
@@ -59,20 +59,22 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.0",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.0.0",
|
|
62
|
-
"@crawlee/memory-storage": "^3.1.5-beta.
|
|
63
|
-
"@crawlee/types": "^3.1.5-beta.
|
|
64
|
-
"@crawlee/utils": "^3.1.5-beta.
|
|
62
|
+
"@crawlee/memory-storage": "^3.1.5-beta.21",
|
|
63
|
+
"@crawlee/types": "^3.1.5-beta.21",
|
|
64
|
+
"@crawlee/utils": "^3.1.5-beta.21",
|
|
65
65
|
"@types/tough-cookie": "^4.0.2",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
67
67
|
"csv-stringify": "^6.2.0",
|
|
68
|
-
"fs-extra": "^
|
|
69
|
-
"json5": "^2.2.
|
|
70
|
-
"minimatch": "^
|
|
68
|
+
"fs-extra": "^11.0.0",
|
|
69
|
+
"json5": "^2.2.3",
|
|
70
|
+
"minimatch": "^6.0.0",
|
|
71
71
|
"ow": "^0.28.1",
|
|
72
72
|
"stream-chain": "^2.2.5",
|
|
73
73
|
"stream-json": "^1.7.4",
|
|
74
74
|
"tldts": "^5.7.80",
|
|
75
75
|
"tough-cookie": "^4.0.0",
|
|
76
|
+
"tslib": "^2.4.0",
|
|
76
77
|
"type-fest": "^3.0.0"
|
|
77
|
-
}
|
|
78
|
+
},
|
|
79
|
+
"gitHead": "02e61e35d42248aba8b8cd04b887ab1e3d8ddb0e"
|
|
78
80
|
}
|
package/router.d.ts
CHANGED
|
@@ -2,11 +2,10 @@ import type { Dictionary } from '@crawlee/types';
|
|
|
2
2
|
import type { CrawlingContext } from './crawlers/crawler_commons';
|
|
3
3
|
import type { Awaitable } from './typedefs';
|
|
4
4
|
import type { Request } from './request';
|
|
5
|
-
import type { Except } from './except';
|
|
6
5
|
export interface RouterHandler<Context extends CrawlingContext = CrawlingContext> extends Router<Context> {
|
|
7
6
|
(ctx: Context): Awaitable<void>;
|
|
8
7
|
}
|
|
9
|
-
|
|
8
|
+
type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
|
|
10
9
|
/**
|
|
11
10
|
* Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
|
|
12
11
|
*
|
|
@@ -69,13 +68,13 @@ export declare class Router<Context extends CrawlingContext> {
|
|
|
69
68
|
/**
|
|
70
69
|
* Registers new route handler for given label.
|
|
71
70
|
*/
|
|
72
|
-
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol, handler: (ctx:
|
|
71
|
+
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol, handler: (ctx: Omit<Context, 'request'> & {
|
|
73
72
|
request: Request<UserData>;
|
|
74
73
|
}) => Awaitable<void>): void;
|
|
75
74
|
/**
|
|
76
75
|
* Registers default route handler.
|
|
77
76
|
*/
|
|
78
|
-
addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx:
|
|
77
|
+
addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: Omit<Context, 'request'> & {
|
|
79
78
|
request: Request<UserData>;
|
|
80
79
|
}) => Awaitable<void>): void;
|
|
81
80
|
/**
|
package/router.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKzC,MAAM,WAAW,aAAa,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,CAAE,SAAQ,MAAM,CAAC,OAAO,CAAC;IACrG,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACnC;AAED,KAAK,sBAAsB,CAAC,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAExE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,MAAM,CAAC,OAAO,SAAS,eAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsE;IAC7F,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6C;IAEzE;;;OAGG;IACH,SAAS;IAET;;OAEG;IACH,UAAU,CAAC,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC/E,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;KAAE,KAAK,SAAS,CAAC,IAAI,CAAC;IAMhG;;OAEG;IACH,iBAAiB,CAAC,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EACtF,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;KAAE,KAAK,SAAS,CAAC,IAAI,CAAC;IAMhG;;;OAGG;IACH,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAIjD;;OAEG;IACH,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAgBtE;;OAEG;IACH,OAAO,CAAC,QAAQ;IAShB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,KAAK,aAAa,CAAC,OAAO,CAAC;CAwB7F"}
|
package/router.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":";;;AAIA,qCAA6C;AAE7C,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAQ7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAa,MAAM;IAIf;;;OAGG;IACH;QAPA;;;;mBAAmF,IAAI,GAAG,EAAE;WAAC;QAC7F;;;;mBAAsE,EAAE;WAAC;IAMhD,CAAC;IAE1B;;OAEG;IACH,UAAU,CACN,KAAsB,EACtB,OAA4F;QAE5F,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,iBAAiB,CACb,OAA4F;QAE5F,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,UAA6C;QAC7C,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAuB;QAC9B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;SAClC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;YAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAE,CAAC;SACzC;QAED,MAAM,IAAI,0BAAiB,CACvB,8BAA8B,MAAM,CAAC,KAAK,CAAC,IAAI;cAC7C,6DAA6D;cAC7D,2EAA2E,CAChF,CAAC;IACN,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,KAAsB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACxB,MAAM,OAAO,GAAG,KAAK,KAAK,YAAY;gBAClC,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,oBAAoB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;SAC5B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM;QACT,MAAM,MAAM,GAAG,IAAI,MAAM,EAAW,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE9C,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAElC,MAAM,IAAI,GAAG,KAAK,WAAW,OAAgB;YACzC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC,CAAC;YAEpE,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE;gBACzC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;aAC7B;YAED,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC;QAEF,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEjC,OAAO,IAAyC,CAAC;IACrD,CAAC;CACJ;AAlHD,wBAkHC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0DAA0B;AAC1B,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,yCAAiC;AACjC,+CAA4C;AAC5C,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAsB;WAAC;QAMnD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,EAAE;YACtC,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0DAA0B;AAC1B,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,yCAAiC;AACjC,+CAA4C;AAC5C,oDAAiD;AAGjD,uDAAmD;AACnD,mCAA+C;AAE/C;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAsB;WAAC;QAMnD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,EAAE;YACtC,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,KAAoB,CAAC,CAAC;qBAC5C,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;aACrC;SACJ;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,MAAM,IAAA,4BAAoB,GAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAEhE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IA4DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAiB,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;QAE/C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,kBAAkB,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEjD,kEAAkE;QAClE,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE;YACxC,MAAM,SAAS,GAAG,IAAA,gBAAI,EAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,SAAS,EAAE,CAAC,CAAC;YACvD,IAAI,KAAa,CAAC;YAElB,4CAA4C;YAC5C,IAAI;gBACA,KAAK,GAAG,MAAM,IAAA,mBAAQ,EAAC,SAAS,CAAC,CAAC;aACrC;YAAC,MAAM;gBACJ,SAAS;aACZ;YAED,gEAAgE;YAChE,IAAI;gBACA,OAAO,eAAK,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAM,CAAC;aAC7C;YAAC,MAAM;gBACJ,OAAO,KAAqB,CAAC;aAChC;SACJ;QAED,OAAO,KAAK,CAAC,QAAQ,CAAI,QAAQ,CAAC,CAAC;IACvC,CAAC;CACJ;AAnfD,sCAmfC"}
|
|
@@ -453,7 +453,7 @@ export interface RequestListState {
|
|
|
453
453
|
/** Array of request keys representing those that being processed at the moment. */
|
|
454
454
|
inProgress: string[];
|
|
455
455
|
}
|
|
456
|
-
export
|
|
456
|
+
export type Source = string | (Partial<RequestOptions> & {
|
|
457
457
|
requestsFromUrl?: string;
|
|
458
458
|
regex?: RegExp;
|
|
459
459
|
}) | Request;
|
|
@@ -461,6 +461,6 @@ interface InternalSource {
|
|
|
461
461
|
requestsFromUrl: string;
|
|
462
462
|
regex?: RegExp;
|
|
463
463
|
}
|
|
464
|
-
export
|
|
464
|
+
export type RequestListSourcesFunction = () => Promise<Source[]>;
|
|
465
465
|
export {};
|
|
466
466
|
//# sourceMappingURL=request_list.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;MAIE;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,MAAM,CAAe;IAE7B;;;;OAIG;IACH,OAAO;IA6CP;;;OAGG;YACW,UAAU;IA4BxB;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,GAAG,kBAAkB,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0BpJ;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,
|
|
1
|
+
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;MAIE;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,MAAM,CAAe;IAE7B;;;;OAIG;IACH,OAAO;IA6CP;;;OAGG;YACW,UAAU;IA4BxB;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,GAAG,kBAAkB,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0BpJ;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AACjH,UAAU,cAAc;IAAG,eAAe,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpE,MAAM,MAAM,0BAA0B,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LruCache } from '@apify/datastructures';
|
|
2
|
-
import type { BatchAddRequestsResult, Dictionary, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
|
|
2
|
+
import type { BatchAddRequestsResult, Dictionary, QueueOperationInfo, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
|
|
3
3
|
import type { StorageManagerOptions } from './storage_manager';
|
|
4
4
|
import type { RequestOptions } from '../request';
|
|
5
5
|
import { Request } from '../request';
|
|
@@ -39,10 +39,9 @@ export declare const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
|
|
|
39
39
|
*/
|
|
40
40
|
export declare function getRequestId(uniqueKey: string): string;
|
|
41
41
|
/**
|
|
42
|
-
*
|
|
43
|
-
* {@apilink RequestQueue} functions as well as {@apilink enqueueLinks}.
|
|
42
|
+
* @internal
|
|
44
43
|
*/
|
|
45
|
-
|
|
44
|
+
interface RequestQueueOperationInfo extends QueueOperationInfo {
|
|
46
45
|
/** Indicates if request was already present in the queue. */
|
|
47
46
|
wasAlreadyPresent: boolean;
|
|
48
47
|
/** Indicates if request was already marked as handled. */
|
|
@@ -135,7 +134,9 @@ export declare class RequestQueue {
|
|
|
135
134
|
recentlyHandled: LruCache<any>;
|
|
136
135
|
assumedTotalCount: number;
|
|
137
136
|
assumedHandledCount: number;
|
|
138
|
-
requestsCache: LruCache<
|
|
137
|
+
requestsCache: LruCache<{
|
|
138
|
+
uniqueKey: string;
|
|
139
|
+
wasAlreadyHandled: boolean;
|
|
139
140
|
isHandled: boolean;
|
|
140
141
|
id: string;
|
|
141
142
|
}>;
|
|
@@ -161,7 +162,7 @@ export declare class RequestQueue {
|
|
|
161
162
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
|
|
162
163
|
* @param [options] Request queue operation options.
|
|
163
164
|
*/
|
|
164
|
-
addRequest(requestLike: Request | RequestOptions, options?: RequestQueueOperationOptions): Promise<
|
|
165
|
+
addRequest(requestLike: Request | RequestOptions, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
|
|
165
166
|
/**
|
|
166
167
|
* Adds requests to the queue in batches of 25.
|
|
167
168
|
*
|
|
@@ -205,14 +206,14 @@ export declare class RequestQueue {
|
|
|
205
206
|
* function as handled after successful processing.
|
|
206
207
|
* Handled requests will never again be returned by the `fetchNextRequest` function.
|
|
207
208
|
*/
|
|
208
|
-
markRequestHandled(request: Request): Promise<
|
|
209
|
+
markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
|
|
209
210
|
/**
|
|
210
211
|
* Reclaims a failed request back to the queue, so that it can be returned for processing later again
|
|
211
212
|
* by another call to {@apilink RequestQueue.fetchNextRequest}.
|
|
212
213
|
* The request record in the queue is updated using the provided `request` parameter.
|
|
213
214
|
* For example, this lets you store the number of retries or error messages for the request.
|
|
214
215
|
*/
|
|
215
|
-
reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<
|
|
216
|
+
reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
|
|
216
217
|
/**
|
|
217
218
|
* Resolves to `true` if the next call to {@apilink RequestQueue.fetchNextRequest}
|
|
218
219
|
* would return `null`, otherwise it resolves to `false`.
|
|
@@ -231,7 +232,7 @@ export declare class RequestQueue {
|
|
|
231
232
|
/**
|
|
232
233
|
* Caches information about request to beware of unneeded addRequest() calls.
|
|
233
234
|
*/
|
|
234
|
-
protected _cacheRequest(cacheKey: string, queueOperationInfo:
|
|
235
|
+
protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
|
|
235
236
|
/**
|
|
236
237
|
* We always request more items than is in progress to ensure that something falls into head.
|
|
237
238
|
*
|
|
@@ -311,8 +312,5 @@ export interface RequestQueueOptions {
|
|
|
311
312
|
name?: string;
|
|
312
313
|
client: StorageClient;
|
|
313
314
|
}
|
|
314
|
-
export
|
|
315
|
-
requestId: string;
|
|
316
|
-
wasAlreadyHandled: boolean;
|
|
317
|
-
}
|
|
315
|
+
export {};
|
|
318
316
|
//# sourceMappingURL=request_queue.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAKjE,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAKjE,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAKjD;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAS7C;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAQ7C;AAED;;GAEG;AACH,UAAU,yBAA0B,SAAQ,kBAAkB;IAE1D,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,SAAS,EAAE,MAAM,CAAC;CAErB;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,qBAAa,YAAY;IAmDqB,QAAQ,CAAC,MAAM;IAlDzD,GAAG,+BAAyC;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,OAAO,CAAC,aAAa,CAAgC;IACrD,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAC5B,eAAe,EAAE,OAAO,CAAC;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,EAAE,IAAI,CAAC;QACtB,cAAc,EAAE,IAAI,CAAC;QACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAChC,CAAC,GAAG,IAAI,CAAQ;IAIjB,UAAU,eAAa;IAKvB,YAAY,OAAc;IAC1B,qBAAqB,SAAY;IAKjC,eAAe,gBAA4D;IAI3E,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAIxB,aAAa;mBACI,MAAM;2BAAqB,OAAO;mBAAa,OAAO;YAAM,MAAM;OAC7C;IAEtC;;OAEG;gBACS,OAAO,EAAE,mBAAmB,EAAW,MAAM,gBAAkC;IAS3F;;OAEG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CAAC,WAAW,EAAE,OAAO,GAAG,cAAc,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,yBAAyB,CAAC;IA+CvI;;;;;;;;;;OAUG;IACG,WAAW,CACb,YAAY,EAAE,CAAC,OAAO,GAAG,cAAc,CAAC,EAAE,EAC1C,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAsFlC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAS3F;;;;;;;;;;;;;;;;OAgBG;IACG,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IA0DvF;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IA8BrF;;;;;OAKG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwC7H;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAapC,OAAO,CAAC,MAAM;IAWd;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAS9F;;;;;;;;;;OAUG;cACa,qBAAqB,CACjC,iBAAiB,UAAQ,EACzB,KAAK,SAA8E,EACnF,SAAS,SAAI,GACd,OAAO,CAAC,OAAO,CAAC;IAmFnB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAQnC;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;OAQG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAItD;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,YAAY,CAAC;CAW/G;AAED,MAAM,WAAW,mBAAmB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB"}
|