@crawlee/core 3.7.4-beta.30 → 3.7.4-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/crawler_commons.d.ts +67 -6
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +185 -0
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/statistics.d.ts +2 -2
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +5 -5
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/shared.d.ts +3 -3
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js.map +1 -1
- package/index.mjs +1 -0
- package/package.json +5 -5
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import type { Dictionary, BatchAddRequestsResult } from '@crawlee/types';
|
|
2
2
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
3
3
|
import type { Response as GotResponse, OptionsInit } from 'got-scraping';
|
|
4
|
+
import type { ReadonlyDeep } from 'type-fest';
|
|
5
|
+
import type { Configuration } from '../configuration';
|
|
4
6
|
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links';
|
|
5
7
|
import type { Log } from '../log';
|
|
6
8
|
import type { ProxyInfo } from '../proxy_configuration';
|
|
7
9
|
import type { Request, Source } from '../request';
|
|
8
10
|
import type { Session } from '../session_pool/session';
|
|
9
|
-
import type { RequestQueueOperationOptions, Dataset,
|
|
10
|
-
|
|
11
|
+
import type { RequestQueueOperationOptions, Dataset, RecordOptions } from '../storages';
|
|
12
|
+
import { KeyValueStore } from '../storages';
|
|
13
|
+
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string, unknown> {
|
|
11
14
|
/**
|
|
12
15
|
* The original {@apilink Request} object.
|
|
13
16
|
*/
|
|
@@ -19,7 +22,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
|
|
|
19
22
|
*
|
|
20
23
|
* @param [data] Data to be pushed to the default dataset.
|
|
21
24
|
*/
|
|
22
|
-
pushData(data: Parameters<Dataset['pushData']>[0]
|
|
25
|
+
pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdOrName?: string): Promise<void>;
|
|
23
26
|
/**
|
|
24
27
|
* This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
|
|
25
28
|
* currently used by the crawler.
|
|
@@ -44,14 +47,14 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
|
|
|
44
47
|
*
|
|
45
48
|
* @param [options] All `enqueueLinks()` parameters are passed via an options object.
|
|
46
49
|
*/
|
|
47
|
-
enqueueLinks: (options?: Omit<EnqueueLinksOptions, 'requestQueue'
|
|
50
|
+
enqueueLinks: (options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>>) => Promise<unknown>;
|
|
48
51
|
/**
|
|
49
52
|
* Add requests directly to the request queue.
|
|
50
53
|
*
|
|
51
54
|
* @param requests The requests to add
|
|
52
55
|
* @param options Options for the request queue
|
|
53
56
|
*/
|
|
54
|
-
addRequests: (requestsLike: (string | Source)[]
|
|
57
|
+
addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
|
|
55
58
|
/**
|
|
56
59
|
* Returns the state - a piece of mutable persistent data shared across all the request handler runs.
|
|
57
60
|
*/
|
|
@@ -99,7 +102,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
|
|
|
99
102
|
* @param [options] All `enqueueLinks()` parameters are passed via an options object.
|
|
100
103
|
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
|
|
101
104
|
*/
|
|
102
|
-
enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
105
|
+
enqueueLinks(options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>): Promise<BatchAddRequestsResult>;
|
|
103
106
|
/**
|
|
104
107
|
* Get a key-value store with given name or id, or the default one for the crawler.
|
|
105
108
|
*/
|
|
@@ -123,4 +126,62 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
|
|
|
123
126
|
*/
|
|
124
127
|
sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
|
|
125
128
|
}
|
|
129
|
+
/**
|
|
130
|
+
* A partial implementation of {@apilink RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
|
|
131
|
+
*
|
|
132
|
+
* @experimental
|
|
133
|
+
*/
|
|
134
|
+
export declare class RequestHandlerResult {
|
|
135
|
+
private config;
|
|
136
|
+
private crawleeStateKey;
|
|
137
|
+
private _keyValueStoreChanges;
|
|
138
|
+
private pushDataCalls;
|
|
139
|
+
private addRequestsCalls;
|
|
140
|
+
private enqueueLinksCalls;
|
|
141
|
+
constructor(config: Configuration, crawleeStateKey: string);
|
|
142
|
+
/**
|
|
143
|
+
* A record of calls to {@apilink RestrictedCrawlingContext.pushData}, {@apilink RestrictedCrawlingContext.addRequests}, {@apilink RestrictedCrawlingContext.enqueueLinks} made by a request handler.
|
|
144
|
+
*/
|
|
145
|
+
get calls(): ReadonlyDeep<{
|
|
146
|
+
pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
|
|
147
|
+
addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
|
|
148
|
+
enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
|
|
149
|
+
}>;
|
|
150
|
+
/**
|
|
151
|
+
* A record of changes made to key-value stores by a request handler.
|
|
152
|
+
*/
|
|
153
|
+
get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
|
|
154
|
+
changedValue: unknown;
|
|
155
|
+
options?: RecordOptions;
|
|
156
|
+
}>>>;
|
|
157
|
+
/**
|
|
158
|
+
* Items added to datasets by a request handler.
|
|
159
|
+
*/
|
|
160
|
+
get datasetItems(): ReadonlyDeep<{
|
|
161
|
+
item: Dictionary;
|
|
162
|
+
datasetIdOrName?: string;
|
|
163
|
+
}[]>;
|
|
164
|
+
/**
|
|
165
|
+
* URLs enqueued to the request queue by a request handler, either via {@apilink RestrictedCrawlingContext.addRequests} or {@apilink RestrictedCrawlingContext.enqueueLinks}
|
|
166
|
+
*/
|
|
167
|
+
get enqueuedUrls(): ReadonlyDeep<{
|
|
168
|
+
url: string;
|
|
169
|
+
label?: string;
|
|
170
|
+
}[]>;
|
|
171
|
+
/**
|
|
172
|
+
* URL lists enqueued to the request queue by a request handler via {@apilink RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
|
|
173
|
+
*/
|
|
174
|
+
get enqueuedUrlLists(): ReadonlyDeep<{
|
|
175
|
+
listUrl: string;
|
|
176
|
+
label?: string;
|
|
177
|
+
}[]>;
|
|
178
|
+
pushData: RestrictedCrawlingContext['pushData'];
|
|
179
|
+
enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
|
|
180
|
+
addRequests: RestrictedCrawlingContext['addRequests'];
|
|
181
|
+
useState: RestrictedCrawlingContext['useState'];
|
|
182
|
+
getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
|
|
183
|
+
private idOrDefault;
|
|
184
|
+
private getKeyValueStoreChangedValue;
|
|
185
|
+
private setKeyValueStoreChangedValue;
|
|
186
|
+
}
|
|
126
187
|
//# sourceMappingURL=crawler_commons.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,QAAQ,IAAI,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,QAAQ,IAAI,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,4BAA4B,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAChH;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtG;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,CAAC,CAAC,CAAC;IAErI;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IACrI,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAC9G,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,QAAQ,GAAG,MAAM,EAAE,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1G;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IAMjB,OAAO,CAAC,MAAM;IAAiB,OAAO,CAAC,eAAe;IALlE,OAAO,CAAC,qBAAqB,CAA0F;IACvH,OAAO,CAAC,aAAa,CAA2D;IAChF,OAAO,CAAC,gBAAgB,CAA8D;IACtF,OAAO,CAAC,iBAAiB,CAA+D;gBAEpE,MAAM,EAAE,aAAa,EAAU,eAAe,EAAE,MAAM;IAE1E;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;QACpE,YAAY,EAAE,UAAU,CAAC,yBAAyB,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC,CAED;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CAAC,CAE3H;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAEjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAkBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAG,MAAM,CAAA;KAAE,EAAE,CAAC,CAY3E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,YAAY,EAAE,yBAAyB,CAAC,cAAc,CAAC,CAErD;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAoB7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CAIlC;CACL"}
|
|
@@ -1,3 +1,188 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RequestHandlerResult = void 0;
|
|
4
|
+
const storages_1 = require("../storages");
|
|
5
|
+
/**
|
|
6
|
+
* A partial implementation of {@apilink RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
|
|
7
|
+
*
|
|
8
|
+
* @experimental
|
|
9
|
+
*/
|
|
10
|
+
class RequestHandlerResult {
|
|
11
|
+
constructor(config, crawleeStateKey) {
|
|
12
|
+
Object.defineProperty(this, "config", {
|
|
13
|
+
enumerable: true,
|
|
14
|
+
configurable: true,
|
|
15
|
+
writable: true,
|
|
16
|
+
value: config
|
|
17
|
+
});
|
|
18
|
+
Object.defineProperty(this, "crawleeStateKey", {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
configurable: true,
|
|
21
|
+
writable: true,
|
|
22
|
+
value: crawleeStateKey
|
|
23
|
+
});
|
|
24
|
+
Object.defineProperty(this, "_keyValueStoreChanges", {
|
|
25
|
+
enumerable: true,
|
|
26
|
+
configurable: true,
|
|
27
|
+
writable: true,
|
|
28
|
+
value: {}
|
|
29
|
+
});
|
|
30
|
+
Object.defineProperty(this, "pushDataCalls", {
|
|
31
|
+
enumerable: true,
|
|
32
|
+
configurable: true,
|
|
33
|
+
writable: true,
|
|
34
|
+
value: []
|
|
35
|
+
});
|
|
36
|
+
Object.defineProperty(this, "addRequestsCalls", {
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
writable: true,
|
|
40
|
+
value: []
|
|
41
|
+
});
|
|
42
|
+
Object.defineProperty(this, "enqueueLinksCalls", {
|
|
43
|
+
enumerable: true,
|
|
44
|
+
configurable: true,
|
|
45
|
+
writable: true,
|
|
46
|
+
value: []
|
|
47
|
+
});
|
|
48
|
+
Object.defineProperty(this, "pushData", {
|
|
49
|
+
enumerable: true,
|
|
50
|
+
configurable: true,
|
|
51
|
+
writable: true,
|
|
52
|
+
value: async (data, datasetIdOrName) => {
|
|
53
|
+
this.pushDataCalls.push([data, datasetIdOrName]);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
Object.defineProperty(this, "enqueueLinks", {
|
|
57
|
+
enumerable: true,
|
|
58
|
+
configurable: true,
|
|
59
|
+
writable: true,
|
|
60
|
+
value: async (options) => {
|
|
61
|
+
this.enqueueLinksCalls.push([options]);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
Object.defineProperty(this, "addRequests", {
|
|
65
|
+
enumerable: true,
|
|
66
|
+
configurable: true,
|
|
67
|
+
writable: true,
|
|
68
|
+
value: async (requests, options = {}) => {
|
|
69
|
+
this.addRequestsCalls.push([requests, options]);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
Object.defineProperty(this, "useState", {
|
|
73
|
+
enumerable: true,
|
|
74
|
+
configurable: true,
|
|
75
|
+
writable: true,
|
|
76
|
+
value: async (defaultValue) => {
|
|
77
|
+
const store = await this.getKeyValueStore(undefined);
|
|
78
|
+
return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
Object.defineProperty(this, "getKeyValueStore", {
|
|
82
|
+
enumerable: true,
|
|
83
|
+
configurable: true,
|
|
84
|
+
writable: true,
|
|
85
|
+
value: async (idOrName) => {
|
|
86
|
+
const store = await storages_1.KeyValueStore.open(idOrName, { config: this.config });
|
|
87
|
+
return {
|
|
88
|
+
id: this.idOrDefault(idOrName),
|
|
89
|
+
name: idOrName,
|
|
90
|
+
getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? await store.getValue(key),
|
|
91
|
+
getAutoSavedValue: async (key, defaultValue = {}) => {
|
|
92
|
+
let value = this.getKeyValueStoreChangedValue(idOrName, key);
|
|
93
|
+
if (value === null) {
|
|
94
|
+
value = await store.getValue(key) ?? defaultValue;
|
|
95
|
+
this.setKeyValueStoreChangedValue(idOrName, key, value);
|
|
96
|
+
}
|
|
97
|
+
return value;
|
|
98
|
+
},
|
|
99
|
+
setValue: async (key, value, options) => {
|
|
100
|
+
this.setKeyValueStoreChangedValue(idOrName, key, value, options);
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
Object.defineProperty(this, "idOrDefault", {
|
|
106
|
+
enumerable: true,
|
|
107
|
+
configurable: true,
|
|
108
|
+
writable: true,
|
|
109
|
+
value: (idOrName) => idOrName ?? this.config.get('defaultKeyValueStoreId')
|
|
110
|
+
});
|
|
111
|
+
Object.defineProperty(this, "getKeyValueStoreChangedValue", {
|
|
112
|
+
enumerable: true,
|
|
113
|
+
configurable: true,
|
|
114
|
+
writable: true,
|
|
115
|
+
value: (idOrName, key) => {
|
|
116
|
+
var _a;
|
|
117
|
+
const id = this.idOrDefault(idOrName);
|
|
118
|
+
(_a = this._keyValueStoreChanges)[id] ?? (_a[id] = {});
|
|
119
|
+
return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
Object.defineProperty(this, "setKeyValueStoreChangedValue", {
|
|
123
|
+
enumerable: true,
|
|
124
|
+
configurable: true,
|
|
125
|
+
writable: true,
|
|
126
|
+
value: (idOrName, key, changedValue, options) => {
|
|
127
|
+
var _a;
|
|
128
|
+
const id = this.idOrDefault(idOrName);
|
|
129
|
+
(_a = this._keyValueStoreChanges)[id] ?? (_a[id] = {});
|
|
130
|
+
this._keyValueStoreChanges[id][key] = { changedValue, options };
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* A record of calls to {@apilink RestrictedCrawlingContext.pushData}, {@apilink RestrictedCrawlingContext.addRequests}, {@apilink RestrictedCrawlingContext.enqueueLinks} made by a request handler.
|
|
136
|
+
*/
|
|
137
|
+
get calls() {
|
|
138
|
+
return { pushData: this.pushDataCalls, addRequests: this.addRequestsCalls, enqueueLinks: this.enqueueLinksCalls };
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* A record of changes made to key-value stores by a request handler.
|
|
142
|
+
*/
|
|
143
|
+
get keyValueStoreChanges() {
|
|
144
|
+
return this._keyValueStoreChanges;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Items added to datasets by a request handler.
|
|
148
|
+
*/
|
|
149
|
+
get datasetItems() {
|
|
150
|
+
return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* URLs enqueued to the request queue by a request handler, either via {@apilink RestrictedCrawlingContext.addRequests} or {@apilink RestrictedCrawlingContext.enqueueLinks}
|
|
154
|
+
*/
|
|
155
|
+
get enqueuedUrls() {
|
|
156
|
+
const result = [];
|
|
157
|
+
for (const [options] of this.enqueueLinksCalls) {
|
|
158
|
+
result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
|
|
159
|
+
}
|
|
160
|
+
for (const [requests] of this.addRequestsCalls) {
|
|
161
|
+
for (const request of requests) {
|
|
162
|
+
if (typeof request === 'object' && (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) && request.url !== undefined) {
|
|
163
|
+
result.push({ url: request.url, label: request.label });
|
|
164
|
+
}
|
|
165
|
+
else if (typeof request === 'string') {
|
|
166
|
+
result.push({ url: request });
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* URL lists enqueued to the request queue by a request handler via {@apilink RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
|
|
174
|
+
*/
|
|
175
|
+
get enqueuedUrlLists() {
|
|
176
|
+
const result = [];
|
|
177
|
+
for (const [requests] of this.addRequestsCalls) {
|
|
178
|
+
for (const request of requests) {
|
|
179
|
+
if (typeof request === 'object' && 'requestsFromUrl' in request && request.requestsFromUrl !== undefined) {
|
|
180
|
+
result.push({ listUrl: request.requestsFromUrl, label: request.label });
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return result;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
exports.RequestHandlerResult = RequestHandlerResult;
|
|
3
188
|
//# sourceMappingURL=crawler_commons.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":";;;AAYA,0CAA4C;AAwI5C;;;;GAIG;AACH,MAAa,oBAAoB;IAM7B,YAAoB,MAAqB,EAAU,eAAuB;QAA9D;;;;mBAAQ,MAAM;WAAe;QAAE;;;;mBAAQ,eAAe;WAAQ;QALlE;;;;mBAA4G,EAAE;WAAC;QAC/G;;;;mBAAqE,EAAE;WAAC;QACxE;;;;mBAA2E,EAAE;WAAC;QAC9E;;;;mBAA6E,EAAE;WAAC;QAqExF;;;;mBAAkD,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;gBAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;YACrD,CAAC;WAAC;QAEF;;;;mBAA0D,KAAK,EAAE,OAAO,EAAE,EAAE;gBACxE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,CAAC;WAAC;QAEF;;;;mBAAwD,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;gBACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YACpD,CAAC;WAAC;QAEF;;;;mBAAkD,KAAK,EAAE,YAAY,EAAE,EAAE;gBACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;gBACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YAC7E,CAAC;WAAC;QAEF;;;;mBAAkE,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACjF,MAAM,KAAK,GAAG,MAAM,wBAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE1E,OAAO;oBACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBAC9B,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;oBACtG,iBAAiB,EAAE,KAAK,EAAqC,GAAW,EAAE,eAAkB,EAAO,EAAE,EAAE;wBACnG,IAAI,KAAK,GAAG,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;wBAC7D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BACjB,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC;4BAClD,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;wBAC5D,CAAC;wBAED,OAAO,KAAU,CAAC;oBACtB,CAAC;oBACD,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;oBACrE,CAAC;iBACJ,CAAC;YACN,CAAC;WAAC;QAEM;;;;mBAAc,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC;WAAC;QAEnG;;;;mBAA+B,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;;gBACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAA,IAAI,CAAC,qBAAqB,EAAC,EAAE,SAAF,EAAE,IAAM,EAAE,EAAC;gBACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACpE,CAAC;WAAC;QAEM;;;;mBAA+B,CAAC,QAA4B,EAAE,GAAW,EAAE,YAAqB,EAAE,OAAuB,EAAE,EAAE;;gBACjI,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAA,IAAI,CAAC,qBAAqB,EAAC,EAAE,SAAF,EAAE,IAAM,EAAE,EAAC;gBACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;YACpE,CAAC;WAAC;IAtH2E,CAAC;IAE9E;;OAEG;IACH,IAAI,KAAK;QAKL,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,gBAAgB,EAAE,YAAY,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IACtH,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;IACrJ,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAqC,EAAE,CAAC;QAEpD,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;oBACzI,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAAyC,EAAE,CAAC;QAExD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,iBAAiB,IAAI,OAAO,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;oBACvG,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CAsDJ;AA7HD,oDA6HC"}
|
package/crawlers/statistics.d.ts
CHANGED
|
@@ -58,8 +58,8 @@ export declare class Statistics {
|
|
|
58
58
|
* Contains the associated Configuration instance
|
|
59
59
|
*/
|
|
60
60
|
private readonly config;
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
protected keyValueStore?: KeyValueStore;
|
|
62
|
+
protected persistStateKey: string;
|
|
63
63
|
private logIntervalMillis;
|
|
64
64
|
private logMessage;
|
|
65
65
|
private listener;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D;;GAEG;AACH,cAAM,GAAG;IACL,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,cAAc,CAAC,CAAS;IAEhC,GAAG;IAKH,MAAM;IAKN,UAAU;CAGb;AAWD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,eAAwC;IAEpD;;OAEG;IACH,iBAAiB,eAAwC;IAEzD;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAmB;IAE9B;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D;;GAEG;AACH,cAAM,GAAG;IACL,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,cAAc,CAAC,CAAS;IAEhC,GAAG;IAKH,MAAM;IAKN,UAAU;CAGb;AAWD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,eAAwC;IAEpD;;OAEG;IACH,iBAAiB,eAAwC;IAEzD;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAmB;IAE9B;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAa;IACpD,SAAS,CAAC,eAAe,SAAuC;IAChE,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA8C;IAClE,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,kBAAkB,CAAqB;IAE/C;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IA+B3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAY7C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAY7B;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAS3B;;OAEG;IACH,SAAS;;;;;;;;;IAqBT;;;OAGG;IACG,cAAc;IAoBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG;IAQxC;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAe/C;;OAEG;cACa,oBAAoB;IA0CpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAU3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CAyBpC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,0CAA8C;AAC9C,oDAAoB;AAEpB,oDAAiD;AAGjD,gCAA2C;AAC3C,iEAA4D;AAE5D;;GAEG;AACH,MAAM,GAAG;IAAT;QACY;;;;mBAA2B,IAAI;WAAC;QAChC;;;;mBAAO,CAAC;WAAC;QACT;;;;;WAAwB;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6CnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7C3C;;WAEG;QACH;;;;mBAAe,IAAI,oBAAY,CAAC,kBAAkB,CAAC;WAAC;QAEpD;;WAEG;QACH;;;;mBAAoB,IAAI,oBAAY,CAAC,kBAAkB,CAAC;WAAC;QAEzD;;WAEG;QACM;;;;mBAAK,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACM;;;;mBAAkC,EAAE;WAAC;QAE9C;;WAEG;QACc;;;;;WAAsB;
|
|
1
|
+
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,0CAA8C;AAC9C,oDAAoB;AAEpB,oDAAiD;AAGjD,gCAA2C;AAC3C,iEAA4D;AAE5D;;GAEG;AACH,MAAM,GAAG;IAAT;QACY;;;;mBAA2B,IAAI;WAAC;QAChC;;;;mBAAO,CAAC;WAAC;QACT;;;;;WAAwB;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6CnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7C3C;;WAEG;QACH;;;;mBAAe,IAAI,oBAAY,CAAC,kBAAkB,CAAC;WAAC;QAEpD;;WAEG;QACH;;;;mBAAoB,IAAI,oBAAY,CAAC,kBAAkB,CAAC;WAAC;QAEzD;;WAEG;QACM;;;;mBAAK,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACM;;;;mBAAkC,EAAE;WAAC;QAE9C;;WAEG;QACc;;;;;WAAsB;QAE7B;;;;mBAAgC,SAAS;WAAC;QAC1C;;;;mBAAkB,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QACxD;;;;;WAA0B;QAC1B;;;;;WAAmB;QACnB;;;;;WAA8B;QAC9B;;;;mBAAqB,IAAI,GAAG,EAAwB;WAAC;QAC5C;;;;mBAAM,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;WAAC;QAC1D;;;;;WAAuB;QACvB;;;;;WAAqB;QACrB;;;;;WAAqB;QACrB;;;;;WAAuC;QAM3C,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,GACJ,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAAE,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YAC/G,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAlB,IAAI,CAAC,aAAa,GAAK,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAC;QAE/E,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,GAAQ;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC7G,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC;;AA7VL,gCA8VC;AA7VkB;;;;WAAK,CAAC;EAAJ,CAAK"}
|
|
@@ -6,7 +6,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
6
6
|
/** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
|
|
7
7
|
limit?: number;
|
|
8
8
|
/** An array of URLs to enqueue. */
|
|
9
|
-
urls?: string[]
|
|
9
|
+
urls?: Readonly<string[]>;
|
|
10
10
|
/** A request queue to which the URLs will be enqueued. */
|
|
11
11
|
requestQueue?: RequestProvider;
|
|
12
12
|
/** A CSS selector matching links to be enqueued. */
|
|
@@ -38,7 +38,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
38
38
|
* If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
|
|
39
39
|
* enqueues the links with the same subdomain.
|
|
40
40
|
*/
|
|
41
|
-
globs?: GlobInput[]
|
|
41
|
+
globs?: Readonly<GlobInput[]>;
|
|
42
42
|
/**
|
|
43
43
|
* An array of glob pattern strings, regexp patterns or plain objects
|
|
44
44
|
* containing patterns matching URLs that will **never** be enqueued.
|
|
@@ -49,7 +49,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
49
49
|
* Glob matching is always case-insensitive.
|
|
50
50
|
* If you need case-sensitive matching, provide a regexp.
|
|
51
51
|
*/
|
|
52
|
-
exclude?: (GlobInput | RegExpInput)[]
|
|
52
|
+
exclude?: Readonly<(GlobInput | RegExpInput)[]>;
|
|
53
53
|
/**
|
|
54
54
|
* An array of regular expressions or plain objects
|
|
55
55
|
* containing regular expressions matching the URLs to be enqueued.
|
|
@@ -60,7 +60,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
60
60
|
* If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
|
|
61
61
|
* enqueues the links with the same subdomain.
|
|
62
62
|
*/
|
|
63
|
-
regexps?: RegExpInput[]
|
|
63
|
+
regexps?: Readonly<RegExpInput[]>;
|
|
64
64
|
/**
|
|
65
65
|
* *NOTE:* In future versions of SDK the options will be removed.
|
|
66
66
|
* Please use `globs` or `regexps` instead.
|
|
@@ -79,7 +79,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
79
79
|
*
|
|
80
80
|
* @deprecated prefer using `globs` or `regexps` instead
|
|
81
81
|
*/
|
|
82
|
-
pseudoUrls?: PseudoUrlInput[]
|
|
82
|
+
pseudoUrls?: Readonly<PseudoUrlInput[]>;
|
|
83
83
|
/**
|
|
84
84
|
* Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
|
|
85
85
|
* to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAU3G,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAU3G,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE1B,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;IAE9B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;IAEhD;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAElC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;CACxF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAqJtI;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -27,13 +27,13 @@ export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpI
|
|
|
27
27
|
* to construct RegExps from PseudoUrl strings.
|
|
28
28
|
* @ignore
|
|
29
29
|
*/
|
|
30
|
-
export declare function constructRegExpObjectsFromPseudoUrls(pseudoUrls: PseudoUrlInput[]): RegExpObject[];
|
|
30
|
+
export declare function constructRegExpObjectsFromPseudoUrls(pseudoUrls: Readonly<PseudoUrlInput[]>): RegExpObject[];
|
|
31
31
|
/**
|
|
32
32
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
33
33
|
* to construct Glob objects from Glob pattern strings.
|
|
34
34
|
* @ignore
|
|
35
35
|
*/
|
|
36
|
-
export declare function constructGlobObjectsFromGlobs(globs: GlobInput[]): GlobObject[];
|
|
36
|
+
export declare function constructGlobObjectsFromGlobs(globs: Readonly<GlobInput[]>): GlobObject[];
|
|
37
37
|
/**
|
|
38
38
|
* @internal
|
|
39
39
|
*/
|
|
@@ -43,7 +43,7 @@ export declare function validateGlobPattern(glob: string): string;
|
|
|
43
43
|
* to check RegExps input and return valid RegExps.
|
|
44
44
|
* @ignore
|
|
45
45
|
*/
|
|
46
|
-
export declare function constructRegExpObjectsFromRegExps(regexps: RegExpInput[]): RegExpObject[];
|
|
46
|
+
export declare function constructRegExpObjectsFromRegExps(regexps: Readonly<RegExpInput[]>): RegExpObject[];
|
|
47
47
|
/**
|
|
48
48
|
* @ignore
|
|
49
49
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE/H,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE1H,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE9H,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE/H,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE1H,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE9H,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,GAAG,YAAY,EAAE,CAiB3G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,GAAG,UAAU,EAAE,CAkCxF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,GAAG,YAAY,EAAE,CAgBlG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC1B,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAC3C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,EACtC,qBAAqB,GAAE,gBAAgB,EAAO,EAC9C,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAC3C,OAAO,EAAE,CAmCX;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAuBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EAC7C,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":";;;AAAA,6BAA0B;AAE1B,kDAAiD;AACjD,yCAAsC;AAItC,wCAAqC;AAErC,wCAAgD;AAAvC,uGAAA,cAAc,OAAA;AAEvB,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAmB3C;;GAEG;AACH,SAAgB,8BAA8B,CAAC,IAA8C,EAAE,OAAkC;IAC7H,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAND,wEAMC;AAED;;;;GAIG;AACH,SAAgB,oCAAoC,CAAC,
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":";;;AAAA,6BAA0B;AAE1B,kDAAiD;AACjD,yCAAsC;AAItC,wCAAqC;AAErC,wCAAgD;AAAvC,uGAAA,cAAc,OAAA;AAEvB,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAmB3C;;GAEG;AACH,SAAgB,8BAA8B,CAAC,IAA8C,EAAE,OAAkC;IAC7H,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAND,wEAMC;AAED;;;;GAIG;AACH,SAAgB,oCAAoC,CAAC,UAAsC;IACvF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAjBD,oFAiBC;AAED;;;;GAIG;AACH,SAAgB,6BAA6B,CAAC,KAA4B;IACtE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACd,8BAA8B;QAC1B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAlCD,sEAkCC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAC5H,OAAO,WAAW,CAAC;AACvB,CAAC;AAJD,kDAIC;AAED;;;;GAIG;AACH,SAAgB,iCAAiC,CAAC,OAAgC;IAC9E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAhBD,8EAgBC;AAED;;GAEG;AACH,SAAgB,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C;IAE1C,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,EAAE;YACxD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC;YAC9C,OAAO,CACC,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,sBAAsB;gBACvD,CAAC,IAAI,IAAI,IAAA,qBAAS,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CACvD,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,IAAI,iBAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IACI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,sBAAsB;gBACvD,CAAC,IAAI,IAAI,IAAA,qBAAS,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAClD,CAAC;gBACC,MAAM,OAAO,GAAG,OAAO,IAAI,KAAK,QAAQ;oBACpC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAEtE,OAAO,IAAI,iBAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAxCD,wCAwCC;AAED,SAAgB,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE,CAAC;YACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IACI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,sBAAsB;gBAC/D,CAAC,IAAI,IAAI,IAAA,qBAAS,EAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC1D,CAAC;gBACC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAvBD,4DAuBC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAChC,OAA6C,EAC7C,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CACA,CAAC,GAAG,EAAE,EAAE,CAAC,CACL,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAoB,CAC9I,CACJ;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,SAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,SAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAvB,cAAc,CAAC,QAAQ,GAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,EAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAlCD,oDAkCC"}
|
package/index.mjs
CHANGED
|
@@ -32,6 +32,7 @@ export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
|
|
|
32
32
|
export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
|
|
33
33
|
export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
34
34
|
export const Request = mod.Request;
|
|
35
|
+
export const RequestHandlerResult = mod.RequestHandlerResult;
|
|
35
36
|
export const RequestList = mod.RequestList;
|
|
36
37
|
export const RequestProvider = mod.RequestProvider;
|
|
37
38
|
export const RequestQueue = mod.RequestQueue;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.7.4-beta.
|
|
3
|
+
"version": "3.7.4-beta.31",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.7.4-beta.
|
|
63
|
-
"@crawlee/types": "3.7.4-beta.
|
|
64
|
-
"@crawlee/utils": "3.7.4-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.7.4-beta.31",
|
|
63
|
+
"@crawlee/types": "3.7.4-beta.31",
|
|
64
|
+
"@crawlee/utils": "3.7.4-beta.31",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@types/tough-cookie": "^4.0.2",
|
|
67
67
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
@@ -85,5 +85,5 @@
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
},
|
|
88
|
-
"gitHead": "
|
|
88
|
+
"gitHead": "d745f4a79600b2c3c641187aeebb6fdad22f7788"
|
|
89
89
|
}
|