@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +3 -21
- package/autoscaling/autoscaled_pool.js +105 -97
- package/autoscaling/concurrency_system.d.ts +7 -22
- package/autoscaling/concurrency_system.js +112 -100
- package/autoscaling/cpu_load_signal.d.ts +1 -2
- package/autoscaling/cpu_load_signal.js +11 -10
- package/autoscaling/event_loop_load_signal.d.ts +1 -4
- package/autoscaling/event_loop_load_signal.js +18 -18
- package/autoscaling/index.d.ts +1 -1
- package/autoscaling/index.js +1 -1
- package/autoscaling/load_signal.d.ts +8 -7
- package/autoscaling/load_signal.js +14 -12
- package/autoscaling/memory_load_signal.d.ts +3 -12
- package/autoscaling/memory_load_signal.js +41 -41
- package/autoscaling/snapshotter.d.ts +7 -10
- package/autoscaling/snapshotter.js +18 -18
- package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts} +14 -17
- package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js} +27 -27
- package/autoscaling/system_status.d.ts +9 -11
- package/autoscaling/system_status.js +13 -13
- package/configuration.d.ts +16 -16
- package/configuration.js +6 -6
- package/crawlers/context_pipeline.js +6 -6
- package/crawlers/crawler_commons.d.ts +14 -112
- package/crawlers/crawler_commons.js +1 -107
- package/crawlers/error_snapshotter.d.ts +2 -4
- package/crawlers/error_snapshotter.js +5 -5
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.js +0 -1
- package/crawlers/statistics.d.ts +164 -45
- package/crawlers/statistics.js +339 -153
- package/debug.js +4 -4
- package/enqueue_links/enqueue_links.d.ts +51 -159
- package/enqueue_links/enqueue_links.js +38 -231
- package/enqueue_links/shared.d.ts +34 -24
- package/enqueue_links/shared.js +54 -36
- package/errors.d.ts +43 -12
- package/errors.js +48 -9
- package/events/event_manager.d.ts +2 -2
- package/events/event_manager.js +7 -6
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +5 -5
- package/http.d.ts +9 -0
- package/http.js +28 -0
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/log.js +5 -1
- package/memory-storage/memory-storage.d.ts +3 -11
- package/memory-storage/memory-storage.js +42 -48
- package/memory-storage/resource-clients/dataset.d.ts +1 -1
- package/memory-storage/resource-clients/dataset.js +8 -13
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
- package/memory-storage/resource-clients/key-value-store.js +36 -38
- package/memory-storage/resource-clients/request-queue.d.ts +4 -23
- package/memory-storage/resource-clients/request-queue.js +68 -80
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +8 -9
- package/proxy_configuration.d.ts +1 -3
- package/proxy_configuration.js +18 -14
- package/recoverable_state.d.ts +80 -45
- package/recoverable_state.js +163 -75
- package/request.d.ts +3 -4
- package/request.js +57 -44
- package/router.d.ts +6 -9
- package/router.js +23 -23
- package/serialization.js +14 -13
- package/service_locator.d.ts +19 -10
- package/service_locator.js +57 -48
- package/session_pool/session.d.ts +3 -14
- package/session_pool/session.js +76 -74
- package/session_pool/session_pool.d.ts +6 -12
- package/session_pool/session_pool.js +85 -77
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +12 -1
- package/storages/dataset.js +133 -32
- package/storages/index.d.ts +2 -1
- package/storages/index.js +2 -1
- package/storages/key_value_store.d.ts +19 -4
- package/storages/key_value_store.js +204 -75
- package/storages/request_dedup_cache.d.ts +1 -2
- package/storages/request_dedup_cache.js +9 -9
- package/storages/request_list.d.ts +4 -23
- package/storages/request_list.js +103 -96
- package/storages/request_loader.d.ts +5 -0
- package/storages/request_manager_tandem.d.ts +1 -10
- package/storages/request_manager_tandem.js +27 -27
- package/storages/request_queue.d.ts +21 -18
- package/storages/request_queue.js +318 -177
- package/storages/sitemap_request_loader.d.ts +22 -70
- package/storages/sitemap_request_loader.js +122 -116
- package/storages/storage_instance_manager.d.ts +1 -2
- package/storages/storage_instance_manager.js +17 -17
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/throttling_request_manager.d.ts +239 -0
- package/storages/throttling_request_manager.js +646 -0
- package/storages/transaction.d.ts +252 -0
- package/storages/transaction.js +251 -0
- package/storages/utils.d.ts +6 -3
- package/storages/utils.js +11 -2
- package/system-info/runtime.js +7 -7
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +22 -25
- package/validators.js +13 -25
- package/storages/access_checking.d.ts +0 -12
- package/storages/access_checking.js +0 -17
package/configuration.d.ts
CHANGED
|
@@ -6,47 +6,47 @@ export interface ConfigField<T extends z.ZodType = z.ZodType> {
|
|
|
6
6
|
}
|
|
7
7
|
export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
|
|
8
8
|
/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
|
|
9
|
-
export declare const coerceBoolean: z.
|
|
10
|
-
export declare const coerceNumber: z.
|
|
9
|
+
export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean>;
|
|
10
|
+
export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber>;
|
|
11
11
|
export declare const crawleeConfigFields: {
|
|
12
12
|
/** @default 'default' */
|
|
13
13
|
defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
14
14
|
/** @default true */
|
|
15
|
-
purgeOnStart: ConfigField<z.ZodDefault<z.
|
|
15
|
+
purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
16
16
|
/** @default 'default' */
|
|
17
17
|
defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
18
18
|
/** @default 'default' */
|
|
19
19
|
defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
20
20
|
/** @default 0.95 */
|
|
21
|
-
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.
|
|
21
|
+
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
22
22
|
/** @default 0.25 */
|
|
23
|
-
availableMemoryRatio: ConfigField<z.ZodDefault<z.
|
|
24
|
-
memoryMbytes: ConfigField<z.ZodOptional<z.
|
|
23
|
+
availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
24
|
+
memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
|
|
25
25
|
/** @default 60_000 */
|
|
26
|
-
persistStateIntervalMillis: ConfigField<z.ZodDefault<z.
|
|
26
|
+
persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
27
27
|
/**
|
|
28
28
|
* Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
|
|
29
29
|
* the request handler timeout (twice it, and never below 5 minutes).
|
|
30
30
|
*/
|
|
31
|
-
internalTimeoutMillis: ConfigField<z.ZodOptional<z.
|
|
31
|
+
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
|
|
32
32
|
/** @default 1_000 */
|
|
33
|
-
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.
|
|
33
|
+
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
34
34
|
/** @default 'INPUT' */
|
|
35
35
|
inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
36
36
|
/** @default true */
|
|
37
|
-
headless: ConfigField<z.ZodDefault<z.
|
|
37
|
+
headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
38
38
|
/** @default false */
|
|
39
|
-
xvfb: ConfigField<z.ZodDefault<z.
|
|
39
|
+
xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
40
40
|
chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
41
41
|
defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
42
42
|
/** @default false */
|
|
43
|
-
disableBrowserSandbox: ConfigField<z.ZodDefault<z.
|
|
44
|
-
logLevel: ConfigField<z.ZodOptional<z.
|
|
43
|
+
disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
44
|
+
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>>>>;
|
|
45
45
|
/** @default true */
|
|
46
|
-
persistStorage: ConfigField<z.ZodDefault<z.
|
|
46
|
+
persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
47
47
|
/** @default './storage' */
|
|
48
48
|
storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
49
|
-
containerized: ConfigField<z.ZodOptional<z.
|
|
49
|
+
containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
50
50
|
};
|
|
51
51
|
export type FieldsInput<F extends Record<string, ConfigField>> = {
|
|
52
52
|
[K in keyof F]?: z.output<F[K]['schema']>;
|
|
@@ -123,12 +123,12 @@ export interface Configuration extends ResolvedConfigValues {
|
|
|
123
123
|
* `containerized` | `CRAWLEE_CONTAINERIZED` | -
|
|
124
124
|
*/
|
|
125
125
|
export declare class Configuration {
|
|
126
|
+
#private;
|
|
126
127
|
/**
|
|
127
128
|
* Field definitions for this configuration class.
|
|
128
129
|
* Subclasses override this to register additional fields.
|
|
129
130
|
*/
|
|
130
131
|
protected static fields: Record<string, ConfigField>;
|
|
131
|
-
private resolvedValues;
|
|
132
132
|
/**
|
|
133
133
|
* Creates new `Configuration` instance with provided options.
|
|
134
134
|
* Constructor options take precedence over environment variables, which take precedence
|
package/configuration.js
CHANGED
|
@@ -36,7 +36,7 @@ const logLevelSchema = z.preprocess((val) => {
|
|
|
36
36
|
if (key in LogLevel)
|
|
37
37
|
return LogLevel[key];
|
|
38
38
|
return val;
|
|
39
|
-
}, z.
|
|
39
|
+
}, z.enum(LogLevel));
|
|
40
40
|
// --- Crawlee config field definitions ---
|
|
41
41
|
export const crawleeConfigFields = {
|
|
42
42
|
/** @default 'default' */
|
|
@@ -147,7 +147,7 @@ export class Configuration {
|
|
|
147
147
|
* Subclasses override this to register additional fields.
|
|
148
148
|
*/
|
|
149
149
|
static fields = crawleeConfigFields;
|
|
150
|
-
resolvedValues;
|
|
150
|
+
#resolvedValues;
|
|
151
151
|
/**
|
|
152
152
|
* Creates new `Configuration` instance with provided options.
|
|
153
153
|
* Constructor options take precedence over environment variables, which take precedence
|
|
@@ -156,7 +156,7 @@ export class Configuration {
|
|
|
156
156
|
constructor(options = {}) {
|
|
157
157
|
const fields = this.constructor.fields;
|
|
158
158
|
const fileOptions = Configuration.loadFileOptions();
|
|
159
|
-
this
|
|
159
|
+
this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
|
|
160
160
|
this.registerAccessors();
|
|
161
161
|
// Set the log level
|
|
162
162
|
const logLevel = this.logLevel;
|
|
@@ -196,8 +196,8 @@ export class Configuration {
|
|
|
196
196
|
continue;
|
|
197
197
|
}
|
|
198
198
|
// 4. Schema default (by parsing undefined through the schema)
|
|
199
|
-
const
|
|
200
|
-
values[key] =
|
|
199
|
+
const parsed = fieldDef.schema.safeParse(undefined);
|
|
200
|
+
values[key] = parsed.success ? parsed.data : undefined;
|
|
201
201
|
}
|
|
202
202
|
return values;
|
|
203
203
|
}
|
|
@@ -209,7 +209,7 @@ export class Configuration {
|
|
|
209
209
|
const descriptors = {};
|
|
210
210
|
for (const key of Object.keys(fields)) {
|
|
211
211
|
descriptors[key] = {
|
|
212
|
-
get: () => this
|
|
212
|
+
get: () => this.#resolvedValues[key],
|
|
213
213
|
set() {
|
|
214
214
|
throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
|
|
215
215
|
},
|
|
@@ -26,12 +26,12 @@ export class ContextPipeline {
|
|
|
26
26
|
* properties from the `ContextPipeline` interface, making type checking more reliable.
|
|
27
27
|
*/
|
|
28
28
|
class ContextPipelineImpl extends ContextPipeline {
|
|
29
|
-
middleware;
|
|
30
|
-
parent;
|
|
29
|
+
#middleware;
|
|
30
|
+
#parent;
|
|
31
31
|
constructor(middleware, parent) {
|
|
32
32
|
super();
|
|
33
|
-
this
|
|
34
|
-
this
|
|
33
|
+
this.#middleware = middleware;
|
|
34
|
+
this.#parent = parent;
|
|
35
35
|
}
|
|
36
36
|
/**
|
|
37
37
|
* @inheritdoc
|
|
@@ -50,8 +50,8 @@ class ContextPipelineImpl extends ContextPipeline {
|
|
|
50
50
|
*middlewareChain() {
|
|
51
51
|
let step = this;
|
|
52
52
|
while (step !== undefined) {
|
|
53
|
-
yield step
|
|
54
|
-
step = step
|
|
53
|
+
yield step.#middleware;
|
|
54
|
+
step = step.#parent;
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
/**
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
|
|
2
|
-
import type { ReadonlyDeep
|
|
3
|
-
import type {
|
|
4
|
-
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
|
|
2
|
+
import type { ReadonlyDeep } from 'type-fest';
|
|
3
|
+
import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
|
|
5
4
|
import type { CrawleeLogger } from '../log.js';
|
|
6
5
|
import type { Request, RequestOptions, Source } from '../request.js';
|
|
7
6
|
import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
|
|
8
7
|
import type { Dataset } from '../storages/dataset.js';
|
|
9
|
-
import { KeyValueStore
|
|
10
|
-
import type {
|
|
8
|
+
import type { KeyValueStore } from '../storages/key_value_store.js';
|
|
9
|
+
import type { AddRequestsBatchedResult } from '../storages/request_queue.js';
|
|
11
10
|
/** @internal */
|
|
12
11
|
export type IsAny<T> = 0 extends 1 & T ? true : false;
|
|
13
12
|
/**
|
|
@@ -38,7 +37,7 @@ export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
|
|
|
38
37
|
* The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
|
|
39
38
|
* bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
|
|
40
39
|
*/
|
|
41
|
-
export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<
|
|
40
|
+
export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
|
|
42
41
|
/**
|
|
43
42
|
* An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
|
|
44
43
|
* declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
|
|
@@ -88,37 +87,15 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
|
|
|
88
87
|
*/
|
|
89
88
|
pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
|
|
90
89
|
/**
|
|
91
|
-
*
|
|
92
|
-
* currently used by the crawler.
|
|
90
|
+
* Add requests directly to the request queue currently used by the crawler.
|
|
93
91
|
*
|
|
94
|
-
* Optionally, the function allows you to filter the target
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
|
|
98
|
-
* for more details regarding its usage.
|
|
99
|
-
*
|
|
100
|
-
* **Example usage**
|
|
101
|
-
*
|
|
102
|
-
* ```ts
|
|
103
|
-
* async requestHandler({ enqueueLinks }) {
|
|
104
|
-
* await enqueueLinks({
|
|
105
|
-
* globs: [
|
|
106
|
-
* 'https://www.example.com/handbags/*',
|
|
107
|
-
* ],
|
|
108
|
-
* });
|
|
109
|
-
* },
|
|
110
|
-
* ```
|
|
111
|
-
*
|
|
112
|
-
* @param [options] All `enqueueLinks()` parameters are passed via an options object.
|
|
113
|
-
*/
|
|
114
|
-
enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
|
|
115
|
-
/**
|
|
116
|
-
* Add requests directly to the request queue.
|
|
92
|
+
* Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
|
|
93
|
+
* the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
|
|
117
94
|
*
|
|
118
95
|
* @param requests The requests to add
|
|
119
96
|
* @param options Options for the request queue
|
|
120
97
|
*/
|
|
121
|
-
addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<
|
|
98
|
+
addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
|
|
122
99
|
/**
|
|
123
100
|
* Returns the state - a piece of mutable persistent data shared across all the request handler runs.
|
|
124
101
|
*/
|
|
@@ -133,32 +110,6 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
|
|
|
133
110
|
log: CrawleeLogger;
|
|
134
111
|
}
|
|
135
112
|
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
|
|
136
|
-
/**
|
|
137
|
-
* This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
|
|
138
|
-
* currently used by the crawler.
|
|
139
|
-
*
|
|
140
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
|
|
141
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
142
|
-
*
|
|
143
|
-
* Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
|
|
144
|
-
* for more details regarding its usage.
|
|
145
|
-
*
|
|
146
|
-
* **Example usage**
|
|
147
|
-
*
|
|
148
|
-
* ```ts
|
|
149
|
-
* async requestHandler({ enqueueLinks }) {
|
|
150
|
-
* await enqueueLinks({
|
|
151
|
-
* globs: [
|
|
152
|
-
* 'https://www.example.com/handbags/*',
|
|
153
|
-
* ],
|
|
154
|
-
* });
|
|
155
|
-
* },
|
|
156
|
-
* ```
|
|
157
|
-
*
|
|
158
|
-
* @param [options] All `enqueueLinks()` parameters are passed via an options object.
|
|
159
|
-
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
160
|
-
*/
|
|
161
|
-
enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
|
|
162
113
|
/**
|
|
163
114
|
* Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
|
|
164
115
|
*
|
|
@@ -178,6 +129,11 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
|
|
|
178
129
|
sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
|
|
179
130
|
/**
|
|
180
131
|
* Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
|
|
132
|
+
*
|
|
133
|
+
* The callback runs *outside* the request's storage transaction, so storage writes made here are
|
|
134
|
+
* applied immediately and are **not** rolled back when the request fails. In
|
|
135
|
+
* {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
|
|
136
|
+
* here can land more than once for a single request. Push results from the request handler itself.
|
|
181
137
|
*/
|
|
182
138
|
registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
|
|
183
139
|
/**
|
|
@@ -200,58 +156,4 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
|
|
|
200
156
|
*/
|
|
201
157
|
extendTimeout(secs: number): void;
|
|
202
158
|
}
|
|
203
|
-
/**
|
|
204
|
-
* A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
|
|
205
|
-
*
|
|
206
|
-
* @experimental
|
|
207
|
-
*/
|
|
208
|
-
export declare class RequestHandlerResult {
|
|
209
|
-
private configuration;
|
|
210
|
-
private crawleeStateKey;
|
|
211
|
-
private _keyValueStoreChanges;
|
|
212
|
-
private pushDataCalls;
|
|
213
|
-
private addRequestsCalls;
|
|
214
|
-
constructor(configuration: Configuration, crawleeStateKey: string);
|
|
215
|
-
/**
|
|
216
|
-
* A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
|
|
217
|
-
*/
|
|
218
|
-
get calls(): ReadonlyDeep<{
|
|
219
|
-
pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
|
|
220
|
-
addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
|
|
221
|
-
}>;
|
|
222
|
-
/**
|
|
223
|
-
* A record of changes made to key-value stores by a request handler.
|
|
224
|
-
*/
|
|
225
|
-
get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
|
|
226
|
-
changedValue: unknown;
|
|
227
|
-
options?: RecordOptions;
|
|
228
|
-
}>>>;
|
|
229
|
-
/**
|
|
230
|
-
* Items added to datasets by a request handler.
|
|
231
|
-
*/
|
|
232
|
-
get datasetItems(): ReadonlyDeep<{
|
|
233
|
-
item: Dictionary;
|
|
234
|
-
datasetIdentifier?: string | StorageIdentifier;
|
|
235
|
-
}[]>;
|
|
236
|
-
/**
|
|
237
|
-
* URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
|
|
238
|
-
*/
|
|
239
|
-
get enqueuedUrls(): ReadonlyDeep<{
|
|
240
|
-
url: string;
|
|
241
|
-
label?: string;
|
|
242
|
-
}[]>;
|
|
243
|
-
/**
|
|
244
|
-
* URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
|
|
245
|
-
*/
|
|
246
|
-
get enqueuedUrlLists(): ReadonlyDeep<{
|
|
247
|
-
listUrl: string;
|
|
248
|
-
label?: string;
|
|
249
|
-
}[]>;
|
|
250
|
-
pushData: RestrictedCrawlingContext['pushData'];
|
|
251
|
-
addRequests: RestrictedCrawlingContext['addRequests'];
|
|
252
|
-
useState: RestrictedCrawlingContext['useState'];
|
|
253
|
-
getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
|
|
254
|
-
private getKeyValueStoreChangedValue;
|
|
255
|
-
private setKeyValueStoreChangedValue;
|
|
256
|
-
}
|
|
257
159
|
export {};
|
|
@@ -1,107 +1 @@
|
|
|
1
|
-
|
|
2
|
-
/**
|
|
3
|
-
* A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
|
|
4
|
-
*
|
|
5
|
-
* @experimental
|
|
6
|
-
*/
|
|
7
|
-
export class RequestHandlerResult {
|
|
8
|
-
configuration;
|
|
9
|
-
crawleeStateKey;
|
|
10
|
-
_keyValueStoreChanges = {};
|
|
11
|
-
pushDataCalls = [];
|
|
12
|
-
addRequestsCalls = [];
|
|
13
|
-
constructor(configuration, crawleeStateKey) {
|
|
14
|
-
this.configuration = configuration;
|
|
15
|
-
this.crawleeStateKey = crawleeStateKey;
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
|
|
19
|
-
*/
|
|
20
|
-
get calls() {
|
|
21
|
-
return {
|
|
22
|
-
pushData: this.pushDataCalls,
|
|
23
|
-
addRequests: this.addRequestsCalls,
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* A record of changes made to key-value stores by a request handler.
|
|
28
|
-
*/
|
|
29
|
-
get keyValueStoreChanges() {
|
|
30
|
-
return this._keyValueStoreChanges;
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Items added to datasets by a request handler.
|
|
34
|
-
*/
|
|
35
|
-
get datasetItems() {
|
|
36
|
-
return this.pushDataCalls.flatMap(([data, datasetIdentifier]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdentifier })));
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
|
|
40
|
-
*/
|
|
41
|
-
get enqueuedUrls() {
|
|
42
|
-
const result = [];
|
|
43
|
-
for (const [requests] of this.addRequestsCalls) {
|
|
44
|
-
for (const request of requests) {
|
|
45
|
-
if (typeof request === 'object' &&
|
|
46
|
-
(!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
|
|
47
|
-
request.url !== undefined) {
|
|
48
|
-
result.push({ url: request.url, label: request.label });
|
|
49
|
-
}
|
|
50
|
-
else if (typeof request === 'string') {
|
|
51
|
-
result.push({ url: request });
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return result;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
|
|
59
|
-
*/
|
|
60
|
-
get enqueuedUrlLists() {
|
|
61
|
-
const result = [];
|
|
62
|
-
for (const [requests] of this.addRequestsCalls) {
|
|
63
|
-
for (const request of requests) {
|
|
64
|
-
if (typeof request === 'object' &&
|
|
65
|
-
'requestsFromUrl' in request &&
|
|
66
|
-
request.requestsFromUrl !== undefined) {
|
|
67
|
-
result.push({ listUrl: request.requestsFromUrl, label: request.label });
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
return result;
|
|
72
|
-
}
|
|
73
|
-
pushData = async (data, datasetIdOrName) => {
|
|
74
|
-
this.pushDataCalls.push([data, datasetIdOrName]);
|
|
75
|
-
};
|
|
76
|
-
addRequests = async (requests, options = {}) => {
|
|
77
|
-
this.addRequestsCalls.push([requests, options]);
|
|
78
|
-
};
|
|
79
|
-
useState = async (defaultValue) => {
|
|
80
|
-
const store = await this.getKeyValueStore(undefined);
|
|
81
|
-
return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
|
|
82
|
-
};
|
|
83
|
-
getKeyValueStore = async (identifier) => {
|
|
84
|
-
const store = await KeyValueStore.open(identifier, { configuration: this.configuration });
|
|
85
|
-
const storeId = store.id;
|
|
86
|
-
return {
|
|
87
|
-
id: storeId ?? this.configuration.defaultKeyValueStoreId,
|
|
88
|
-
name: store.name,
|
|
89
|
-
getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
|
|
90
|
-
setValue: async (key, value, options) => {
|
|
91
|
-
this.setKeyValueStoreChangedValue(storeId, key, value, options);
|
|
92
|
-
},
|
|
93
|
-
getAutoSavedValue: store.getAutoSavedValue.bind(store),
|
|
94
|
-
getPublicUrl: store.getPublicUrl.bind(store),
|
|
95
|
-
};
|
|
96
|
-
};
|
|
97
|
-
getKeyValueStoreChangedValue = (storeKey, key) => {
|
|
98
|
-
const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
|
|
99
|
-
this._keyValueStoreChanges[id] ??= {};
|
|
100
|
-
return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
|
|
101
|
-
};
|
|
102
|
-
setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
|
|
103
|
-
const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
|
|
104
|
-
this._keyValueStoreChanges[id] ??= {};
|
|
105
|
-
this._keyValueStoreChanges[id][key] = { changedValue, options };
|
|
106
|
-
};
|
|
107
|
-
}
|
|
1
|
+
export {};
|
|
@@ -25,9 +25,7 @@ interface ErrorSnapshot {
|
|
|
25
25
|
* ```ts
|
|
26
26
|
* const crawler = new BasicCrawler({
|
|
27
27
|
* // ...
|
|
28
|
-
*
|
|
29
|
-
* saveErrorSnapshots: true,
|
|
30
|
-
* },
|
|
28
|
+
* statistics: new Statistics({ saveErrorSnapshots: true }),
|
|
31
29
|
* });
|
|
32
30
|
* ```
|
|
33
31
|
*/
|
|
@@ -48,7 +46,7 @@ export declare class ErrorSnapshotter {
|
|
|
48
46
|
*/
|
|
49
47
|
contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
|
|
50
48
|
/**
|
|
51
|
-
* Save the HTML snapshot of the page, and return the
|
|
49
|
+
* Save the HTML snapshot of the page, and return the key it was stored under.
|
|
52
50
|
*/
|
|
53
51
|
saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
|
|
54
52
|
/**
|
|
@@ -7,9 +7,7 @@ import crypto from 'node:crypto';
|
|
|
7
7
|
* ```ts
|
|
8
8
|
* const crawler = new BasicCrawler({
|
|
9
9
|
* // ...
|
|
10
|
-
*
|
|
11
|
-
* saveErrorSnapshots: true,
|
|
12
|
-
* },
|
|
10
|
+
* statistics: new Statistics({ saveErrorSnapshots: true }),
|
|
13
11
|
* });
|
|
14
12
|
* ```
|
|
15
13
|
*/
|
|
@@ -79,12 +77,14 @@ export class ErrorSnapshotter {
|
|
|
79
77
|
}
|
|
80
78
|
}
|
|
81
79
|
/**
|
|
82
|
-
* Save the HTML snapshot of the page, and return the
|
|
80
|
+
* Save the HTML snapshot of the page, and return the key it was stored under.
|
|
83
81
|
*/
|
|
84
82
|
async saveHTMLSnapshot(html, keyValueStore, fileName) {
|
|
85
83
|
try {
|
|
86
84
|
await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
|
|
87
|
-
|
|
85
|
+
// The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
|
|
86
|
+
// where local storage put the extension in the key) would break `getPublicUrl`.
|
|
87
|
+
return fileName;
|
|
88
88
|
}
|
|
89
89
|
catch {
|
|
90
90
|
return undefined;
|
package/crawlers/index.d.ts
CHANGED