@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +3 -21
  2. package/autoscaling/autoscaled_pool.js +105 -97
  3. package/autoscaling/concurrency_system.d.ts +7 -22
  4. package/autoscaling/concurrency_system.js +112 -100
  5. package/autoscaling/cpu_load_signal.d.ts +1 -2
  6. package/autoscaling/cpu_load_signal.js +11 -10
  7. package/autoscaling/event_loop_load_signal.d.ts +1 -4
  8. package/autoscaling/event_loop_load_signal.js +18 -18
  9. package/autoscaling/index.d.ts +1 -1
  10. package/autoscaling/index.js +1 -1
  11. package/autoscaling/load_signal.d.ts +8 -7
  12. package/autoscaling/load_signal.js +14 -12
  13. package/autoscaling/memory_load_signal.d.ts +3 -12
  14. package/autoscaling/memory_load_signal.js +41 -41
  15. package/autoscaling/snapshotter.d.ts +7 -10
  16. package/autoscaling/snapshotter.js +18 -18
  17. package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts} +14 -17
  18. package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js} +27 -27
  19. package/autoscaling/system_status.d.ts +9 -11
  20. package/autoscaling/system_status.js +13 -13
  21. package/configuration.d.ts +16 -16
  22. package/configuration.js +6 -6
  23. package/crawlers/context_pipeline.js +6 -6
  24. package/crawlers/crawler_commons.d.ts +14 -112
  25. package/crawlers/crawler_commons.js +1 -107
  26. package/crawlers/error_snapshotter.d.ts +2 -4
  27. package/crawlers/error_snapshotter.js +5 -5
  28. package/crawlers/index.d.ts +1 -1
  29. package/crawlers/index.js +0 -1
  30. package/crawlers/statistics.d.ts +164 -45
  31. package/crawlers/statistics.js +339 -153
  32. package/debug.js +4 -4
  33. package/enqueue_links/enqueue_links.d.ts +51 -159
  34. package/enqueue_links/enqueue_links.js +38 -231
  35. package/enqueue_links/shared.d.ts +34 -24
  36. package/enqueue_links/shared.js +54 -36
  37. package/errors.d.ts +43 -12
  38. package/errors.js +48 -9
  39. package/events/event_manager.d.ts +2 -2
  40. package/events/event_manager.js +7 -6
  41. package/events/local_event_manager.d.ts +1 -1
  42. package/events/local_event_manager.js +5 -5
  43. package/http.d.ts +9 -0
  44. package/http.js +28 -0
  45. package/index.d.ts +1 -1
  46. package/index.js +1 -1
  47. package/log.js +5 -1
  48. package/memory-storage/memory-storage.d.ts +3 -11
  49. package/memory-storage/memory-storage.js +42 -48
  50. package/memory-storage/resource-clients/dataset.d.ts +1 -1
  51. package/memory-storage/resource-clients/dataset.js +8 -13
  52. package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
  53. package/memory-storage/resource-clients/key-value-store.js +36 -38
  54. package/memory-storage/resource-clients/request-queue.d.ts +4 -23
  55. package/memory-storage/resource-clients/request-queue.js +68 -80
  56. package/owned_or_injected.d.ts +1 -3
  57. package/owned_or_injected.js +17 -17
  58. package/package.json +8 -9
  59. package/proxy_configuration.d.ts +1 -3
  60. package/proxy_configuration.js +18 -14
  61. package/recoverable_state.d.ts +80 -45
  62. package/recoverable_state.js +163 -75
  63. package/request.d.ts +3 -4
  64. package/request.js +57 -44
  65. package/router.d.ts +6 -9
  66. package/router.js +23 -23
  67. package/serialization.js +14 -13
  68. package/service_locator.d.ts +19 -10
  69. package/service_locator.js +57 -48
  70. package/session_pool/session.d.ts +3 -14
  71. package/session_pool/session.js +76 -74
  72. package/session_pool/session_pool.d.ts +6 -12
  73. package/session_pool/session_pool.js +85 -77
  74. package/storages/batched_adds.d.ts +37 -0
  75. package/storages/batched_adds.js +73 -0
  76. package/storages/dataset.d.ts +12 -1
  77. package/storages/dataset.js +133 -32
  78. package/storages/index.d.ts +2 -1
  79. package/storages/index.js +2 -1
  80. package/storages/key_value_store.d.ts +19 -4
  81. package/storages/key_value_store.js +204 -75
  82. package/storages/request_dedup_cache.d.ts +1 -2
  83. package/storages/request_dedup_cache.js +9 -9
  84. package/storages/request_list.d.ts +4 -23
  85. package/storages/request_list.js +103 -96
  86. package/storages/request_loader.d.ts +5 -0
  87. package/storages/request_manager_tandem.d.ts +1 -10
  88. package/storages/request_manager_tandem.js +27 -27
  89. package/storages/request_queue.d.ts +21 -18
  90. package/storages/request_queue.js +318 -177
  91. package/storages/sitemap_request_loader.d.ts +22 -70
  92. package/storages/sitemap_request_loader.js +122 -116
  93. package/storages/storage_instance_manager.d.ts +1 -2
  94. package/storages/storage_instance_manager.js +17 -17
  95. package/storages/storage_stats.d.ts +1 -1
  96. package/storages/storage_stats.js +4 -4
  97. package/storages/throttling_request_manager.d.ts +239 -0
  98. package/storages/throttling_request_manager.js +646 -0
  99. package/storages/transaction.d.ts +252 -0
  100. package/storages/transaction.js +251 -0
  101. package/storages/utils.d.ts +6 -3
  102. package/storages/utils.js +11 -2
  103. package/system-info/runtime.js +7 -7
  104. package/url.d.ts +9 -0
  105. package/url.js +11 -0
  106. package/validators.d.ts +22 -25
  107. package/validators.js +13 -25
  108. package/storages/access_checking.d.ts +0 -12
  109. package/storages/access_checking.js +0 -17
@@ -6,47 +6,47 @@ export interface ConfigField<T extends z.ZodType = z.ZodType> {
6
6
  }
7
7
  export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
8
8
  /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
9
- export declare const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
10
- export declare const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
9
+ export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean>;
10
+ export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber>;
11
11
  export declare const crawleeConfigFields: {
12
12
  /** @default 'default' */
13
13
  defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
14
14
  /** @default true */
15
- purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
15
+ purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
16
16
  /** @default 'default' */
17
17
  defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
18
18
  /** @default 'default' */
19
19
  defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
20
20
  /** @default 0.95 */
21
- maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
21
+ maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
22
22
  /** @default 0.25 */
23
- availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
24
- memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
23
+ availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
24
+ memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
25
25
  /** @default 60_000 */
26
- persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
26
+ persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
27
27
  /**
28
28
  * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
29
29
  * the request handler timeout (twice it, and never below 5 minutes).
30
30
  */
31
- internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
31
+ internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
32
32
  /** @default 1_000 */
33
- systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
33
+ systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
34
34
  /** @default 'INPUT' */
35
35
  inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
36
36
  /** @default true */
37
- headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
37
+ headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
38
38
  /** @default false */
39
- xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
39
+ xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
40
40
  chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
41
41
  defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
42
42
  /** @default false */
43
- disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
44
- logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
43
+ disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
44
+ logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>>>>;
45
45
  /** @default true */
46
- persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
46
+ persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
47
47
  /** @default './storage' */
48
48
  storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
49
- containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
49
+ containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
50
50
  };
51
51
  export type FieldsInput<F extends Record<string, ConfigField>> = {
52
52
  [K in keyof F]?: z.output<F[K]['schema']>;
@@ -123,12 +123,12 @@ export interface Configuration extends ResolvedConfigValues {
123
123
  * `containerized` | `CRAWLEE_CONTAINERIZED` | -
124
124
  */
125
125
  export declare class Configuration {
126
+ #private;
126
127
  /**
127
128
  * Field definitions for this configuration class.
128
129
  * Subclasses override this to register additional fields.
129
130
  */
130
131
  protected static fields: Record<string, ConfigField>;
131
- private resolvedValues;
132
132
  /**
133
133
  * Creates new `Configuration` instance with provided options.
134
134
  * Constructor options take precedence over environment variables, which take precedence
package/configuration.js CHANGED
@@ -36,7 +36,7 @@ const logLevelSchema = z.preprocess((val) => {
36
36
  if (key in LogLevel)
37
37
  return LogLevel[key];
38
38
  return val;
39
- }, z.nativeEnum(LogLevel));
39
+ }, z.enum(LogLevel));
40
40
  // --- Crawlee config field definitions ---
41
41
  export const crawleeConfigFields = {
42
42
  /** @default 'default' */
@@ -147,7 +147,7 @@ export class Configuration {
147
147
  * Subclasses override this to register additional fields.
148
148
  */
149
149
  static fields = crawleeConfigFields;
150
- resolvedValues;
150
+ #resolvedValues;
151
151
  /**
152
152
  * Creates new `Configuration` instance with provided options.
153
153
  * Constructor options take precedence over environment variables, which take precedence
@@ -156,7 +156,7 @@ export class Configuration {
156
156
  constructor(options = {}) {
157
157
  const fields = this.constructor.fields;
158
158
  const fileOptions = Configuration.loadFileOptions();
159
- this.resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
159
+ this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
160
160
  this.registerAccessors();
161
161
  // Set the log level
162
162
  const logLevel = this.logLevel;
@@ -196,8 +196,8 @@ export class Configuration {
196
196
  continue;
197
197
  }
198
198
  // 4. Schema default (by parsing undefined through the schema)
199
- const result = fieldDef.schema.safeParse(undefined);
200
- values[key] = result.success ? result.data : undefined;
199
+ const parsed = fieldDef.schema.safeParse(undefined);
200
+ values[key] = parsed.success ? parsed.data : undefined;
201
201
  }
202
202
  return values;
203
203
  }
@@ -209,7 +209,7 @@ export class Configuration {
209
209
  const descriptors = {};
210
210
  for (const key of Object.keys(fields)) {
211
211
  descriptors[key] = {
212
- get: () => this.resolvedValues[key],
212
+ get: () => this.#resolvedValues[key],
213
213
  set() {
214
214
  throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
215
215
  },
@@ -26,12 +26,12 @@ export class ContextPipeline {
26
26
  * properties from the `ContextPipeline` interface, making type checking more reliable.
27
27
  */
28
28
  class ContextPipelineImpl extends ContextPipeline {
29
- middleware;
30
- parent;
29
+ #middleware;
30
+ #parent;
31
31
  constructor(middleware, parent) {
32
32
  super();
33
- this.middleware = middleware;
34
- this.parent = parent;
33
+ this.#middleware = middleware;
34
+ this.#parent = parent;
35
35
  }
36
36
  /**
37
37
  * @inheritdoc
@@ -50,8 +50,8 @@ class ContextPipelineImpl extends ContextPipeline {
50
50
  *middlewareChain() {
51
51
  let step = this;
52
52
  while (step !== undefined) {
53
- yield step.middleware;
54
- step = step.parent;
53
+ yield step.#middleware;
54
+ step = step.#parent;
55
55
  }
56
56
  }
57
57
  /**
@@ -1,13 +1,12 @@
1
1
  import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
2
- import type { ReadonlyDeep, SetRequired } from 'type-fest';
3
- import type { Configuration } from '../configuration.js';
4
- import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
2
+ import type { ReadonlyDeep } from 'type-fest';
3
+ import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
5
4
  import type { CrawleeLogger } from '../log.js';
6
5
  import type { Request, RequestOptions, Source } from '../request.js';
7
6
  import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
8
7
  import type { Dataset } from '../storages/dataset.js';
9
- import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
10
- import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
8
+ import type { KeyValueStore } from '../storages/key_value_store.js';
9
+ import type { AddRequestsBatchedResult } from '../storages/request_queue.js';
11
10
  /** @internal */
12
11
  export type IsAny<T> = 0 extends 1 & T ? true : false;
13
12
  /**
@@ -38,7 +37,7 @@ export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
38
37
  * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
39
38
  * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
40
39
  */
41
- export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
40
+ export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
42
41
  /**
43
42
  * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
44
43
  * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
@@ -88,37 +87,15 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
88
87
  */
89
88
  pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
90
89
  /**
91
- * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
92
- * currently used by the crawler.
90
+ * Add requests directly to the request queue currently used by the crawler.
93
91
  *
94
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
95
- * and override settings of the enqueued {@link Request} objects.
96
- *
97
- * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
98
- * for more details regarding its usage.
99
- *
100
- * **Example usage**
101
- *
102
- * ```ts
103
- * async requestHandler({ enqueueLinks }) {
104
- * await enqueueLinks({
105
- * globs: [
106
- * 'https://www.example.com/handbags/*',
107
- * ],
108
- * });
109
- * },
110
- * ```
111
- *
112
- * @param [options] All `enqueueLinks()` parameters are passed via an options object.
113
- */
114
- enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
115
- /**
116
- * Add requests directly to the request queue.
92
+ * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
93
+ * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
117
94
  *
118
95
  * @param requests The requests to add
119
96
  * @param options Options for the request queue
120
97
  */
121
- addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
98
+ addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
122
99
  /**
123
100
  * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
124
101
  */
@@ -133,32 +110,6 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
133
110
  log: CrawleeLogger;
134
111
  }
135
112
  export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
136
- /**
137
- * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
138
- * currently used by the crawler.
139
- *
140
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
141
- * and override settings of the enqueued {@link Request} objects.
142
- *
143
- * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
144
- * for more details regarding its usage.
145
- *
146
- * **Example usage**
147
- *
148
- * ```ts
149
- * async requestHandler({ enqueueLinks }) {
150
- * await enqueueLinks({
151
- * globs: [
152
- * 'https://www.example.com/handbags/*',
153
- * ],
154
- * });
155
- * },
156
- * ```
157
- *
158
- * @param [options] All `enqueueLinks()` parameters are passed via an options object.
159
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
160
- */
161
- enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
162
113
  /**
163
114
  * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
164
115
  *
@@ -178,6 +129,11 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
178
129
  sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
179
130
  /**
180
131
  * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
132
+ *
133
+ * The callback runs *outside* the request's storage transaction, so storage writes made here are
134
+ * applied immediately and are **not** rolled back when the request fails. In
135
+ * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
136
+ * here can land more than once for a single request. Push results from the request handler itself.
181
137
  */
182
138
  registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
183
139
  /**
@@ -200,58 +156,4 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
200
156
  */
201
157
  extendTimeout(secs: number): void;
202
158
  }
203
- /**
204
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
205
- *
206
- * @experimental
207
- */
208
- export declare class RequestHandlerResult {
209
- private configuration;
210
- private crawleeStateKey;
211
- private _keyValueStoreChanges;
212
- private pushDataCalls;
213
- private addRequestsCalls;
214
- constructor(configuration: Configuration, crawleeStateKey: string);
215
- /**
216
- * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
217
- */
218
- get calls(): ReadonlyDeep<{
219
- pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
220
- addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
221
- }>;
222
- /**
223
- * A record of changes made to key-value stores by a request handler.
224
- */
225
- get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
226
- changedValue: unknown;
227
- options?: RecordOptions;
228
- }>>>;
229
- /**
230
- * Items added to datasets by a request handler.
231
- */
232
- get datasetItems(): ReadonlyDeep<{
233
- item: Dictionary;
234
- datasetIdentifier?: string | StorageIdentifier;
235
- }[]>;
236
- /**
237
- * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
238
- */
239
- get enqueuedUrls(): ReadonlyDeep<{
240
- url: string;
241
- label?: string;
242
- }[]>;
243
- /**
244
- * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
245
- */
246
- get enqueuedUrlLists(): ReadonlyDeep<{
247
- listUrl: string;
248
- label?: string;
249
- }[]>;
250
- pushData: RestrictedCrawlingContext['pushData'];
251
- addRequests: RestrictedCrawlingContext['addRequests'];
252
- useState: RestrictedCrawlingContext['useState'];
253
- getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
254
- private getKeyValueStoreChangedValue;
255
- private setKeyValueStoreChangedValue;
256
- }
257
159
  export {};
@@ -1,107 +1 @@
1
- import { KeyValueStore } from '../storages/key_value_store.js';
2
- /**
3
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
4
- *
5
- * @experimental
6
- */
7
- export class RequestHandlerResult {
8
- configuration;
9
- crawleeStateKey;
10
- _keyValueStoreChanges = {};
11
- pushDataCalls = [];
12
- addRequestsCalls = [];
13
- constructor(configuration, crawleeStateKey) {
14
- this.configuration = configuration;
15
- this.crawleeStateKey = crawleeStateKey;
16
- }
17
- /**
18
- * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
19
- */
20
- get calls() {
21
- return {
22
- pushData: this.pushDataCalls,
23
- addRequests: this.addRequestsCalls,
24
- };
25
- }
26
- /**
27
- * A record of changes made to key-value stores by a request handler.
28
- */
29
- get keyValueStoreChanges() {
30
- return this._keyValueStoreChanges;
31
- }
32
- /**
33
- * Items added to datasets by a request handler.
34
- */
35
- get datasetItems() {
36
- return this.pushDataCalls.flatMap(([data, datasetIdentifier]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdentifier })));
37
- }
38
- /**
39
- * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
40
- */
41
- get enqueuedUrls() {
42
- const result = [];
43
- for (const [requests] of this.addRequestsCalls) {
44
- for (const request of requests) {
45
- if (typeof request === 'object' &&
46
- (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
47
- request.url !== undefined) {
48
- result.push({ url: request.url, label: request.label });
49
- }
50
- else if (typeof request === 'string') {
51
- result.push({ url: request });
52
- }
53
- }
54
- }
55
- return result;
56
- }
57
- /**
58
- * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
59
- */
60
- get enqueuedUrlLists() {
61
- const result = [];
62
- for (const [requests] of this.addRequestsCalls) {
63
- for (const request of requests) {
64
- if (typeof request === 'object' &&
65
- 'requestsFromUrl' in request &&
66
- request.requestsFromUrl !== undefined) {
67
- result.push({ listUrl: request.requestsFromUrl, label: request.label });
68
- }
69
- }
70
- }
71
- return result;
72
- }
73
- pushData = async (data, datasetIdOrName) => {
74
- this.pushDataCalls.push([data, datasetIdOrName]);
75
- };
76
- addRequests = async (requests, options = {}) => {
77
- this.addRequestsCalls.push([requests, options]);
78
- };
79
- useState = async (defaultValue) => {
80
- const store = await this.getKeyValueStore(undefined);
81
- return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
82
- };
83
- getKeyValueStore = async (identifier) => {
84
- const store = await KeyValueStore.open(identifier, { configuration: this.configuration });
85
- const storeId = store.id;
86
- return {
87
- id: storeId ?? this.configuration.defaultKeyValueStoreId,
88
- name: store.name,
89
- getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
90
- setValue: async (key, value, options) => {
91
- this.setKeyValueStoreChangedValue(storeId, key, value, options);
92
- },
93
- getAutoSavedValue: store.getAutoSavedValue.bind(store),
94
- getPublicUrl: store.getPublicUrl.bind(store),
95
- };
96
- };
97
- getKeyValueStoreChangedValue = (storeKey, key) => {
98
- const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
99
- this._keyValueStoreChanges[id] ??= {};
100
- return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
101
- };
102
- setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
103
- const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
104
- this._keyValueStoreChanges[id] ??= {};
105
- this._keyValueStoreChanges[id][key] = { changedValue, options };
106
- };
107
- }
1
+ export {};
@@ -25,9 +25,7 @@ interface ErrorSnapshot {
25
25
  * ```ts
26
26
  * const crawler = new BasicCrawler({
27
27
  * // ...
28
- * statisticsOptions: {
29
- * saveErrorSnapshots: true,
30
- * },
28
+ * statistics: new Statistics({ saveErrorSnapshots: true }),
31
29
  * });
32
30
  * ```
33
31
  */
@@ -48,7 +46,7 @@ export declare class ErrorSnapshotter {
48
46
  */
49
47
  contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
50
48
  /**
51
- * Save the HTML snapshot of the page, and return the fileName with the extension.
49
+ * Save the HTML snapshot of the page, and return the key it was stored under.
52
50
  */
53
51
  saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
54
52
  /**
@@ -7,9 +7,7 @@ import crypto from 'node:crypto';
7
7
  * ```ts
8
8
  * const crawler = new BasicCrawler({
9
9
  * // ...
10
- * statisticsOptions: {
11
- * saveErrorSnapshots: true,
12
- * },
10
+ * statistics: new Statistics({ saveErrorSnapshots: true }),
13
11
  * });
14
12
  * ```
15
13
  */
@@ -79,12 +77,14 @@ export class ErrorSnapshotter {
79
77
  }
80
78
  }
81
79
  /**
82
- * Save the HTML snapshot of the page, and return the fileName with the extension.
80
+ * Save the HTML snapshot of the page, and return the key it was stored under.
83
81
  */
84
82
  async saveHTMLSnapshot(html, keyValueStore, fileName) {
85
83
  try {
86
84
  await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
87
- return `${fileName}.html`;
85
+ // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
86
+ // where local storage put the extension in the key) would break `getPublicUrl`.
87
+ return fileName;
88
88
  }
89
89
  catch {
90
90
  return undefined;
@@ -1,5 +1,5 @@
1
1
  export * from './context_pipeline.js';
2
- export * from './crawler_commons.js';
2
+ export type * from './crawler_commons.js';
3
3
  export * from './statistics.js';
4
4
  export * from './error_tracker.js';
5
5
  export * from './error_snapshotter.js';
package/crawlers/index.js CHANGED
@@ -1,5 +1,4 @@
1
1
  export * from './context_pipeline.js';
2
- export * from './crawler_commons.js';
3
2
  export * from './statistics.js';
4
3
  export * from './error_tracker.js';
5
4
  export * from './error_snapshotter.js';