@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/crawlers/crawler_commons.d.ts

@@ -1,19 +1,63 @@
-import type { Dictionary } from '@crawlee/types';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { OptionsInit } from 'got-scraping';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import type { Configuration } from '../configuration.js';
-import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
-import type { Log } from '../log.js';
-import type { ProxyInfo } from '../proxy_configuration.js';
-import type { Request, Source } from '../request.js';
-import type { Session } from '../session_pool/session.js';
+import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
+import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
+import type { CrawleeLogger } from '../log.js';
+import type { Request, RequestOptions, Source } from '../request.js';
+import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 import type { Dataset } from '../storages/dataset.js';
-import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
-import type { RequestQueueOperationOptions } from '../storages/request_provider.js';
+import type { KeyValueStore } from '../storages/key_value_store.js';
+import type { AddRequestsBatchedResult } from '../storages/request_queue.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
-/** @internal */
+/**
+ * A request input (URL string, request-options object, or {@link Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
+    requestsFromUrl?: string;
+    regex?: RegExp;
+} & ({
+    [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+        label: Label;
+    };
+}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
+    label?: undefined;
+})));
+/**
+ * The iterable/array of {@link LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ * @internal
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> = AsyncIterable<LabeledSource<Routes>> | Iterable<LabeledSource<Routes>> | LabeledSource<Routes>[];
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
+    [Label in keyof Routes & string]: {
+        label: Label;
+        userData?: Routes[Label];
+    };
+}[keyof Routes & string] | {
+    label?: undefined;
+    userData?: Dictionary;
+});
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ */
+export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
 export type WithRequired<T, K extends keyof T> = T & {
     [P in K]-?: T[P];
 };
@@ -24,7 +68,7 @@ export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Con
 } & Omit<Context, 'request'>;
 export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
     id: string;
-    session?: Session;
+    session: ISession;
     /**
      * An object with information about currently used proxy by the crawler
      * and configured by the {@link ProxyConfiguration} class.
@@ -41,39 +85,17 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      *
      * @param [data] Data to be pushed to the default dataset.
      */
-    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdOrName?: string): Promise<void>;
+    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
-     *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@link Request} objects.
+     * Add requests directly to the request queue currently used by the crawler.
      *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       globs: [
-     *           'https://www.example.com/handbags/*',
-     *       ],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     */
-    enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>>) => Promise<unknown>;
-    /**
-     * Add requests directly to the request queue.
+     * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
+     * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
      *
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
     /**
      * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
      */
@@ -81,42 +103,15 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     /**
      * Get a key-value store with given name or id, or the default one for the crawler.
      */
-    getKeyValueStore: (idOrName?: string) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
+    getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
     /**
      * A preconfigured logger for the request handler.
      */
-    log: Log;
+    log: CrawleeLogger;
 }
 export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
     /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
-     *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@link Request} objects.
-     *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       globs: [
-     *           'https://www.example.com/handbags/*',
-     *       ],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
-     */
-    enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestQueue' | 'robotsTxtFile'>): Promise<unknown>;
-    /**
-     * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
-     * options on the fly.
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *
      * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
      * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
@@ -131,68 +126,34 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * },
      * ```
      */
-    sendRequest(overrideOptions?: Partial<OptionsInit>): Promise<Response>;
+    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
     /**
      * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     *
+     * The callback runs *outside* the request's storage transaction, so storage writes made here are
+     * applied immediately and are **not** rolled back when the request fails. In
+     * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
+     * here can land more than once for a single request. Push results from the request handler itself.
      */
     registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
-}
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export declare class RequestHandlerResult {
-    private config;
-    private crawleeStateKey;
-    private _keyValueStoreChanges;
-    private pushDataCalls;
-    private addRequestsCalls;
-    private enqueueLinksCalls;
-    constructor(config: Configuration, crawleeStateKey: string);
     /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls(): ReadonlyDeep<{
-        pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
-        addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
-        enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
-    }>;
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
-        changedValue: unknown;
-        options?: RecordOptions;
-    }>>>;
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems(): ReadonlyDeep<{
-        item: Dictionary;
-        datasetIdOrName?: string;
-    }[]>;
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls(): ReadonlyDeep<{
-        url: string;
-        label?: string;
-    }[]>;
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
+     * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
+     * once it is already running - a listing page that turns out to have far more to scroll through than
+     * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
+     * {@link Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
+     *
+     * ```ts
+     * router.addHandler('LIST', async ({ extendTimeout, page }) => {
+     *     const pageCount = await countPages(page);
+     *     extendTimeout(pageCount * 10);
+     *     await scrapeAllPages(page);
+     * });
+     * ```
+     *
+     * Extends the request handler's own timeout and the crawler's internal one together, so the extension
+     * is not immediately undone by the latter. Calling it from a handler that has already timed out does
+     * nothing.
      */
-    get enqueuedUrlLists(): ReadonlyDeep<{
-        listUrl: string;
-        label?: string;
-    }[]>;
-    pushData: RestrictedCrawlingContext['pushData'];
-    enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
-    addRequests: RestrictedCrawlingContext['addRequests'];
-    useState: RestrictedCrawlingContext['useState'];
-    getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
-    private idOrDefault;
-    private getKeyValueStoreChangedValue;
-    private setKeyValueStoreChangedValue;
+    extendTimeout(secs: number): void;
 }
-//# sourceMappingURL=crawler_commons.d.ts.map
+export {};

package/crawlers/crawler_commons.js

@@ -1,116 +1 @@
-import { KeyValueStore } from '../storages/key_value_store.js';
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export class RequestHandlerResult {
-    config;
-    crawleeStateKey;
-    _keyValueStoreChanges = {};
-    pushDataCalls = [];
-    addRequestsCalls = [];
-    enqueueLinksCalls = [];
-    constructor(config, crawleeStateKey) {
-        this.config = config;
-        this.crawleeStateKey = crawleeStateKey;
-    }
-    /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls() {
-        return {
-            pushData: this.pushDataCalls,
-            addRequests: this.addRequestsCalls,
-            enqueueLinks: this.enqueueLinksCalls,
-        };
-    }
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges() {
-        return this._keyValueStoreChanges;
-    }
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems() {
-        return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
-    }
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls() {
-        const result = [];
-        for (const [options] of this.enqueueLinksCalls) {
-            result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
-        }
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
-                    request.url !== undefined) {
-                    result.push({ url: request.url, label: request.label });
-                }
-                else if (typeof request === 'string') {
-                    result.push({ url: request });
-                }
-            }
-        }
-        return result;
-    }
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
-     */
-    get enqueuedUrlLists() {
-        const result = [];
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    'requestsFromUrl' in request &&
-                    request.requestsFromUrl !== undefined) {
-                    result.push({ listUrl: request.requestsFromUrl, label: request.label });
-                }
-            }
-        }
-        return result;
-    }
-    pushData = async (data, datasetIdOrName) => {
-        this.pushDataCalls.push([data, datasetIdOrName]);
-    };
-    enqueueLinks = async (options) => {
-        this.enqueueLinksCalls.push([options]);
-    };
-    addRequests = async (requests, options = {}) => {
-        this.addRequestsCalls.push([requests, options]);
-    };
-    useState = async (defaultValue) => {
-        const store = await this.getKeyValueStore(undefined);
-        return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
-    };
-    getKeyValueStore = async (idOrName) => {
-        const store = await KeyValueStore.open(idOrName, { config: this.config });
-        return {
-            id: this.idOrDefault(idOrName),
-            name: idOrName,
-            getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? (await store.getValue(key)),
-            setValue: async (key, value, options) => {
-                this.setKeyValueStoreChangedValue(idOrName, key, value, options);
-            },
-            getAutoSavedValue: store.getAutoSavedValue.bind(store),
-            getPublicUrl: store.getPublicUrl.bind(store),
-        };
-    };
-    idOrDefault = (idOrName) => idOrName ?? this.config.get('defaultKeyValueStoreId');
-    getKeyValueStoreChangedValue = (idOrName, key) => {
-        const id = this.idOrDefault(idOrName);
-        this._keyValueStoreChanges[id] ??= {};
-        return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
-    };
-    setKeyValueStoreChangedValue = (idOrName, key, changedValue, options) => {
-        const id = this.idOrDefault(idOrName);
-        this._keyValueStoreChanges[id] ??= {};
-        this._keyValueStoreChanges[id][key] = { changedValue, options };
-    };
-}
-//# sourceMappingURL=crawler_commons.js.map
+export {};

package/crawlers/error_snapshotter.d.ts

@@ -25,9 +25,7 @@ interface ErrorSnapshot {
  * ```ts
  * const crawler = new BasicCrawler({
  *   // ...
- *   statisticsOptions: {
- *     saveErrorSnapshots: true,
- *   },
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
  * });
  * ```
  */
@@ -48,7 +46,7 @@ export declare class ErrorSnapshotter {
      */
     contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
     /**
-     * Save the HTML snapshot of the page, and return the fileName with the extension.
+     * Save the HTML snapshot of the page, and return the key it was stored under.
      */
     saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
     /**
@@ -57,4 +55,3 @@ export declare class ErrorSnapshotter {
     generateFilename(error: ErrnoException): string;
 }
 export {};
-//# sourceMappingURL=error_snapshotter.d.ts.map

package/crawlers/error_snapshotter.js

@@ -7,9 +7,7 @@ import crypto from 'node:crypto';
  * ```ts
  * const crawler = new BasicCrawler({
  *   // ...
- *   statisticsOptions: {
- *     saveErrorSnapshots: true,
- *   },
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
  * });
  * ```
  */
@@ -52,9 +50,9 @@ export class ErrorSnapshotter {
             }
             return {
                 screenshotFileName,
-                screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),
+                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
                 htmlFileName,
-                htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),
+                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
             };
         }
         catch {
@@ -79,12 +77,14 @@ export class ErrorSnapshotter {
         }
     }
     /**
-     * Save the HTML snapshot of the page, and return the fileName with the extension.
+     * Save the HTML snapshot of the page, and return the key it was stored under.
      */
     async saveHTMLSnapshot(html, keyValueStore, fileName) {
         try {
             await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
-            return `${fileName}.html`;
+            // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
+            // where local storage put the extension in the key) would break `getPublicUrl`.
+            return fileName;
         }
         catch {
             return undefined;
@@ -115,4 +115,3 @@ export class ErrorSnapshotter {
         return fileName;
     }
 }
-//# sourceMappingURL=error_snapshotter.js.map

package/crawlers/error_tracker.d.ts

@@ -52,4 +52,3 @@ export declare class ErrorTracker {
     captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
     reset(): void;
 }
-//# sourceMappingURL=error_tracker.d.ts.map

package/crawlers/error_tracker.js

@@ -306,4 +306,3 @@ export class ErrorTracker {
         }
     }
 }
-//# sourceMappingURL=error_tracker.js.map

package/crawlers/index.d.ts

@@ -1,7 +1,5 @@
 export * from './context_pipeline.js';
-export * from './crawler_commons.js';
-export * from './crawler_utils.js';
+export type * from './crawler_commons.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';
-//# sourceMappingURL=index.d.ts.map

package/crawlers/index.js

@@ -1,7 +1,4 @@
 export * from './context_pipeline.js';
-export * from './crawler_commons.js';
-export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';
-//# sourceMappingURL=index.js.map

package/crawlers/internals/types.d.ts

@@ -5,4 +5,3 @@ export interface SnapshottableProperties {
     body?: unknown;
     page?: BrowserPage;
 }
-//# sourceMappingURL=types.d.ts.map

package/crawlers/internals/types.js

@@ -1,2 +1 @@
 export {};
-//# sourceMappingURL=types.js.map