@crawlee/core 4.0.0-beta.1 → 4.0.0-beta.100

package/crawlers/context_pipeline.js

@@ -0,0 +1,122 @@
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
+import { serviceLocator } from '../service_locator.js';
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export class ContextPipeline {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create() {
+        return new ContextPipelineImpl({ action: async (context) => context });
+    }
+}
+/**
+ * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
+ * properties from the `ContextPipeline` interface, making type checking more reliable.
+ */
+class ContextPipelineImpl extends ContextPipeline {
+    middleware;
+    parent;
+    constructor(middleware, parent) {
+        super();
+        this.middleware = middleware;
+        this.parent = parent;
+    }
+    /**
+     * @inheritdoc
+     */
+    compose(middleware) {
+        return new ContextPipelineImpl(middleware, this);
+    }
+    chain(other) {
+        const otherMiddlewares = Array.from(other.middlewareChain()).reverse();
+        let result = this;
+        for (const middleware of otherMiddlewares) {
+            result = result.compose(middleware);
+        }
+        return result;
+    }
+    *middlewareChain() {
+        let step = this;
+        while (step !== undefined) {
+            yield step.middleware;
+            step = step.parent;
+        }
+    }
+    /**
+     * @inheritdoc
+     */
+    async call(crawlingContext, finalContextConsumer) {
+        const middlewares = Array.from(this.middlewareChain()).reverse();
+        const cleanupStack = [];
+        let consumerException;
+        try {
+            for (const { action, cleanup } of middlewares) {
+                try {
+                    const contextExtension = await action(crawlingContext);
+                    const extensionNames = [
+                        ...Object.getOwnPropertyNames(contextExtension),
+                        ...Object.getOwnPropertySymbols(contextExtension),
+                    ];
+                    for (const key of extensionNames) {
+                        try {
+                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                                Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
+                            }
+                        }
+                        catch (error) {
+                            serviceLocator
+                                .getLogger()
+                                .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                        }
+                    }
+                    if (cleanup) {
+                        cleanupStack.push(cleanup);
+                    }
+                }
+                catch (exception) {
+                    if (exception instanceof SessionError) {
+                        throw exception; // Session errors are re-thrown as-is
+                    }
+                    if (exception instanceof ContextPipelineInterruptedError) {
+                        throw exception;
+                    }
+                    throw new ContextPipelineInitializationError(exception);
+                }
+            }
+            try {
+                await finalContextConsumer(crawlingContext);
+            }
+            catch (exception) {
+                if (exception instanceof SessionError) {
+                    consumerException = exception;
+                    throw exception; // Session errors are re-thrown as-is
+                }
+                consumerException = exception;
+                throw new RequestHandlerError(exception);
+            }
+        }
+        finally {
+            try {
+                for (const cleanup of cleanupStack.reverse()) {
+                    await cleanup(crawlingContext, consumerException);
+                }
+            }
+            catch (exception) {
+                // eslint-disable-next-line no-unsafe-finally
+                throw new ContextPipelineCleanupError(exception);
+            }
+        }
+    }
+}

package/crawlers/crawler_commons.d.ts

@@ -1,19 +1,64 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { OptionsInit, Response as GotResponse } from 'got-scraping';
-import type { ReadonlyDeep } from 'type-fest';
+import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
-import type { Log } from '../log.js';
-import type { ProxyInfo } from '../proxy_configuration.js';
-import type { Request, Source } from '../request.js';
-import type { Session } from '../session_pool/session.js';
+import type { CrawleeLogger } from '../log.js';
+import type { Request, RequestOptions, Source } from '../request.js';
+import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 import type { Dataset } from '../storages/dataset.js';
 import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
-import type { RequestQueueOperationOptions } from '../storages/request_provider.js';
+import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
-/** @internal */
+/**
+ * A request input (URL string, request-options object, or {@link Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
+    requestsFromUrl?: string;
+    regex?: RegExp;
+} & ({
+    [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+        label: Label;
+    };
+}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
+    label?: undefined;
+})));
+/**
+ * The iterable/array of {@link LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ * @internal
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> = AsyncIterable<LabeledSource<Routes>> | Iterable<LabeledSource<Routes>> | LabeledSource<Routes>[];
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
+    [Label in keyof Routes & string]: {
+        label: Label;
+        userData?: Routes[Label];
+    };
+}[keyof Routes & string] | {
+    label?: undefined;
+    userData?: Dictionary;
+});
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ */
+export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
 export type WithRequired<T, K extends keyof T> = T & {
     [P in K]-?: T[P];
 };
@@ -22,9 +67,9 @@ export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl
 export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
     request: LoadedRequest<Context['request']>;
 } & Omit<Context, 'request'>;
-export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
     id: string;
-    session?: Session;
+    session: ISession;
     /**
      * An object with information about currently used proxy by the crawler
      * and configured by the {@link ProxyConfiguration} class.
@@ -41,13 +86,12 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      *
      * @param [data] Data to be pushed to the default dataset.
      */
-    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdOrName?: string): Promise<void>;
+    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
      * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -57,7 +101,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      * ```ts
      * async requestHandler({ enqueueLinks }) {
      *     await enqueueLinks({
-     *       globs: [
+     *       include: [
      *           'https://www.example.com/handbags/*',
      *       ],
      *     });
@@ -66,7 +110,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      *
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
      */
-    enqueueLinks: (options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>>) => Promise<unknown>;
+    enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
     /**
      * Add requests directly to the request queue.
      *
@@ -81,20 +125,18 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     /**
      * Get a key-value store with given name or id, or the default one for the crawler.
      */
-    getKeyValueStore: (idOrName?: string) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
+    getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
     /**
      * A preconfigured logger for the request handler.
      */
-    log: Log;
+    log: CrawleeLogger;
 }
-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
-    crawler: Crawler;
+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
     /**
      * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -104,7 +146,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * ```ts
      * async requestHandler({ enqueueLinks }) {
      *     await enqueueLinks({
-     *       globs: [
+     *       include: [
      *           'https://www.example.com/handbags/*',
      *       ],
      *     });
@@ -114,14 +156,9 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinks(options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
     /**
-     * Get a key-value store with given name or id, or the default one for the crawler.
-     */
-    getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;
-    /**
-     * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
-     * options on the fly.
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *
      * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
      * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
@@ -136,7 +173,30 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * },
      * ```
      */
-    sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
+    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
+    /**
+     * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     */
+    registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
+    /**
+     * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
+     * once it is already running - a listing page that turns out to have far more to scroll through than
+     * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
+     * {@link Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
+     *
+     * ```ts
+     * router.addHandler('LIST', async ({ extendTimeout, page }) => {
+     *     const pageCount = await countPages(page);
+     *     extendTimeout(pageCount * 10);
+     *     await scrapeAllPages(page);
+     * });
+     * ```
+     *
+     * Extends the request handler's own timeout and the crawler's internal one together, so the extension
+     * is not immediately undone by the latter. Calling it from a handler that has already timed out does
+     * nothing.
+     */
+    extendTimeout(secs: number): void;
 }
 /**
  * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
@@ -144,20 +204,18 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
  * @experimental
  */
 export declare class RequestHandlerResult {
-    private config;
+    private configuration;
     private crawleeStateKey;
     private _keyValueStoreChanges;
     private pushDataCalls;
     private addRequestsCalls;
-    private enqueueLinksCalls;
-    constructor(config: Configuration, crawleeStateKey: string);
+    constructor(configuration: Configuration, crawleeStateKey: string);
     /**
      * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
      */
     get calls(): ReadonlyDeep<{
         pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
         addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
-        enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
     }>;
     /**
      * A record of changes made to key-value stores by a request handler.
@@ -171,7 +229,7 @@ export declare class RequestHandlerResult {
      */
     get datasetItems(): ReadonlyDeep<{
         item: Dictionary;
-        datasetIdOrName?: string;
+        datasetIdentifier?: string | StorageIdentifier;
     }[]>;
     /**
      * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
@@ -188,12 +246,10 @@ export declare class RequestHandlerResult {
         label?: string;
     }[]>;
     pushData: RestrictedCrawlingContext['pushData'];
-    enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
     addRequests: RestrictedCrawlingContext['addRequests'];
     useState: RestrictedCrawlingContext['useState'];
     getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
-    private idOrDefault;
     private getKeyValueStoreChangedValue;
     private setKeyValueStoreChangedValue;
 }
-//# sourceMappingURL=crawler_commons.d.ts.map
+export {};

package/crawlers/crawler_commons.js

@@ -5,14 +5,13 @@ import { KeyValueStore } from '../storages/key_value_store.js';
  * @experimental
  */
 export class RequestHandlerResult {
-    config;
+    configuration;
     crawleeStateKey;
     _keyValueStoreChanges = {};
     pushDataCalls = [];
     addRequestsCalls = [];
-    enqueueLinksCalls = [];
-    constructor(config, crawleeStateKey) {
-        this.config = config;
+    constructor(configuration, crawleeStateKey) {
+        this.configuration = configuration;
         this.crawleeStateKey = crawleeStateKey;
     }
     /**
@@ -22,7 +21,6 @@ export class RequestHandlerResult {
         return {
             pushData: this.pushDataCalls,
             addRequests: this.addRequestsCalls,
-            enqueueLinks: this.enqueueLinksCalls,
         };
     }
     /**
@@ -35,16 +33,13 @@ export class RequestHandlerResult {
      * Items added to datasets by a request handler.
      */
     get datasetItems() {
-        return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
+        return this.pushDataCalls.flatMap(([data, datasetIdentifier]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdentifier })));
     }
     /**
      * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
      */
     get enqueuedUrls() {
         const result = [];
-        for (const [options] of this.enqueueLinksCalls) {
-            result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
-        }
         for (const [requests] of this.addRequestsCalls) {
             for (const request of requests) {
                 if (typeof request === 'object' &&
@@ -78,9 +73,6 @@ export class RequestHandlerResult {
     pushData = async (data, datasetIdOrName) => {
         this.pushDataCalls.push([data, datasetIdOrName]);
     };
-    enqueueLinks = async (options) => {
-        this.enqueueLinksCalls.push([options]);
-    };
     addRequests = async (requests, options = {}) => {
         this.addRequestsCalls.push([requests, options]);
     };
@@ -88,29 +80,28 @@ export class RequestHandlerResult {
         const store = await this.getKeyValueStore(undefined);
         return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
     };
-    getKeyValueStore = async (idOrName) => {
-        const store = await KeyValueStore.open(idOrName, { config: this.config });
+    getKeyValueStore = async (identifier) => {
+        const store = await KeyValueStore.open(identifier, { configuration: this.configuration });
+        const storeId = store.id;
         return {
-            id: this.idOrDefault(idOrName),
-            name: idOrName,
-            getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? (await store.getValue(key)),
+            id: storeId ?? this.configuration.defaultKeyValueStoreId,
+            name: store.name,
+            getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
             setValue: async (key, value, options) => {
-                this.setKeyValueStoreChangedValue(idOrName, key, value, options);
+                this.setKeyValueStoreChangedValue(storeId, key, value, options);
             },
             getAutoSavedValue: store.getAutoSavedValue.bind(store),
             getPublicUrl: store.getPublicUrl.bind(store),
         };
     };
-    idOrDefault = (idOrName) => idOrName ?? this.config.get('defaultKeyValueStoreId');
-    getKeyValueStoreChangedValue = (idOrName, key) => {
-        const id = this.idOrDefault(idOrName);
+    getKeyValueStoreChangedValue = (storeKey, key) => {
+        const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
         this._keyValueStoreChanges[id] ??= {};
         return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
     };
-    setKeyValueStoreChangedValue = (idOrName, key, changedValue, options) => {
-        const id = this.idOrDefault(idOrName);
+    setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
+        const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
         this._keyValueStoreChanges[id] ??= {};
         this._keyValueStoreChanges[id][key] = { changedValue, options };
     };
 }
-//# sourceMappingURL=crawler_commons.js.map

package/crawlers/error_snapshotter.d.ts

@@ -1,6 +1,7 @@
 import type { CrawlingContext } from '../crawlers/crawler_commons.js';
 import type { KeyValueStore } from '../storages/key_value_store.js';
 import type { ErrnoException } from './error_tracker.js';
+import type { SnapshottableProperties } from './internals/types.js';
 interface BrowserCrawlingContext {
     saveSnapshot: (options: {
         key: string;
@@ -39,7 +40,7 @@ export declare class ErrorSnapshotter {
     /**
      * Capture a snapshot of the error context.
      */
-    captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot>;
+    captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
     /**
      * Captures a snapshot of the current page using the context.saveSnapshot function.
      * This function is applicable for browser contexts only.
@@ -49,11 +50,10 @@ export declare class ErrorSnapshotter {
     /**
      * Save the HTML snapshot of the page, and return the fileName with the extension.
      */
-    saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined>;
+    saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
     /**
      * Generate a unique fileName for each error snapshot.
      */
     generateFilename(error: ErrnoException): string;
 }
 export {};
-//# sourceMappingURL=error_snapshotter.d.ts.map

package/crawlers/error_snapshotter.js

@@ -52,9 +52,9 @@ export class ErrorSnapshotter {
             }
             return {
                 screenshotFileName,
-                screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),
+                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
                 htmlFileName,
-                htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),
+                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
             };
         }
         catch {
@@ -115,4 +115,3 @@ export class ErrorSnapshotter {
         return fileName;
     }
 }
-//# sourceMappingURL=error_snapshotter.js.map

package/crawlers/error_tracker.d.ts

@@ -1,5 +1,6 @@
 import type { CrawlingContext } from '../crawlers/crawler_commons.js';
 import { ErrorSnapshotter } from './error_snapshotter.js';
+import type { SnapshottableProperties } from './internals/types.js';
 /**
  * Node.js Error interface
  */
@@ -48,7 +49,6 @@ export declare class ErrorTracker {
     addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
     getUniqueErrorCount(): number;
     getMostPopularErrors(count: number): [number, string[]][];
-    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext): Promise<void>;
+    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
     reset(): void;
 }
-//# sourceMappingURL=error_tracker.d.ts.map

package/crawlers/error_tracker.js

@@ -306,4 +306,3 @@ export class ErrorTracker {
         }
     }
 }
-//# sourceMappingURL=error_tracker.js.map

package/crawlers/index.d.ts

@@ -1,7 +1,5 @@
+export * from './context_pipeline.js';
 export * from './crawler_commons.js';
-export * from './crawler_extension.js';
-export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';
-//# sourceMappingURL=index.d.ts.map

package/crawlers/index.js

@@ -1,7 +1,5 @@
+export * from './context_pipeline.js';
 export * from './crawler_commons.js';
-export * from './crawler_extension.js';
-export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';
-//# sourceMappingURL=index.js.map

package/crawlers/internals/types.d.ts

@@ -0,0 +1,7 @@
+export interface BrowserPage {
+    content: () => Promise<string>;
+}
+export interface SnapshottableProperties {
+    body?: unknown;
+    page?: BrowserPage;
+}

package/crawlers/internals/types.js

@@ -0,0 +1 @@
+export {};