@crawlee/core 4.0.0-beta.9 → 4.0.0-beta.90

package/crawlers/statistics.d.ts

@@ -1,5 +1,4 @@
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
+import type { CrawleeLogger } from '../log.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
@@ -17,7 +16,7 @@ export interface PersistenceOptions {
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
@@ -27,15 +26,15 @@ export declare class Statistics {
     /**
      * An error tracker for final retry errors.
      */
-    errorTracker: ErrorTracker;
+    readonly errorTracker: ErrorTracker;
     /**
      * An error tracker for retry errors prior to the final retry.
      */
-    errorTrackerRetry: ErrorTracker;
+    readonly errorTrackerRetry: ErrorTracker;
     /**
      * Statistic instance id.
      */
-    readonly id: number;
+    readonly id: string;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
@@ -44,12 +43,8 @@ export declare class Statistics {
      * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
      */
     readonly requestRetryHistogram: number[];
-    /**
-     * Contains the associated Configuration instance
-     */
-    private readonly config;
     protected keyValueStore?: KeyValueStore;
-    protected persistStateKey: string;
+    protected readonly persistStateKey: string;
     private logIntervalMillis;
     private logMessage;
     private listener;
@@ -57,8 +52,9 @@ export declare class Statistics {
     private readonly log;
     private instanceStart;
     private logInterval;
-    private events;
+    private _events?;
     private persistenceOptions;
+    private get events();
     /**
      * @internal
      */
@@ -90,6 +86,12 @@ export declare class Statistics {
      * @ignore
      */
     failJob(id: number | string, retryCount: number): void;
+    /**
+     * Discards a started job without affecting the finished/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * @ignore
+     */
+    discardJob(id: number | string): void;
     /**
      * Calculate the current statistics
      */
@@ -111,7 +113,7 @@ export declare class Statistics {
      * Stops logging and remove event listeners, then persist
      */
     stopCapturing(): Promise<void>;
-    protected _saveRetryCountForJob(retryCount: number): void;
+    private saveRetryCountForJob;
     /**
      * Persist internal state to the key value store
      * @param options - Override the persistence options provided in the constructor
@@ -120,8 +122,8 @@ export declare class Statistics {
     /**
      * Loads the current statistic from the key value store if any
      */
-    protected _maybeLoadStatistics(): Promise<void>;
-    protected _teardown(): void;
+    protected maybeLoadStatistics(): Promise<void>;
+    private teardown;
     /**
      * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
      * or through `keyValueStore.setValue('KEY', statsInstance)`
@@ -146,17 +148,12 @@ export interface StatisticsOptions {
      * Parent logger instance, the statistics will create a child logger from this.
      * @default crawler.log
      */
-    log?: Log;
+    log?: CrawleeLogger;
     /**
      * Key value store instance to persist the statistics.
      * If not provided, the default one will be used when capturing starts
      */
     keyValueStore?: KeyValueStore;
-    /**
-     * Configuration instance to use
-     * @default Configuration.getGlobalConfig()
-     */
-    config?: Configuration;
     /**
      * Control how and when to persist the statistics.
      */
@@ -166,13 +163,22 @@ export interface StatisticsOptions {
      * @default false
      */
     saveErrorSnapshots?: boolean;
+    /**
+     * A unique identifier for this statistics instance. This ID is used for persistence
+     * to the key value store, ensuring the same statistics can be loaded after script restarts.
+     *
+     * If not provided, an auto-incremented ID will be used for backward compatibility.
+     * This means statistics may not persist correctly across script restarts
+     * if crawler creation order changes.
+     */
+    id?: string;
 }
 /**
  * Format of the persisted stats
  */
 export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
     requestRetryHistogram: number[];
-    statsId: number;
+    statsId: string;
     requestAvgFailedDurationMillis: number;
     requestAvgFinishedDurationMillis: number;
     requestTotalDurationMillis: number;
@@ -201,4 +207,3 @@ export interface StatisticState {
     retryErrors: Record<string, unknown>;
     requestsWithStatusCode: Record<string, number>;
 }
-//# sourceMappingURL=statistics.d.ts.map

package/crawlers/statistics.js

@@ -1,6 +1,5 @@
 import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
@@ -30,7 +29,7 @@ const errorTrackerConfig = {
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
@@ -48,7 +47,7 @@ export class Statistics {
     /**
      * Statistic instance id.
      */
-    id = Statistics.id++; // assign an id while incrementing so it can be saved/restored from KV
+    id;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
@@ -57,12 +56,8 @@ export class Statistics {
      * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
      */
     requestRetryHistogram = [];
-    /**
-     * Contains the associated Configuration instance
-     */
-    config;
     keyValueStore = undefined;
-    persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
+    persistStateKey;
     logIntervalMillis;
     logMessage;
     listener;
@@ -70,8 +65,14 @@ export class Statistics {
     log;
     instanceStart;
     logInterval;
-    events;
+    _events;
     persistenceOptions;
+    get events() {
+        if (!this._events) {
+            this._events = serviceLocator.getEventManager();
+        }
+        return this._events;
+    }
     /**
      * @internal
      */
@@ -81,22 +82,22 @@ export class Statistics {
             logMessage: ow.optional.string,
             log: ow.optional.object,
             keyValueStore: ow.optional.object,
-            config: ow.optional.object,
             persistenceOptions: ow.optional.object,
             saveErrorSnapshots: ow.optional.boolean,
+            id: ow.optional.any(ow.number, ow.string),
         }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = Configuration.getGlobalConfig(), persistenceOptions = {
+        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
             enable: true,
-        }, saveErrorSnapshots = false, } = options;
-        this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
+        }, saveErrorSnapshots = false, id, } = options;
+        this.id = id ?? String(Statistics.id++);
+        this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
+        this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.logIntervalMillis = logIntervalSecs * 1000;
         this.logMessage = logMessage;
         this.keyValueStore = keyValueStore;
         this.listener = this.persistState.bind(this);
-        this.events = config.getEventManager();
-        this.config = config;
         this.persistenceOptions = persistenceOptions;
         // initialize by "resetting"
         this.reset();
@@ -128,7 +129,7 @@ export class Statistics {
         this.requestRetryHistogram.length = 0;
         this.requestsInProgress.clear();
         this.instanceStart = Date.now();
-        this._teardown();
+        this.teardown();
     }
     /**
      * @param options - Override the persistence options provided in the constructor
@@ -174,7 +175,7 @@ export class Statistics {
         const jobDurationMillis = job.finish();
         this.state.requestsFinished++;
         this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
-        this._saveRetryCountForJob(retryCount);
+        this.saveRetryCountForJob(retryCount);
         if (jobDurationMillis < this.state.requestMinDurationMillis)
             this.state.requestMinDurationMillis = jobDurationMillis;
         if (jobDurationMillis > this.state.requestMaxDurationMillis)
@@ -191,7 +192,15 @@ export class Statistics {
             return;
         this.state.requestTotalFailedDurationMillis += job.finish();
         this.state.requestsFailed++;
-        this._saveRetryCountForJob(retryCount);
+        this.saveRetryCountForJob(retryCount);
+        this.requestsInProgress.delete(id);
+    }
+    /**
+     * Discards a started job without affecting the finished/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * @ignore
+     */
+    discardJob(id) {
         this.requestsInProgress.delete(id);
     }
     /**
@@ -216,12 +225,12 @@ export class Statistics {
      * displaying the current state in predefined intervals
      */
     async startCapturing() {
-        this.keyValueStore ??= await KeyValueStore.open(null, { config: this.config });
+        this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
         if (this.state.crawlerStartedAt === null) {
             this.state.crawlerStartedAt = new Date();
         }
         if (this.persistenceOptions.enable) {
-            await this._maybeLoadStatistics();
+            await this.maybeLoadStatistics();
             this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
         }
         this.logInterval = setInterval(() => {
@@ -235,11 +244,11 @@ export class Statistics {
      * Stops logging and remove event listeners, then persist
      */
     async stopCapturing() {
-        this._teardown();
+        this.teardown();
         this.state.crawlerFinishedAt = new Date();
         await this.persistState();
     }
-    _saveRetryCountForJob(retryCount) {
+    saveRetryCountForJob(retryCount) {
         if (retryCount > 0)
             this.state.requestsRetries++;
         this.requestRetryHistogram[retryCount] ??= 0;
@@ -258,20 +267,14 @@ export class Statistics {
             return;
         }
         this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
         await this.keyValueStore
-            .setValue(this.persistStateKey, this.toJSON(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
+            .setValue(this.persistStateKey, this.toJSON())
             .catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
     }
     /**
      * Loads the current statistic from the key value store if any
      */
-    async _maybeLoadStatistics() {
+    async maybeLoadStatistics() {
         // this might be called before startCapturing was called without using await, should not crash
         if (!this.keyValueStore) {
             return;
@@ -306,9 +309,11 @@ export class Statistics {
         this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
         this.log.debug('Loaded from KeyValueStore');
     }
-    _teardown() {
+    teardown() {
         // this can be called before a call to startCapturing happens (or in a 'finally' block)
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
+        // (e.g. during the constructor's reset() call, which would capture the wrong context)
+        this._events?.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
         if (this.logInterval) {
             clearInterval(this.logInterval);
             this.logInterval = null;
@@ -343,4 +348,3 @@ export class Statistics {
         return result;
     }
 }
-//# sourceMappingURL=statistics.js.map

package/debug.d.ts

@@ -0,0 +1,36 @@
+import type { IncomingMessage } from 'node:http';
+import type { Dictionary } from '@crawlee/types';
+import type { Request } from './request.js';
+interface BrowserResponseLike {
+    status(): number;
+}
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
+ * @param [response]
+ *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
+ *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
+ * @param [additionalFields] Object containing additional fields to be added.
+ *
+ * @internal
+ */
+export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
+/**
+ * Returns a human-readable label for an unknown value,
+ * suitable for embedding in error messages and log output.
+ *
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
+ *
+ * @internal
+ */
+export declare function inspectValue(value: unknown): string;
+/**
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
+ * by their constructor name. Used for building validation error messages.
+ *
+ * @internal
+ */
+export declare function getObjectType(value: unknown): string;
+export {};

package/debug.js

@@ -0,0 +1,70 @@
+import { inspect } from 'node:util';
+import ow from 'ow';
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
+ * @param [response]
+ *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
+ *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
+ * @param [additionalFields] Object containing additional fields to be added.
+ *
+ * @internal
+ */
+export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
+    ow(request, ow.object);
+    ow(response, ow.object);
+    ow(additionalFields, ow.object);
+    return {
+        requestId: request.id,
+        url: request.url,
+        loadedUrl: request.loadedUrl,
+        method: request.method,
+        retryCount: request.retryCount,
+        errorMessages: request.errorMessages,
+        // Puppeteer response has .status() function and NodeJS response, statusCode property.
+        statusCode: 'status' in response && response.status instanceof Function
+            ? response.status()
+            : response.statusCode,
+        ...additionalFields,
+    };
+}
+/**
+ * Returns a human-readable label for an unknown value,
+ * suitable for embedding in error messages and log output.
+ *
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
+ *
+ * @internal
+ */
+export function inspectValue(value) {
+    if (typeof value === 'object' && value !== null && value.constructor?.name) {
+        return value.constructor.name;
+    }
+    return inspect(value, {
+        depth: 0,
+        compact: true,
+        maxStringLength: 64,
+        breakLength: Infinity,
+        colors: false,
+    });
+}
+/**
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
+ * by their constructor name. Used for building validation error messages.
+ *
+ * @internal
+ */
+export function getObjectType(value) {
+    const simple = typeof value;
+    if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
+        return simple;
+    }
+    const objectType = Object.prototype.toString.call(value);
+    const type = /\[object (\w+)]/.exec(objectType)[1];
+    if (type === 'Uint8Array') {
+        return 'Buffer';
+    }
+    return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
+}

package/enqueue_links/enqueue_links.d.ts

@@ -1,25 +1,30 @@
-import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 import { type RobotsTxtFile } from '@crawlee/utils';
 import type { SetRequired } from 'type-fest';
-import type { RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform } from './shared.js';
-export type SkippedRequestCallback = (args: {
-    url: string;
-    reason: 'robotsTxt';
-}) => Awaitable<void>;
+import { Request } from '../request.js';
+import type { IRequestManager } from '../storages/request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
     /** An array of URLs to enqueue. */
     urls?: readonly string[];
-    /** A request queue to which the URLs will be enqueued. */
-    requestQueue?: RequestProvider;
+    /** A request manager to which the URLs will be enqueued. */
+    requestManager?: IRequestManager;
     /** A CSS selector matching links to be enqueued. */
     selector?: string;
     /** Sets {@link Request.userData} for newly enqueued requests. */
     userData?: Dictionary;
-    /** Sets {@link Request.label} for newly enqueued requests. */
+    /**
+     * Sets {@link Request.label} for newly enqueued requests.
+     *
+     * This option has the lowest priority and can be overwritten by request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
+     */
     label?: string;
+    /** Sets {@link Request.sessionId} for newly enqueued requests. */
+    sessionId?: string;
     /**
      * If set to `true`, tells the crawler to skip navigation and process the request directly.
      * @default false
@@ -49,7 +54,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * containing patterns matching URLs that will **never** be enqueued.
      *
      * The plain objects must include either the `glob` property or the `regexp` property.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
      *
      * Glob matching is always case-insensitive.
      * If you need case-sensitive matching, provide a regexp.
@@ -86,12 +90,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     pseudoUrls?: readonly PseudoUrlInput[];
     /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * After request options are filtered by patterns, this function can be used
+     * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
      * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
      * or to dynamically update or create `userData`.
      *
-     * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
+     * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
      * when `uniqueKey` is computed.
      *
      * **Example:**
@@ -105,9 +109,13 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has a priority over request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects,
-     * and thus some options could be over-written by `transformRequestFunction`.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
     /**
@@ -138,10 +146,21 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
      * If provided, disallowed URLs will be ignored.
      */
-    robotsTxtFile?: RobotsTxtFile;
+    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
+    /**
+     * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
+     * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
+     * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
+     */
+    respectRobotsTxtFile?: boolean | {
+        userAgent?: string;
+    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
-     * This is currently fired only for requests skipped based on robots.txt file.
+     * This is currently fired for requests skipped
+     * 1. based on robots.txt file,
+     * 2. because they don't match enqueueLinks filters,
+     * 3. or because the maxRequestsPerCrawl limit has been reached
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
@@ -208,7 +227,7 @@ export declare enum EnqueueStrategy {
  * ```javascript
  * await enqueueLinks({
  *   urls: aListOfFoundUrls,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   globs: [
  *       'https://www.example.com/handbags/*',
@@ -220,7 +239,11 @@ export declare enum EnqueueStrategy {
  * @param options All `enqueueLinks()` parameters are passed via an options object.
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
  */
-export declare function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'requestQueue' | 'urls'>): Promise<BatchAddRequestsResult>;
+export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
+    requestManager: {
+        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
+    };
+}): Promise<BatchAddRequestsResult>;
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -239,4 +262,3 @@ export interface ResolveBaseUrl {
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
-//# sourceMappingURL=enqueue_links.d.ts.map