@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/crawlers/statistics.js

@@ -1,20 +1,20 @@
 import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
+import { z } from 'zod';
+import { RecoverableState } from '../recoverable_state.js';
+import { serviceLocator } from '../service_locator.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
  * @ignore
  */
 class Job {
-    lastRunAt = null;
-    durationMillis;
+    #lastRunAt = null;
+    #durationMillis;
     run() {
-        this.lastRunAt = Date.now();
+        this.#lastRunAt = Date.now();
     }
     finish() {
-        this.durationMillis = Date.now() - this.lastRunAt;
-        return this.durationMillis;
+        this.#durationMillis = Date.now() - this.#lastRunAt;
+        return this.#durationMillis;
     }
 }
 const errorTrackerConfig = {
@@ -25,17 +25,127 @@ const errorTrackerConfig = {
     showErrorMessage: true,
     showFullMessage: false,
 };
+/**
+ * The persisted record, in the order it is written - the schema rebuilds the object on the way out, so the field
+ * order here *is* the record's field order (guarded by a test).
+ *
+ * JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as
+ * `null`. Both {@link Statistics.serializeState} and {@link Statistics.deserializeState} run through this,
+ * which is what keeps them describing the same record.
+ *
+ * Nothing is optional on purpose: `serializeState` has always written every field, so a record missing one is not
+ * one of ours and is discarded whole rather than partially trusted - a counter restored as a string would poison
+ * every later increment.
+ */
+const persistedStatisticState = z
+    .object({
+    requestsFinished: z.number(),
+    requestsFailed: z.number(),
+    requestsRetries: z.number(),
+    requestsFailedPerMinute: z.number().nullable(),
+    requestsFinishedPerMinute: z.number().nullable(),
+    requestMinDurationMillis: z.number().nullable(),
+    requestMaxDurationMillis: z.number(),
+    requestTotalFailedDurationMillis: z.number(),
+    requestTotalFinishedDurationMillis: z.number(),
+    crawlerStartedAt: z.string().nullable(),
+    crawlerFinishedAt: z.string().nullable(),
+    statsPersistedAt: z.string(),
+    crawlerRuntimeMillis: z.number(),
+    crawlerLastStartTimestamp: z.number(),
+    // A retry count that never occurred leaves a hole in the live histogram, written out as a `null`. We
+    // once saw a record whose histogram was not an array at all and crashed the crawler on load.
+    requestRetryHistogram: z.array(z.number().nullable()),
+    statsId: z.string(),
+    requestAvgFailedDurationMillis: z.number().nullable(),
+    requestAvgFinishedDurationMillis: z.number().nullable(),
+    requestTotalDurationMillis: z.number(),
+    requestsTotal: z.number(),
+    requestsWithStatusCode: z.record(z.string(), z.number()),
+    errors: z.record(z.string(), z.unknown()),
+    retryErrors: z.record(z.string(), z.unknown()),
+})
+    // A subclass tracking extra fields spreads them into the record; they are none of this schema's business,
+    // but they must not be dropped on the way through it.
+    .catchall(z.unknown());
+/** `Infinity` is what the statistics use for "nothing to average yet"; JSON has only `null` for it. */
+function finiteOrNull(value) {
+    return Number.isFinite(value) ? value : null;
+}
+/**
+ * The conversion between the live state and the record above, in both directions.
+ *
+ * Built per instance rather than kept as a constant because a record carries three things the state does not: the
+ * instance `id`, the derived aggregates of the overridable {@link Statistics.calculate}, and - on the way back -
+ * the fields that are rebuilt from {@link Statistics.defaultState} rather than restored, the error trackers
+ * among them.
+ *
+ * The model side is deliberately opaque: zod rebuilds what it validates, and `state.errors` has to stay the very
+ * object the error trackers write into, not a copy of it.
+ */
+function buildStatisticStateCodec(statistics) {
+    return z.codec(persistedStatisticState, z.custom(), {
+        decode: (record) => ({
+            ...statistics.defaultState(),
+            requestsFinished: record.requestsFinished,
+            requestsFailed: record.requestsFailed,
+            requestsRetries: record.requestsRetries,
+            requestTotalFailedDurationMillis: record.requestTotalFailedDurationMillis,
+            requestTotalFinishedDurationMillis: record.requestTotalFinishedDurationMillis,
+            // Restoring the `null` as-is would make every later `duration < min` comparison fail, leaving the
+            // minimum `null` for the rest of the run.
+            requestMinDurationMillis: record.requestMinDurationMillis ?? Infinity,
+            requestMaxDurationMillis: record.requestMaxDurationMillis,
+            crawlerRuntimeMillis: record.crawlerRuntimeMillis,
+            // A `null` stands for the zero requests that reached that retry count - restore it as such.
+            requestRetryHistogram: record.requestRetryHistogram.map((count) => count ?? 0),
+            // The record keeps ISO strings, the live state keeps `Date`s.
+            crawlerStartedAt: record.crawlerStartedAt === null ? null : new Date(record.crawlerStartedAt),
+            crawlerFinishedAt: record.crawlerFinishedAt === null ? null : new Date(record.crawlerFinishedAt),
+            statsPersistedAt: new Date(record.statsPersistedAt),
+            // Rebased so that the runtime reported by `calculate()` spans the migration instead of restarting.
+            instanceStart: Date.now() - (new Date(record.statsPersistedAt).getTime() - record.crawlerLastStartTimestamp),
+        }),
+        encode: (state) => {
+            const { requestsWithStatusCode, errors, retryErrors, requestRetryHistogram, instanceStart, ...counters } = state;
+            // Every rate and average `calculate()` derives is `Infinity` until the run is long enough, or until
+            // something has finished or failed, to divide by.
+            const { requestAvgFailedDurationMillis, requestAvgFinishedDurationMillis, requestsFailedPerMinute, requestsFinishedPerMinute, ...aggregates } = statistics.calculate();
+            return {
+                ...counters,
+                requestMinDurationMillis: finiteOrNull(state.requestMinDurationMillis),
+                crawlerStartedAt: state.crawlerStartedAt ? new Date(state.crawlerStartedAt).toISOString() : null,
+                crawlerFinishedAt: state.crawlerFinishedAt ? new Date(state.crawlerFinishedAt).toISOString() : null,
+                statsPersistedAt: new Date().toISOString(),
+                crawlerLastStartTimestamp: instanceStart,
+                // `Array.from`, not `map` - a hole left by a retry count no request reached is skipped by `map`
+                // and would stay a hole, which is not something the record can carry.
+                requestRetryHistogram: Array.from(requestRetryHistogram, (count) => count ?? null),
+                statsId: statistics.statsId,
+                ...aggregates,
+                requestAvgFailedDurationMillis: finiteOrNull(requestAvgFailedDurationMillis),
+                requestAvgFinishedDurationMillis: finiteOrNull(requestAvgFinishedDurationMillis),
+                requestsFailedPerMinute: finiteOrNull(requestsFailedPerMinute),
+                requestsFinishedPerMinute: finiteOrNull(requestsFinishedPerMinute),
+                requestsWithStatusCode,
+                errors,
+                retryErrors,
+            };
+        },
+    });
+}
 /**
  * The statistics class provides an interface to collecting and logging run
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
  */
 export class Statistics {
+    // kept as TS-private: statistics tests read the static counter directly
     static id = 0;
     /**
      * An error tracker for final retry errors.
@@ -48,32 +158,30 @@ export class Statistics {
     /**
      * Statistic instance id.
      */
-    id = Statistics.id++; // assign an id while incrementing so it can be saved/restored from KV
+    id;
+    persistStateKey;
+    #stateCodec;
+    #recoverableState;
+    #logIntervalMillis;
+    #logMessage;
+    #requestsInProgress = new Map();
+    log;
+    #logInterval;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
-    state;
+    get state() {
+        return this.#recoverableState.currentValue;
+    }
     /**
      * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
      */
-    requestRetryHistogram = [];
-    /**
-     * Contains the associated Configuration instance
-     */
-    config;
-    keyValueStore = undefined;
-    persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
-    logIntervalMillis;
-    logMessage;
-    listener;
-    requestsInProgress = new Map();
-    log;
-    instanceStart;
-    logInterval;
-    events;
-    persistenceOptions;
+    get requestRetryHistogram() {
+        return this.state.requestRetryHistogram;
+    }
     /**
-     * @internal
+     * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
+     * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
      */
     constructor(options = {}) {
         ow(options, ow.object.exactShape({
@@ -81,33 +189,56 @@ export class Statistics {
             logMessage: ow.optional.string,
             log: ow.optional.object,
             keyValueStore: ow.optional.object,
-            config: ow.optional.object,
             persistenceOptions: ow.optional.object,
             saveErrorSnapshots: ow.optional.boolean,
+            id: ow.optional.any(ow.number, ow.string),
         }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = Configuration.getGlobalConfig(), persistenceOptions = {
+        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
             enable: true,
-        }, saveErrorSnapshots = false, } = options;
-        this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
+        }, saveErrorSnapshots = false, id, } = options;
+        this.id = id ?? String(Statistics.id++);
+        this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
+        this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
-        this.logIntervalMillis = logIntervalSecs * 1000;
-        this.logMessage = logMessage;
-        this.keyValueStore = keyValueStore;
-        this.listener = this.persistState.bind(this);
-        this.events = config.getEventManager();
-        this.config = config;
-        this.persistenceOptions = persistenceOptions;
+        this.#logIntervalMillis = logIntervalSecs * 1000;
+        this.#logMessage = logMessage;
+        // Late-bound on purpose - both hooks are override points, and a subclass's must be the ones that run.
+        this.#stateCodec = buildStatisticStateCodec({
+            statsId: this.id,
+            defaultState: () => this.defaultState(),
+            calculate: () => this.calculate(),
+        });
+        this.#recoverableState = new RecoverableState({
+            persistStateKey: this.persistStateKey,
+            persistenceEnabled: persistenceOptions.enable,
+            keyValueStore,
+            logger: this.log,
+            defaultState: () => this.defaultState(),
+            serialize: (state) => this.serializeState(state),
+            deserialize: (persistedState) => this.deserializeState(persistedState),
+        });
         // initialize by "resetting"
         this.reset();
     }
     /**
-     * Set the current statistic instance to pristine values
+     * Set the current statistic instance to pristine values.
+     *
+     * The persisted record is left alone - use {@link Statistics.resetStore} to clear that as well.
      */
     reset() {
         this.errorTracker.reset();
         this.errorTrackerRetry.reset();
-        this.state = {
+        this.#recoverableState.reset();
+        this.#requestsInProgress.clear();
+    }
+    /**
+     * The pristine state a new instance starts with and {@link Statistics.reset} restores.
+     *
+     * A subclass tracking extra fields declares their initial values here.
+     */
+    defaultState() {
+        return {
             requestsFinished: 0,
             requestsFailed: 0,
             requestsRetries: 0,
@@ -122,25 +253,20 @@ export class Statistics {
             statsPersistedAt: null,
             crawlerRuntimeMillis: 0,
             requestsWithStatusCode: {},
+            // Aliases, not copies - the trackers keep writing into these objects.
             errors: this.errorTracker.result,
             retryErrors: this.errorTrackerRetry.result,
+            requestRetryHistogram: [],
+            instanceStart: Date.now(),
         };
-        this.requestRetryHistogram.length = 0;
-        this.requestsInProgress.clear();
-        this.instanceStart = Date.now();
-        this._teardown();
     }
     /**
-     * @param options - Override the persistence options provided in the constructor
+     * Clear the persisted statistics record, leaving the in-memory state alone.
+     *
+     * Throws while capturing - the next PERSIST_STATE event would write the record straight back.
      */
-    async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        if (!this.keyValueStore) {
-            return;
-        }
-        await this.keyValueStore.setValue(this.persistStateKey, null);
+    async resetStore() {
+        await this.#recoverableState.resetStore();
     }
     /**
      * Increments the status code counter.
@@ -157,49 +283,57 @@ export class Statistics {
      * @ignore
      */
     startJob(id) {
-        let job = this.requestsInProgress.get(id);
+        let job = this.#requestsInProgress.get(id);
         if (!job)
             job = new Job();
         job.run();
-        this.requestsInProgress.set(id, job);
+        this.#requestsInProgress.set(id, job);
     }
     /**
      * Mark job as finished and sets the state
      * @ignore
      */
     finishJob(id, retryCount) {
-        const job = this.requestsInProgress.get(id);
+        const job = this.#requestsInProgress.get(id);
         if (!job)
             return;
         const jobDurationMillis = job.finish();
         this.state.requestsFinished++;
         this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
-        this._saveRetryCountForJob(retryCount);
+        this.saveRetryCountForJob(retryCount);
         if (jobDurationMillis < this.state.requestMinDurationMillis)
             this.state.requestMinDurationMillis = jobDurationMillis;
         if (jobDurationMillis > this.state.requestMaxDurationMillis)
             this.state.requestMaxDurationMillis = jobDurationMillis;
-        this.requestsInProgress.delete(id);
+        this.#requestsInProgress.delete(id);
     }
     /**
      * Mark job as failed and sets the state
      * @ignore
      */
     failJob(id, retryCount) {
-        const job = this.requestsInProgress.get(id);
+        const job = this.#requestsInProgress.get(id);
         if (!job)
             return;
         this.state.requestTotalFailedDurationMillis += job.finish();
         this.state.requestsFailed++;
-        this._saveRetryCountForJob(retryCount);
-        this.requestsInProgress.delete(id);
+        this.saveRetryCountForJob(retryCount);
+        this.#requestsInProgress.delete(id);
+    }
+    /**
+     * Discards a started job without affecting the finished/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * @ignore
+     */
+    discardJob(id) {
+        this.#requestsInProgress.delete(id);
     }
     /**
      * Calculate the current statistics
      */
     calculate() {
         const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
-        const totalMillis = Date.now() - this.instanceStart;
+        const totalMillis = Date.now() - this.state.instanceStart;
         const totalMinutes = totalMillis / 1000 / 60;
         return {
             requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
@@ -216,131 +350,87 @@ export class Statistics {
      * displaying the current state in predefined intervals
      */
     async startCapturing() {
-        this.keyValueStore ??= await KeyValueStore.open(null, { config: this.config });
+        // A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
+        // capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
+        if (this.#logInterval) {
+            throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
+        }
+        await this.#recoverableState.initialize();
+        // After the load, so that a restored record keeps the timestamp of the run it belongs to.
         if (this.state.crawlerStartedAt === null) {
             this.state.crawlerStartedAt = new Date();
         }
-        if (this.persistenceOptions.enable) {
-            await this._maybeLoadStatistics();
-            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
-        }
-        this.logInterval = setInterval(() => {
-            this.log.info(this.logMessage, {
+        this.#logInterval = setInterval(() => {
+            this.log.info(this.#logMessage, {
                 ...this.calculate(),
                 retryHistogram: this.requestRetryHistogram,
             });
-        }, this.logIntervalMillis);
+        }, this.#logIntervalMillis);
     }
     /**
      * Stops logging and remove event listeners, then persist
      */
     async stopCapturing() {
-        this._teardown();
+        this.#stopLogging();
         this.state.crawlerFinishedAt = new Date();
-        await this.persistState();
+        await this.#recoverableState.teardown();
     }
-    _saveRetryCountForJob(retryCount) {
+    saveRetryCountForJob(retryCount) {
         if (retryCount > 0)
             this.state.requestsRetries++;
         this.requestRetryHistogram[retryCount] ??= 0;
         this.requestRetryHistogram[retryCount]++;
     }
     /**
-     * Persist internal state to the key value store
-     * @param options - Override the persistence options provided in the constructor
+     * Persist internal state to the key value store.
+     *
+     * Statistics are bookkeeping - a store that refuses the write is worth a warning, not a failed crawl. The
+     * crawler calls this from its migration handler, where a rejection would go unhandled.
      */
-    async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        // this might be called before startCapturing was called without using await, should not crash
-        if (!this.keyValueStore) {
-            return;
-        }
-        this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
-        await this.keyValueStore
-            .setValue(this.persistStateKey, this.toJSON(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
+    async persistState() {
+        await this.#recoverableState
+            .persistState()
             .catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
     }
     /**
-     * Loads the current statistic from the key value store if any
+     * Rebuilds the state from a persisted record.
+     *
+     * A subclass tracking extra fields restores them here, on top of the result of `super.deserializeState()`.
      */
-    async _maybeLoadStatistics() {
-        // this might be called before startCapturing was called without using await, should not crash
-        if (!this.keyValueStore) {
-            return;
-        }
-        const savedState = await this.keyValueStore.getValue(this.persistStateKey);
-        if (!savedState)
-            return;
-        // We saw a run where the requestRetryHistogram was not iterable and crashed
-        // the crawler. Adding some logging to monitor this problem in the future.
-        if (!Array.isArray(savedState.requestRetryHistogram)) {
-            this.log.warning('Received invalid state from Key-value store.', {
+    deserializeState(persistedState) {
+        // The cast is the index signature the `catchall` puts on the schema and an interface cannot have - the
+        // record is an unvalidated blob off the key-value store either way, which is what the decode is for.
+        const restored = z.safeDecode(this.#stateCodec, persistedState);
+        if (!restored.success) {
+            // Statistics are bookkeeping - a record that cannot be made sense of is worth a warning and a fresh
+            // start, not a failed crawl.
+            this.log.warning('Received invalid state from Key-value store, starting the statistics from scratch.', {
                 persistStateKey: this.persistStateKey,
-                state: savedState,
+                issues: restored.error.issues,
             });
+            return this.defaultState();
         }
         this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.persistStateKey });
-        // the `requestRetryHistogram` array might be very large, we could end up with
-        // `RangeError: Maximum call stack size exceeded` if we use `a.push(...b)`
-        savedState.requestRetryHistogram.forEach((idx) => this.requestRetryHistogram.push(idx));
-        this.state.requestsFinished = savedState.requestsFinished;
-        this.state.requestsFailed = savedState.requestsFailed;
-        this.state.requestsRetries = savedState.requestsRetries;
-        this.state.requestTotalFailedDurationMillis = savedState.requestTotalFailedDurationMillis;
-        this.state.requestTotalFinishedDurationMillis = savedState.requestTotalFinishedDurationMillis;
-        this.state.requestMinDurationMillis = savedState.requestMinDurationMillis;
-        this.state.requestMaxDurationMillis = savedState.requestMaxDurationMillis;
-        // persisted state uses ISO date strings
-        this.state.crawlerFinishedAt = savedState.crawlerFinishedAt ? new Date(savedState.crawlerFinishedAt) : null;
-        this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
-        this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
-        this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
-        this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
-        this.log.debug('Loaded from KeyValueStore');
+        return { ...this.defaultState(), ...restored.data };
     }
-    _teardown() {
-        // this can be called before a call to startCapturing happens (or in a 'finally' block)
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
-        if (this.logInterval) {
-            clearInterval(this.logInterval);
-            this.logInterval = null;
+    #stopLogging() {
+        if (this.#logInterval) {
+            clearInterval(this.#logInterval);
+            this.#logInterval = null;
         }
     }
+    /**
+     * Builds the record written to the key value store, merging in the derived aggregates so that a consumer
+     * reading the record does not have to reconstruct them.
+     */
+    serializeState(state) {
+        return z.encode(this.#stateCodec, state);
+    }
     /**
      * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
      * or through `keyValueStore.setValue('KEY', statsInstance)`
      */
     toJSON() {
-        // merge all the current state information that can be used from the outside
-        // without the need to reconstruct for the sake of stats.calculate()
-        // omit duplicated information
-        const result = {
-            ...this.state,
-            crawlerLastStartTimestamp: this.instanceStart,
-            crawlerFinishedAt: this.state.crawlerFinishedAt
-                ? new Date(this.state.crawlerFinishedAt).toISOString()
-                : null,
-            crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,
-            requestRetryHistogram: this.requestRetryHistogram,
-            statsId: this.id,
-            statsPersistedAt: new Date().toISOString(),
-            ...this.calculate(),
-        };
-        Reflect.deleteProperty(result, 'requestsWithStatusCode');
-        Reflect.deleteProperty(result, 'errors');
-        Reflect.deleteProperty(result, 'retryErrors');
-        result.requestsWithStatusCode = this.state.requestsWithStatusCode;
-        result.errors = this.state.errors;
-        result.retryErrors = this.state.retryErrors;
-        return result;
+        return this.serializeState(this.state);
     }
 }
-//# sourceMappingURL=statistics.js.map

package/debug.d.ts

@@ -0,0 +1,36 @@
+import type { IncomingMessage } from 'node:http';
+import type { Dictionary } from '@crawlee/types';
+import type { Request } from './request.js';
+interface BrowserResponseLike {
+    status(): number;
+}
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
+ * @param [response]
+ *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
+ *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
+ * @param [additionalFields] Object containing additional fields to be added.
+ *
+ * @internal
+ */
+export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
+/**
+ * Returns a human-readable label for an unknown value,
+ * suitable for embedding in error messages and log output.
+ *
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
+ *
+ * @internal
+ */
+export declare function inspectValue(value: unknown): string;
+/**
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
+ * by their constructor name. Used for building validation error messages.
+ *
+ * @internal
+ */
+export declare function getObjectType(value: unknown): string;
+export {};

package/debug.js

@@ -0,0 +1,70 @@
+import { inspect } from 'node:util';
+import ow from 'ow';
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
+ * @param [response]
+ *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
+ *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
+ * @param [additionalFields] Object containing additional fields to be added.
+ *
+ * @internal
+ */
+export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
+    ow(request, ow.object);
+    ow(response, ow.object);
+    ow(additionalFields, ow.object);
+    return {
+        requestId: request.id,
+        url: request.url,
+        loadedUrl: request.loadedUrl,
+        method: request.method,
+        retryCount: request.retryCount,
+        errorMessages: request.errorMessages,
+        // Puppeteer response has .status() function and NodeJS response, statusCode property.
+        statusCode: 'status' in response && response.status instanceof Function
+            ? response.status()
+            : response.statusCode,
+        ...additionalFields,
+    };
+}
+/**
+ * Returns a human-readable label for an unknown value,
+ * suitable for embedding in error messages and log output.
+ *
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
+ *
+ * @internal
+ */
+export function inspectValue(value) {
+    if (typeof value === 'object' && value !== null && value.constructor?.name) {
+        return value.constructor.name;
+    }
+    return inspect(value, {
+        depth: 0,
+        compact: true,
+        maxStringLength: 64,
+        breakLength: Infinity,
+        colors: false,
+    });
+}
+/**
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
+ * by their constructor name. Used for building validation error messages.
+ *
+ * @internal
+ */
+export function getObjectType(value) {
+    const simple = typeof value;
+    if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
+        return simple;
+    }
+    const objectType = Object.prototype.toString.call(value);
+    const type = /\[object (\w+)]/.exec(objectType)[1];
+    if (type === 'Uint8Array') {
+        return 'Buffer';
+    }
+    return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
+}