@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/autoscaling/storage_backend_load_signal.js

@@ -0,0 +1,73 @@
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { serviceLocator } from '../service_locator.js';
+import { SnapshotStore } from './load_signal.js';
+const RATE_LIMIT_ERROR_RETRY_COUNT = 2;
+/**
+ * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
+ * exceeds a threshold.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * Switch it off entirely ({@link LoadSignalsOptions.storageBackend|`storageBackend: false`}) if the storage backend
+ * reports no rate-limit statistics, since it otherwise polls it every second to no purpose.
+ *
+ * @category Scaling
+ */
+export class StorageBackendLoadSignal {
+    name = 'storageBackendInfo';
+    overloadedRatio;
+    #store = new SnapshotStore();
+    #intervalMillis;
+    #maxErrors;
+    #interval;
+    #storageBackend;
+    constructor(options = {}) {
+        this.overloadedRatio = options.overloadedRatio ?? 0.3;
+        this.#intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;
+        this.#maxErrors = options.maxErrors ?? 3;
+        this.handle = this.handle.bind(this);
+    }
+    async start(context) {
+        this.#store.useSampleWindow(context.maxSampleWindowMillis);
+        // A new session starts from a clean slate, or its first measurement diffs the error count against the previous
+        // session's — possibly against a different backend, since it is resolved afresh just below.
+        this.#store.clear();
+        // Resolved here rather than in the constructor, where asking for the backend would instantiate a default one
+        // as a side effect - long before the crawler that owns the run has had a chance to register its own.
+        this.#storageBackend = serviceLocator.getStorageBackend();
+        this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
+    }
+    async stop() {
+        if (this.#interval)
+            betterClearInterval(this.#interval);
+        this.#interval = undefined;
+        this.#storageBackend = undefined;
+    }
+    getSample(sampleDurationMillis) {
+        return this.#store.getSample(sampleDurationMillis);
+    }
+    /**
+     * Records one snapshot, overloaded when rate-limit errors grew by more than the configured limit since the
+     * previous one.
+     * @internal Also lets tests drive the measurement without waiting on a timer.
+     */
+    handle(intervalCallback) {
+        const now = new Date();
+        const allErrorCounts = this.#storageBackend?.stats?.rateLimitErrors ?? [];
+        const currentErrCount = allErrorCounts[RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
+        const snapshot = {
+            createdAt: now,
+            isOverloaded: false,
+            rateLimitErrorCount: currentErrCount,
+        };
+        const all = this.#store.getAll();
+        const previousSnapshot = all[all.length - 1];
+        if (previousSnapshot) {
+            const delta = currentErrCount - previousSnapshot.rateLimitErrorCount;
+            if (delta > this.#maxErrors)
+                snapshot.isOverloaded = true;
+        }
+        this.#store.push(snapshot, now);
+        intervalCallback();
+    }
+}

package/autoscaling/system_status.d.ts

@@ -1,15 +1,16 @@
-import type { Configuration } from '../configuration.js';
-import { Snapshotter } from './snapshotter.js';
+import type { LoadSignal } from './load_signal.js';
+import type { Snapshotter } from './snapshotter.js';
 /**
  * Represents the current status of the system.
  */
 export interface SystemInfo {
     /** If false, system is being overloaded. */
     isSystemIdle: boolean;
-    memInfo: ClientInfo;
-    eventLoopInfo: ClientInfo;
-    cpuInfo: ClientInfo;
-    clientInfo: ClientInfo;
+    memInfo: LoadSignalInfo;
+    eventLoopInfo: LoadSignalInfo;
+    cpuInfo: LoadSignalInfo;
+    storageBackendInfo: LoadSignalInfo;
+    memTotalBytes?: number;
     memCurrentBytes?: number;
     /**
      * Platform only property
@@ -26,7 +27,28 @@ export interface SystemInfo {
      * @internal
      */
     createdAt?: Date;
+    /**
+     * Status of additional load signals beyond the built-in four.
+     * Keys are `LoadSignal.name` values, values are overload info.
+     */
+    loadSignalInfo?: Record<string, LoadSignalInfo>;
 }
+/**
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
+ * @internal
+ */
+export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
+/**
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
+ * therefore how much history the signals are asked to retain.
+ * @internal
+ */
+export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
+/**
+ * An implementation detail of the {@link ConcurrencySystem} — configure it through
+ * {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
+ * @internal
+ */
 export interface SystemStatusOptions {
     /**
      * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
@@ -34,37 +56,25 @@ export interface SystemStatusOptions {
      */
     currentHistorySecs?: number;
     /**
-     * Sets the maximum ratio of overloaded snapshots in a memory sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.2
-     */
-    maxMemoryOverloadedRatio?: number;
-    /**
-     * Sets the maximum ratio of overloaded snapshots in an event loop sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.6
+     * Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
+     * autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
+     * private retention cannot silently widen the window.
+     * @default 30
      */
-    maxEventLoopOverloadedRatio?: number;
+    historySecs?: number;
     /**
-     * Sets the maximum ratio of overloaded snapshots in a CPU sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.4
+     * The `Snapshotter` whose built-in signals are evaluated.
      */
-    maxCpuOverloadedRatio?: number;
+    snapshotter: Snapshotter;
     /**
-     * Sets the maximum ratio of overloaded snapshots in a Client sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.3
+     * Additional load signals to include in the system status evaluation.
+     * These are evaluated alongside the built-in memory, CPU, event loop,
+     * and storage backend signals. If any signal reports overload, the system
+     * is considered overloaded. Each signal carries its own overload ratio.
      */
-    maxClientOverloadedRatio?: number;
-    /**
-     * The `Snapshotter` instance to be queried for `SystemStatus`.
-     */
-    snapshotter?: Snapshotter;
-    /** @internal */
-    config?: Configuration;
+    loadSignals?: LoadSignal[];
 }
-export interface ClientInfo {
+export interface LoadSignalInfo {
     isOverloaded: boolean;
     limitRatio: number;
     actualRatio: number;
@@ -82,35 +92,33 @@ export interface FinalStatistics {
     crawlerRuntimeMillis: number;
 }
 /**
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
- * and {@link SystemStatus.getHistoricalStatus}.
- * The system status is calculated using a weighted average of overloaded
- * messages in the snapshots, with the weights being the time intervals
- * between the snapshots. Each resource is calculated separately
- * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@link AutoscaledPool} class.
+ * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
+ * is overloaded whenever at least one of them is.
  *
- * {@link SystemStatus.getCurrentStatus}
- * returns a boolean that represents the current status of the system.
- * The length of the current timeframe in seconds is configurable
- * by the `currentHistorySecs` option and represents the max age
- * of snapshots to be considered for the calculation.
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
  *
- * {@link SystemStatus.getHistoricalStatus}
- * returns a boolean that represents the long-term status
- * of the system. It considers the full snapshot history available
- * in the {@link Snapshotter} instance.
- * @category Scaling
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
+ * {@link ConcurrencySystemOptions}.
+ * @internal
  */
 export declare class SystemStatus {
-    private readonly currentHistoryMillis;
-    private readonly maxMemoryOverloadedRatio;
-    private readonly maxEventLoopOverloadedRatio;
-    private readonly maxCpuOverloadedRatio;
-    private readonly maxClientOverloadedRatio;
-    private readonly snapshotter;
-    constructor(options?: SystemStatusOptions);
+    #private;
+    constructor(options: SystemStatusOptions);
+    /**
+     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
+     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
+     * defaults.
+     */
+    get maxSampleWindowMillis(): number;
+    /**
+     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
+     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
+     * only the last is reported.
+     */
+    private assertUniqueSignalNames;
     /**
      * Returns an {@link SystemInfo} object with the following structure:
      *
@@ -140,42 +148,12 @@ export declare class SystemStatus {
      * }
      * ```
      *
-     * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@link Snapshotter}
-     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
+     * `historySecs` seconds and `true` otherwise.
      */
     getHistoricalStatus(): SystemInfo;
     /**
      * Returns a system status object.
      */
-    protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the memory has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the event loop has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the CPU has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the client has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with sample information and an isOverloaded property
-     * set to true if at least the ratio of snapshots in the sample are overloaded.
-     */
-    protected _isSampleOverloaded<T extends {
-        createdAt: Date;
-        isOverloaded: boolean;
-    }>(sample: T[], ratio: number): ClientInfo;
+    private isSystemIdle;
 }
-//# sourceMappingURL=system_status.d.ts.map

package/autoscaling/system_status.js

@@ -1,52 +1,72 @@
-import { weightedAvg } from '@crawlee/utils';
-import ow from 'ow';
-import { Snapshotter } from './snapshotter.js';
+import { evaluateLoadSignalSample } from './load_signal.js';
 /**
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
- * and {@link SystemStatus.getHistoricalStatus}.
- * The system status is calculated using a weighted average of overloaded
- * messages in the snapshots, with the weights being the time intervals
- * between the snapshots. Each resource is calculated separately
- * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@link AutoscaledPool} class.
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
+ * @internal
+ */
+export const DEFAULT_CURRENT_HISTORY_SECS = 5;
+/**
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
+ * therefore how much history the signals are asked to retain.
+ * @internal
+ */
+export const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
+/** The four built-in signal names that map to typed `SystemInfo` fields, and the option that switches each off. */
+const BUILTIN_SIGNAL_OPTION_KEYS = {
+    memInfo: 'memory',
+    eventLoopInfo: 'eventLoop',
+    cpuInfo: 'cpu',
+    storageBackendInfo: 'storageBackend',
+};
+const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
+/**
+ * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
+ * is overloaded whenever at least one of them is.
  *
- * {@link SystemStatus.getCurrentStatus}
- * returns a boolean that represents the current status of the system.
- * The length of the current timeframe in seconds is configurable
- * by the `currentHistorySecs` option and represents the max age
- * of snapshots to be considered for the calculation.
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
  *
- * {@link SystemStatus.getHistoricalStatus}
- * returns a boolean that represents the long-term status
- * of the system. It considers the full snapshot history available
- * in the {@link Snapshotter} instance.
- * @category Scaling
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
+ * {@link ConcurrencySystemOptions}.
+ * @internal
  */
 export class SystemStatus {
-    currentHistoryMillis;
-    maxMemoryOverloadedRatio;
-    maxEventLoopOverloadedRatio;
-    maxCpuOverloadedRatio;
-    maxClientOverloadedRatio;
-    snapshotter;
-    constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            currentHistorySecs: ow.optional.number,
-            maxMemoryOverloadedRatio: ow.optional.number,
-            maxEventLoopOverloadedRatio: ow.optional.number,
-            maxCpuOverloadedRatio: ow.optional.number,
-            maxClientOverloadedRatio: ow.optional.number,
-            snapshotter: ow.optional.object,
-            config: ow.optional.object,
-        }));
-        const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
-        this.currentHistoryMillis = currentHistorySecs * 1000;
-        this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
-        this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
-        this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
-        this.maxClientOverloadedRatio = maxClientOverloadedRatio;
-        this.snapshotter = snapshotter || new Snapshotter({ config });
+    #currentHistoryMillis;
+    #historyMillis;
+    #signals;
+    constructor(options) {
+        const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
+        this.#currentHistoryMillis = currentHistorySecs * 1000;
+        this.#historyMillis = historySecs * 1000;
+        this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
+        this.assertUniqueSignalNames();
+    }
+    /**
+     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
+     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
+     * defaults.
+     */
+    get maxSampleWindowMillis() {
+        return Math.max(this.#currentHistoryMillis, this.#historyMillis);
+    }
+    /**
+     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
+     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
+     * only the last is reported.
+     */
+    assertUniqueSignalNames() {
+        const seen = new Set();
+        for (const { name } of this.#signals) {
+            if (!seen.has(name)) {
+                seen.add(name);
+                continue;
+            }
+            const hint = BUILTIN_SIGNAL_NAMES.has(name)
+                ? `it is the name of a built-in signal. To replace that signal, switch it off with \`loadSignals: { ${BUILTIN_SIGNAL_OPTION_KEYS[name]}: false }\` and keep your implementation in \`loadSignals.custom\`; to run yours alongside it, give it a different name.`
+                : 'two custom signals cannot share a name - rename one of them.';
+            throw new Error(`Duplicate load signal name ${JSON.stringify(name)}: ${hint}`);
+        }
     }
     /**
      * Returns an {@link SystemInfo} object with the following structure:
@@ -65,7 +85,7 @@ export class SystemStatus {
      * and `true` otherwise.
      */
     getCurrentStatus() {
-        return this._isSystemIdle(this.currentHistoryMillis);
+        return this.isSystemIdle(this.#currentHistoryMillis);
     }
     /**
      * Returns an {@link SystemInfo} object with the following structure:
@@ -79,91 +99,41 @@ export class SystemStatus {
      * }
      * ```
      *
-     * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@link Snapshotter}
-     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
+     * `historySecs` seconds and `true` otherwise.
      */
     getHistoricalStatus() {
-        return this._isSystemIdle();
+        return this.isSystemIdle(this.#historyMillis);
     }
     /**
      * Returns a system status object.
      */
-    _isSystemIdle(sampleDurationMillis) {
-        const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
-        const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
-        const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
-        const clientInfo = this._isClientOverloaded(sampleDurationMillis);
-        return {
-            isSystemIdle: !memInfo.isOverloaded &&
-                !eventLoopInfo.isOverloaded &&
-                !cpuInfo.isOverloaded &&
-                !clientInfo.isOverloaded,
-            memInfo,
-            eventLoopInfo,
-            cpuInfo,
-            clientInfo,
+    isSystemIdle(sampleDurationMillis) {
+        const result = {
+            isSystemIdle: true,
+            memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            storageBackendInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
         };
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the memory has been overloaded in the last sampleDurationMillis.
-     */
-    _isMemoryOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the event loop has been overloaded in the last sampleDurationMillis.
-     */
-    _isEventLoopOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the CPU has been overloaded in the last sampleDurationMillis.
-     */
-    _isCpuOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the client has been overloaded in the last sampleDurationMillis.
-     */
-    _isClientOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getClientSample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
-    }
-    /**
-     * Returns an object with sample information and an isOverloaded property
-     * set to true if at least the ratio of snapshots in the sample are overloaded.
-     */
-    _isSampleOverloaded(sample, ratio) {
-        if (sample.length === 0) {
-            return {
-                isOverloaded: false,
-                limitRatio: ratio,
-                actualRatio: 0,
-            };
+        let loadSignalInfo;
+        for (const signal of this.#signals) {
+            const sample = signal.getSample(sampleDurationMillis);
+            const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
+            if (info.isOverloaded) {
+                result.isSystemIdle = false;
+            }
+            if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
+                result[signal.name] = info;
+            }
+            else {
+                loadSignalInfo ??= {};
+                loadSignalInfo[signal.name] = info;
+            }
         }
-        const weights = [];
-        const values = [];
-        for (let i = 1; i < sample.length; i++) {
-            const previous = sample[i - 1];
-            const current = sample[i];
-            const weight = +current.createdAt - +previous.createdAt;
-            weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
-            values.push(+current.isOverloaded);
+        if (loadSignalInfo) {
+            result.loadSignalInfo = loadSignalInfo;
         }
-        const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
-        return {
-            isOverloaded: wAvg > ratio,
-            limitRatio: ratio,
-            actualRatio: Math.round(wAvg * 1000) / 1000,
-        };
+        return result;
     }
 }
-//# sourceMappingURL=system_status.js.map

package/autoscaling/weighted_avg.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;

package/autoscaling/weighted_avg.js

@@ -0,0 +1,14 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export function weightedAvg(arrValues, arrWeights) {
+    const result = arrValues
+        .map((value, i) => {
+        const weight = arrWeights[i];
+        const sum = value * weight;
+        return [sum, weight];
+    })
+        .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
+    return result[0] / result[1];
+}

package/byte_utils.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export declare function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export declare function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;

package/byte_utils.js

@@ -0,0 +1,42 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export function isStream(value) {
+    if (typeof value !== 'object' || value === null) {
+        return false;
+    }
+    // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.
+    // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would
+    // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.
+    const isNodeStream = typeof value.pipe === 'function' && typeof value[Symbol.asyncIterator] === 'function';
+    const isWebStream = typeof value.pipeTo === 'function';
+    return isNodeStream || isWebStream;
+}
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export function isBuffer(value) {
+    return (value != null &&
+        typeof value === 'object' &&
+        (Buffer.isBuffer(value) ||
+            value instanceof ArrayBuffer ||
+            ArrayBuffer.isView(value) ||
+            value.constructor?.name === 'Buffer'));
+}
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export function toBuffer(value) {
+    if (Buffer.isBuffer(value)) {
+        return value;
+    }
+    if (value instanceof ArrayBuffer) {
+        return Buffer.from(value);
+    }
+    return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+}