@crawlee/core 4.0.0-beta.91 → 4.0.0-beta.93

package/autoscaling/autoscaled_pool.d.ts

@@ -1,11 +1,10 @@
 import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency_system.js';
 import type { CrawleeLogger } from '../log.js';
 /**
- * The task-readiness predicates a consumer may supply to steer an {@link AutoscaledPool}'s run loop — the parts of
- * the loop a higher-level driver (e.g. a crawler) legitimately overrides, as opposed to the crawler-owned
- * `runTaskFunction`.
+ * The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
+ * a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
  */
-export interface AutoscaledPoolPredicateOptions {
+export interface TaskLoopPredicates {
     /**
      * A function that indicates whether `runTaskFunction` should be called.
      * This function is called every time there is free capacity for a new task and it should
@@ -15,14 +14,14 @@ export interface AutoscaledPoolPredicateOptions {
     isTaskReadyFunction?: () => Promise<boolean>;
     /**
      * A function that is called only when there are no tasks to be processed.
-     * If it resolves to `true` then the pool's run finishes. Being called only
+     * If it resolves to `true` then the run finishes. Being called only
      * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
      * keeps resolving to `true`, `isFinishedFunction()` will never be called.
-     * To abort a run, use the {@link AutoscaledPool.abort} method.
      */
     isFinishedFunction?: () => Promise<boolean>;
 }
-export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {
+/** @internal */
+export interface AutoscaledPoolOptions extends TaskLoopPredicates {
     /**
      * The governor that decides whether there is free compute for one more task. Typically a
      * {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
@@ -62,8 +61,8 @@ export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {
  *
  * Before running the pool, you need to implement the following three functions:
  * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link AutoscaledPoolPredicateOptions.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link AutoscaledPoolPredicateOptions.isFinishedFunction|`isFinishedFunction`}.
+ * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
+ * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
  *
  * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
  * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
@@ -104,7 +103,8 @@ export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {
  *     await concurrencySystem.stop();
  * }
  * ```
- * @category Scaling
+ *
+ * @internal
  */
 export declare class AutoscaledPool {
     private readonly log;

package/autoscaling/autoscaled_pool.js

@@ -10,8 +10,8 @@ import { serviceLocator } from '../service_locator.js';
  *
  * Before running the pool, you need to implement the following three functions:
  * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link AutoscaledPoolPredicateOptions.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link AutoscaledPoolPredicateOptions.isFinishedFunction|`isFinishedFunction`}.
+ * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
+ * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
  *
  * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
  * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
@@ -52,7 +52,8 @@ import { serviceLocator } from '../service_locator.js';
  *     await concurrencySystem.stop();
  * }
  * ```
- * @category Scaling
+ *
+ * @internal
  */
 export class AutoscaledPool {
     log;

package/autoscaling/load_signal.js

@@ -1,4 +1,4 @@
-import { weightedAvg } from '@crawlee/utils';
+import { weightedAvg } from './weighted_avg.js';
 /**
  * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
  * and so can yours — it is the only part of their machinery worth reusing.

package/autoscaling/weighted_avg.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;

package/autoscaling/weighted_avg.js

@@ -0,0 +1,14 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export function weightedAvg(arrValues, arrWeights) {
+    const result = arrValues
+        .map((value, i) => {
+        const weight = arrWeights[i];
+        const sum = value * weight;
+        return [sum, weight];
+    })
+        .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
+    return result[0] / result[1];
+}

package/byte_utils.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export declare function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export declare function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;

package/byte_utils.js

@@ -0,0 +1,42 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export function isStream(value) {
+    if (typeof value !== 'object' || value === null) {
+        return false;
+    }
+    // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.
+    // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would
+    // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.
+    const isNodeStream = typeof value.pipe === 'function' && typeof value[Symbol.asyncIterator] === 'function';
+    const isWebStream = typeof value.pipeTo === 'function';
+    return isNodeStream || isWebStream;
+}
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export function isBuffer(value) {
+    return (value != null &&
+        typeof value === 'object' &&
+        (Buffer.isBuffer(value) ||
+            value instanceof ArrayBuffer ||
+            ArrayBuffer.isView(value) ||
+            value.constructor?.name === 'Buffer'));
+}
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export function toBuffer(value) {
+    if (Buffer.isBuffer(value)) {
+        return value;
+    }
+    if (value instanceof ArrayBuffer) {
+        return Buffer.from(value);
+    }
+    return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+}

package/memory-storage/utils.d.ts

@@ -1,4 +1,3 @@
-import { isBuffer, isStream, toBuffer } from '@crawlee/utils';
 /**
  * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
  * Storage names and record keys are used as filesystem path components, so a value containing `..`
@@ -14,4 +13,4 @@ export declare function purgeNullsFromObject<T>(object: T): T;
  * Creates a standard request ID (same as Platform).
  */
 export declare function uniqueKeyToRequestId(uniqueKey: string): string;
-export { isBuffer, isStream, toBuffer };
+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';

package/memory-storage/utils.js

@@ -1,6 +1,5 @@
 import { createHash } from 'node:crypto';
 import { resolve, sep } from 'node:path';
-import { isBuffer, isStream, toBuffer } from '@crawlee/utils';
 import { REQUEST_ID_LENGTH } from './consts.js';
 /**
  * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
@@ -39,4 +38,4 @@ export function uniqueKeyToRequestId(uniqueKey) {
         .replace(/(\+|\/|=)/g, '');
     return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;
 }
-export { isBuffer, isStream, toBuffer };
+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.91",
+    "version": "4.0.0-beta.93",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.91",
-        "@crawlee/types": "4.0.0-beta.91",
-        "@crawlee/utils": "4.0.0-beta.91",
+        "@crawlee/fs-storage": "4.0.0-beta.93",
+        "@crawlee/types": "4.0.0-beta.93",
+        "@crawlee/utils": "4.0.0-beta.93",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "1e3e1ca10f24be53e1d527b3ea25f456baebecbf"
+    "gitHead": "b9d21e80d94d01e21d4e1c19191610d0157dd172"
 }

package/storages/key_value_store.js

@@ -7,7 +7,7 @@ import { parseValue, serializeValue } from './key_value_store_codec.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { createDualIterable, purgeDefaultStorages } from './utils.js';
-import { isBuffer, isStream } from '@crawlee/utils';
+import { isBuffer, isStream } from '../byte_utils.js';
 /** @internal */
 const KVS_KEYS_DEFAULT_LIMIT = 1000;
 /**

package/storages/key_value_store_codec.js

@@ -1,7 +1,7 @@
 import contentTypeParser from 'content-type';
-import { isBuffer, isStream } from '@crawlee/utils';
 import JSON5 from 'json5';
 import { jsonStringifyExtended } from '@apify/utilities';
+import { isBuffer, isStream } from '../byte_utils.js';
 const CONTENT_TYPE_JSON = 'application/json';
 const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}$`, 'i'), /^application\/.*xml$/i, /^text\//i];
 /**