@crawlee/core 4.0.0-beta.91 → 4.0.0-beta.92

package/autoscaling/load_signal.js

@@ -1,4 +1,4 @@
-import { weightedAvg } from '@crawlee/utils';
+import { weightedAvg } from './weighted_avg.js';
 /**
  * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
  * and so can yours — it is the only part of their machinery worth reusing.

package/autoscaling/weighted_avg.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;

package/autoscaling/weighted_avg.js

@@ -0,0 +1,14 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export function weightedAvg(arrValues, arrWeights) {
+    const result = arrValues
+        .map((value, i) => {
+        const weight = arrWeights[i];
+        const sum = value * weight;
+        return [sum, weight];
+    })
+        .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
+    return result[0] / result[1];
+}

package/byte_utils.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export declare function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export declare function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;

package/byte_utils.js

@@ -0,0 +1,42 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export function isStream(value) {
+    if (typeof value !== 'object' || value === null) {
+        return false;
+    }
+    // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.
+    // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would
+    // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.
+    const isNodeStream = typeof value.pipe === 'function' && typeof value[Symbol.asyncIterator] === 'function';
+    const isWebStream = typeof value.pipeTo === 'function';
+    return isNodeStream || isWebStream;
+}
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export function isBuffer(value) {
+    return (value != null &&
+        typeof value === 'object' &&
+        (Buffer.isBuffer(value) ||
+            value instanceof ArrayBuffer ||
+            ArrayBuffer.isView(value) ||
+            value.constructor?.name === 'Buffer'));
+}
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export function toBuffer(value) {
+    if (Buffer.isBuffer(value)) {
+        return value;
+    }
+    if (value instanceof ArrayBuffer) {
+        return Buffer.from(value);
+    }
+    return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+}

package/memory-storage/utils.d.ts

@@ -1,4 +1,3 @@
-import { isBuffer, isStream, toBuffer } from '@crawlee/utils';
 /**
  * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
  * Storage names and record keys are used as filesystem path components, so a value containing `..`
@@ -14,4 +13,4 @@ export declare function purgeNullsFromObject<T>(object: T): T;
  * Creates a standard request ID (same as Platform).
  */
 export declare function uniqueKeyToRequestId(uniqueKey: string): string;
-export { isBuffer, isStream, toBuffer };
+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';

package/memory-storage/utils.js

@@ -1,6 +1,5 @@
 import { createHash } from 'node:crypto';
 import { resolve, sep } from 'node:path';
-import { isBuffer, isStream, toBuffer } from '@crawlee/utils';
 import { REQUEST_ID_LENGTH } from './consts.js';
 /**
  * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
@@ -39,4 +38,4 @@ export function uniqueKeyToRequestId(uniqueKey) {
         .replace(/(\+|\/|=)/g, '');
     return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;
 }
-export { isBuffer, isStream, toBuffer };
+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.91",
+    "version": "4.0.0-beta.92",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.91",
-        "@crawlee/types": "4.0.0-beta.91",
-        "@crawlee/utils": "4.0.0-beta.91",
+        "@crawlee/fs-storage": "4.0.0-beta.92",
+        "@crawlee/types": "4.0.0-beta.92",
+        "@crawlee/utils": "4.0.0-beta.92",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "1e3e1ca10f24be53e1d527b3ea25f456baebecbf"
+    "gitHead": "9470e6cd2fe57b354ba30643a2348f2e737c37fe"
 }

package/storages/key_value_store.js

@@ -7,7 +7,7 @@ import { parseValue, serializeValue } from './key_value_store_codec.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { createDualIterable, purgeDefaultStorages } from './utils.js';
-import { isBuffer, isStream } from '@crawlee/utils';
+import { isBuffer, isStream } from '../byte_utils.js';
 /** @internal */
 const KVS_KEYS_DEFAULT_LIMIT = 1000;
 /**

package/storages/key_value_store_codec.js

@@ -1,7 +1,7 @@
 import contentTypeParser from 'content-type';
-import { isBuffer, isStream } from '@crawlee/utils';
 import JSON5 from 'json5';
 import { jsonStringifyExtended } from '@apify/utilities';
+import { isBuffer, isStream } from '../byte_utils.js';
 const CONTENT_TYPE_JSON = 'application/json';
 const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}$`, 'i'), /^application\/.*xml$/i, /^text\//i];
 /**