@crawlee/core 4.0.0-beta.92 → 4.0.0-beta.94

package/autoscaling/autoscaled_pool.d.ts

@@ -1,11 +1,10 @@
 import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency_system.js';
 import type { CrawleeLogger } from '../log.js';
 /**
- * The task-readiness predicates a consumer may supply to steer an {@link AutoscaledPool}'s run loop — the parts of
- * the loop a higher-level driver (e.g. a crawler) legitimately overrides, as opposed to the crawler-owned
- * `runTaskFunction`.
+ * The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
+ * a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
  */
-export interface AutoscaledPoolPredicateOptions {
+export interface TaskLoopPredicates {
     /**
      * A function that indicates whether `runTaskFunction` should be called.
      * This function is called every time there is free capacity for a new task and it should
@@ -15,14 +14,14 @@ export interface AutoscaledPoolPredicateOptions {
     isTaskReadyFunction?: () => Promise<boolean>;
     /**
      * A function that is called only when there are no tasks to be processed.
-     * If it resolves to `true` then the pool's run finishes. Being called only
+     * If it resolves to `true` then the run finishes. Being called only
      * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
      * keeps resolving to `true`, `isFinishedFunction()` will never be called.
-     * To abort a run, use the {@link AutoscaledPool.abort} method.
      */
     isFinishedFunction?: () => Promise<boolean>;
 }
-export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {
+/** @internal */
+export interface AutoscaledPoolOptions extends TaskLoopPredicates {
     /**
      * The governor that decides whether there is free compute for one more task. Typically a
      * {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
@@ -62,8 +61,8 @@ export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {
  *
  * Before running the pool, you need to implement the following three functions:
  * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link AutoscaledPoolPredicateOptions.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link AutoscaledPoolPredicateOptions.isFinishedFunction|`isFinishedFunction`}.
+ * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
+ * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
  *
  * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
  * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
@@ -104,7 +103,8 @@ export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {
  *     await concurrencySystem.stop();
  * }
  * ```
- * @category Scaling
+ *
+ * @internal
  */
 export declare class AutoscaledPool {
     private readonly log;

package/autoscaling/autoscaled_pool.js

@@ -10,8 +10,8 @@ import { serviceLocator } from '../service_locator.js';
  *
  * Before running the pool, you need to implement the following three functions:
  * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link AutoscaledPoolPredicateOptions.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link AutoscaledPoolPredicateOptions.isFinishedFunction|`isFinishedFunction`}.
+ * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
+ * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
  *
  * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
  * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
@@ -52,7 +52,8 @@ import { serviceLocator } from '../service_locator.js';
  *     await concurrencySystem.stop();
  * }
  * ```
- * @category Scaling
+ *
+ * @internal
  */
 export class AutoscaledPool {
     log;

package/cookie_utils.js

@@ -103,8 +103,10 @@ export function mergeCookies(url, sourceCookies) {
             if (!cookieString)
                 continue;
             const cookie = Cookie.parse(cookieString);
-            if (!cookie)
-                throw new CookieParseError(cookieString);
+            if (!cookie) {
+                serviceLocator.getLogger().warning(`Skipping malformed cookie fragment: '${cookieString}'`);
+                continue;
+            }
             const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });

package/iterables.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
+/**
+ * An async iterator that also supports peeking at the next value without consuming it.
+ * Extends both AsyncIterator and AsyncIterable interfaces.
+ * @internal
+ */
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+    /**
+     * Peeks at the next value without consuming it from the iterator.
+     * Subsequent calls to peek() will return the same value until next() is called.
+     *
+     * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
+     */
+    peek(): Promise<T | undefined>;
+}
+/**
+ * An async iterable that yields peekable async iterators.
+ * @internal
+ */
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+    [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;

package/iterables.js

@@ -0,0 +1,134 @@
+import { inspect } from 'node:util';
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export async function* asyncifyIterable(iterable) {
+    yield* iterable;
+}
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export async function* chunkedAsyncIterable(iterable, chunkSize) {
+    const getChunkSize = typeof chunkSize === 'function' ? chunkSize : () => chunkSize;
+    if (typeof chunkSize === 'number' && chunkSize < 1) {
+        throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
+    }
+    const iterator = Symbol.asyncIterator in iterable
+        ? iterable[Symbol.asyncIterator]()
+        : iterable[Symbol.iterator]();
+    while (true) {
+        const currentSize = getChunkSize();
+        if (currentSize < 1)
+            break;
+        const chunk = [];
+        for (let i = 0; i < currentSize; i++) {
+            const next = await iterator.next();
+            if (next.done) {
+                break;
+            }
+            chunk.push(next.value);
+        }
+        if (chunk.length === 0)
+            break;
+        yield chunk;
+    }
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export function peekableAsyncIterable(iterable) {
+    const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
+    let peekedValue;
+    let isExhausted = false;
+    const peekableIterator = {
+        async next() {
+            // If we have peeked a value, return it and clear the peek
+            if (peekedValue !== undefined) {
+                const result = peekedValue;
+                peekedValue = undefined;
+                if (result.done) {
+                    isExhausted = true;
+                    return { done: true, value: undefined };
+                }
+                return { done: false, value: result.value };
+            }
+            if (isExhausted) {
+                return { done: true, value: undefined };
+            }
+            const result = await iterator.next();
+            if (result.done) {
+                isExhausted = true;
+            }
+            return result;
+        },
+        async peek() {
+            if (peekedValue !== undefined) {
+                return peekedValue.done ? undefined : peekedValue.value;
+            }
+            if (isExhausted) {
+                return undefined;
+            }
+            const result = await iterator.next();
+            peekedValue = { done: result.done ?? false, value: result.value };
+            if (result.done) {
+                isExhausted = true;
+                return undefined;
+            }
+            return result.value;
+        },
+        [Symbol.asyncIterator]() {
+            return this;
+        },
+    };
+    return {
+        [Symbol.asyncIterator]() {
+            return peekableIterator;
+        },
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.92",
+    "version": "4.0.0-beta.94",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.92",
-        "@crawlee/types": "4.0.0-beta.92",
-        "@crawlee/utils": "4.0.0-beta.92",
+        "@crawlee/fs-storage": "4.0.0-beta.94",
+        "@crawlee/types": "4.0.0-beta.94",
+        "@crawlee/utils": "4.0.0-beta.94",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "9470e6cd2fe57b354ba30643a2348f2e737c37fe"
+    "gitHead": "35f62988d94ddf8a3e304f81be16202ca6479c07"
 }

package/storages/request_queue.js

@@ -1,9 +1,10 @@
 import { inspect } from 'node:util';
-import { chunkedAsyncIterable, downloadListOfUrls, isAsyncIterable, isIterable, peekableAsyncIterable, sleep, } from '@crawlee/utils';
+import { downloadListOfUrls, isAsyncIterable, isIterable, sleep } from '@crawlee/utils';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
 import { Configuration } from '../configuration.js';
 import { getObjectType } from '../debug.js';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
 import { checkStorageAccess } from './access_checking.js';