@crawlee/core 4.0.0-beta.93 → 4.0.0-beta.95

package/cookie_utils.js

@@ -103,8 +103,10 @@ export function mergeCookies(url, sourceCookies) {
             if (!cookieString)
                 continue;
             const cookie = Cookie.parse(cookieString);
-            if (!cookie)
-                throw new CookieParseError(cookieString);
+            if (!cookie) {
+                serviceLocator.getLogger().warning(`Skipping malformed cookie fragment: '${cookieString}'`);
+                continue;
+            }
             const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });

package/iterables.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
+/**
+ * An async iterator that also supports peeking at the next value without consuming it.
+ * Extends both AsyncIterator and AsyncIterable interfaces.
+ * @internal
+ */
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+    /**
+     * Peeks at the next value without consuming it from the iterator.
+     * Subsequent calls to peek() will return the same value until next() is called.
+     *
+     * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
+     */
+    peek(): Promise<T | undefined>;
+}
+/**
+ * An async iterable that yields peekable async iterators.
+ * @internal
+ */
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+    [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;

package/iterables.js

@@ -0,0 +1,134 @@
+import { inspect } from 'node:util';
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export async function* asyncifyIterable(iterable) {
+    yield* iterable;
+}
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export async function* chunkedAsyncIterable(iterable, chunkSize) {
+    const getChunkSize = typeof chunkSize === 'function' ? chunkSize : () => chunkSize;
+    if (typeof chunkSize === 'number' && chunkSize < 1) {
+        throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
+    }
+    const iterator = Symbol.asyncIterator in iterable
+        ? iterable[Symbol.asyncIterator]()
+        : iterable[Symbol.iterator]();
+    while (true) {
+        const currentSize = getChunkSize();
+        if (currentSize < 1)
+            break;
+        const chunk = [];
+        for (let i = 0; i < currentSize; i++) {
+            const next = await iterator.next();
+            if (next.done) {
+                break;
+            }
+            chunk.push(next.value);
+        }
+        if (chunk.length === 0)
+            break;
+        yield chunk;
+    }
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export function peekableAsyncIterable(iterable) {
+    const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
+    let peekedValue;
+    let isExhausted = false;
+    const peekableIterator = {
+        async next() {
+            // If we have peeked a value, return it and clear the peek
+            if (peekedValue !== undefined) {
+                const result = peekedValue;
+                peekedValue = undefined;
+                if (result.done) {
+                    isExhausted = true;
+                    return { done: true, value: undefined };
+                }
+                return { done: false, value: result.value };
+            }
+            if (isExhausted) {
+                return { done: true, value: undefined };
+            }
+            const result = await iterator.next();
+            if (result.done) {
+                isExhausted = true;
+            }
+            return result;
+        },
+        async peek() {
+            if (peekedValue !== undefined) {
+                return peekedValue.done ? undefined : peekedValue.value;
+            }
+            if (isExhausted) {
+                return undefined;
+            }
+            const result = await iterator.next();
+            peekedValue = { done: result.done ?? false, value: result.value };
+            if (result.done) {
+                isExhausted = true;
+                return undefined;
+            }
+            return result.value;
+        },
+        [Symbol.asyncIterator]() {
+            return this;
+        },
+    };
+    return {
+        [Symbol.asyncIterator]() {
+            return peekableIterator;
+        },
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.93",
+    "version": "4.0.0-beta.95",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.93",
-        "@crawlee/types": "4.0.0-beta.93",
-        "@crawlee/utils": "4.0.0-beta.93",
+        "@crawlee/fs-storage": "4.0.0-beta.95",
+        "@crawlee/types": "4.0.0-beta.95",
+        "@crawlee/utils": "4.0.0-beta.95",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "b9d21e80d94d01e21d4e1c19191610d0157dd172"
+    "gitHead": "ba8602d011706fb5c9930232402156eb5b657eb3"
 }

package/service_locator.js

@@ -189,9 +189,15 @@ const serviceLocatorStorage = new AsyncLocalStorage();
  */
 export function bindMethodsToServiceLocator(serviceLocator, target) {
     let proto = Object.getPrototypeOf(target);
+    const seenKeys = new Set();
     while (proto !== null && proto !== Object.prototype) {
         const propertyKeys = [...Object.getOwnPropertyNames(proto), ...Object.getOwnPropertySymbols(proto)];
         for (const propertyKey of propertyKeys) {
+            // The chain is walked derived-first, so the first occurrence of a key is the one dynamic
+            // dispatch would pick — a subclass override must not be clobbered by its base version.
+            if (seenKeys.has(propertyKey))
+                continue;
+            seenKeys.add(propertyKey);
             const descriptor = Object.getOwnPropertyDescriptor(proto, propertyKey);
             // We use property descriptors rather than accessing target[propertyKey] directly,
             // because that would trigger getters and cause unwanted side effects.

package/storages/request_queue.js

@@ -1,9 +1,10 @@
 import { inspect } from 'node:util';
-import { chunkedAsyncIterable, downloadListOfUrls, isAsyncIterable, isIterable, peekableAsyncIterable, sleep, } from '@crawlee/utils';
+import { downloadListOfUrls, isAsyncIterable, isIterable, sleep } from '@crawlee/utils';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
 import { Configuration } from '../configuration.js';
 import { getObjectType } from '../debug.js';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
 import { checkStorageAccess } from './access_checking.js';