@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.170

package/storages/key_value_store_codec.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Canonical write path for key-value store records.
+ *
+ * When a content type is provided, the value passes through unchanged — it is the caller's
+ * responsibility to supply a String/Buffer/Stream (the frontend validates this).
+ *
+ * When no content type is provided, it is inferred from the value's shape:
+ * - Buffer / typed array / ArrayBuffer / stream → `application/octet-stream` (passthrough)
+ * - `string` → `text/plain; charset=utf-8` (passthrough)
+ * - anything else → `application/json; charset=utf-8` (serialized via `jsonStringifyExtended`)
+ *
+ * Does NOT drain streams — that is storage mechanics and stays in the storage backend.
+ *
+ * Backend-independent.
+ */
+export declare function serializeValue(value: unknown, contentType?: string): {
+    value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
+    contentType: string;
+};
+/**
+ * Parses a Buffer or ArrayBuffer using the provided content type header.
+ *
+ * - application/json is returned as a parsed object.
+ * - application/*xml and text/* are returned as strings.
+ * - everything else is returned as original body.
+ *
+ * If the header includes a charset, the body will be stringified only
+ * if the charset represents a known encoding to Node.js or Browser.
+ *
+ * Backend-independent — this is the canonical read path for the {@link KeyValueStore} frontend.
+ */
+export declare function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;

package/storages/key_value_store_codec.js

@@ -0,0 +1,113 @@
+import contentTypeParser from 'content-type';
+import JSON5 from 'json5';
+import { jsonStringifyExtended } from '@apify/utilities';
+import { isBuffer, isStream } from '../byte_utils.js';
+const CONTENT_TYPE_JSON = 'application/json';
+const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}$`, 'i'), /^application\/.*xml$/i, /^text\//i];
+/**
+ * Canonical write path for key-value store records.
+ *
+ * When a content type is provided, the value passes through unchanged — it is the caller's
+ * responsibility to supply a String/Buffer/Stream (the frontend validates this).
+ *
+ * When no content type is provided, it is inferred from the value's shape:
+ * - Buffer / typed array / ArrayBuffer / stream → `application/octet-stream` (passthrough)
+ * - `string` → `text/plain; charset=utf-8` (passthrough)
+ * - anything else → `application/json; charset=utf-8` (serialized via `jsonStringifyExtended`)
+ *
+ * Does NOT drain streams — that is storage mechanics and stays in the storage backend.
+ *
+ * Backend-independent.
+ */
+export function serializeValue(value, contentType) {
+    if (contentType !== null && contentType !== undefined) {
+        return { value: value, contentType };
+    }
+    if (isStream(value) || isBuffer(value)) {
+        return {
+            value,
+            contentType: 'application/octet-stream',
+        };
+    }
+    if (typeof value === 'string') {
+        return { value, contentType: 'text/plain; charset=utf-8' };
+    }
+    let serialized;
+    try {
+        // Format JSON to simplify debugging, the overheads with compression is negligible
+        serialized = jsonStringifyExtended(value, null, 2);
+    }
+    catch (e) {
+        const error = e;
+        // Give more meaningful error message
+        if (error.message?.includes('Invalid string length')) {
+            error.message = 'Object is too large';
+        }
+        throw new Error(`The "value" parameter cannot be stringified to JSON: ${error.message}`);
+    }
+    if (serialized === undefined) {
+        throw new Error('The "value" parameter was stringified to JSON and returned undefined. ' +
+            "Make sure you're not trying to stringify an undefined value.");
+    }
+    return { value: serialized, contentType: 'application/json; charset=utf-8' };
+}
+/**
+ * Parses a Buffer or ArrayBuffer using the provided content type header.
+ *
+ * - application/json is returned as a parsed object.
+ * - application/*xml and text/* are returned as strings.
+ * - everything else is returned as original body.
+ *
+ * If the header includes a charset, the body will be stringified only
+ * if the charset represents a known encoding to Node.js or Browser.
+ *
+ * Backend-independent — this is the canonical read path for the {@link KeyValueStore} frontend.
+ */
+export function parseValue(body, contentTypeHeader) {
+    // No content type at all → we have no basis for interpretation; hand back the raw value.
+    if (contentTypeHeader === null)
+        return body;
+    let contentType;
+    let charset;
+    try {
+        const result = contentTypeParser.parse(contentTypeHeader);
+        contentType = result.type;
+        charset = result.parameters.charset;
+    }
+    catch {
+        // Unparseable header → keep the original value rather than a mangled string.
+        return body;
+    }
+    // If we can't successfully interpret it, we return the original value rather than mangling it.
+    if (!areDataStringifiable(contentType, charset))
+        return body;
+    // Decode raw bytes using the resolved charset. An already-decoded string passes through (callers
+    // may hand us one directly), avoiding a needless re-encode round-trip.
+    const dataString = typeof body === 'string' ? body : isomorphicBufferToString(body, charset);
+    return contentType === CONTENT_TYPE_JSON ? JSON5.parse(dataString) : dataString;
+}
+function isomorphicBufferToString(buffer, encoding) {
+    if (buffer.constructor.name !== ArrayBuffer.name) {
+        return buffer.toString(encoding);
+    }
+    // In Node, wrap the ArrayBuffer in a Buffer so the resolved charset is honored (the caller already
+    // checked it via `Buffer.isEncoding`). Only the browser, which lacks Buffer, is limited to UTF-8.
+    if (typeof Buffer !== 'undefined') {
+        return Buffer.from(buffer).toString(encoding);
+    }
+    const decoder = new TextDecoder(encoding);
+    return decoder.decode(new Uint8Array(buffer));
+}
+function isCharsetStringifiable(charset) {
+    if (!charset)
+        return true; // hope that it's utf-8
+    return Buffer.isEncoding(charset);
+}
+function isContentTypeStringifiable(contentType) {
+    if (!contentType)
+        return false; // keep buffer
+    return STRINGIFIABLE_CONTENT_TYPE_RXS.some((rx) => rx.test(contentType));
+}
+function areDataStringifiable(contentType, charset) {
+    return isContentTypeStringifiable(contentType) && isCharsetStringifiable(charset);
+}

package/storages/request_dedup_cache.d.ts

@@ -0,0 +1,22 @@
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+export declare class RequestDeduplicationCache {
+    #private;
+    private readonly size;
+    constructor(size?: number);
+    get(cacheKey: string): string | null;
+    add(cacheKey: string, requestId: string): void;
+    clear(): void;
+    private indexOf;
+}

package/storages/request_dedup_cache.js

@@ -0,0 +1,48 @@
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+export class RequestDeduplicationCache {
+    size;
+    #keys;
+    #ids;
+    // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
+    constructor(size = 1_000_000) {
+        this.size = size;
+        this.#keys = new Array(size);
+        this.#ids = new Array(size);
+    }
+    get(cacheKey) {
+        const index = this.indexOf(cacheKey);
+        return this.#keys[index] === cacheKey ? this.#ids[index] : null;
+    }
+    add(cacheKey, requestId) {
+        const index = this.indexOf(cacheKey);
+        this.#keys[index] = cacheKey;
+        this.#ids[index] = requestId;
+    }
+    clear() {
+        this.#keys = new Array(this.size);
+        this.#ids = new Array(this.size);
+    }
+    // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
+    indexOf(cacheKey) {
+        /* eslint-disable no-bitwise */
+        let hash = 0x811c9dc5;
+        for (let i = 0; i < cacheKey.length; i++) {
+            hash ^= cacheKey.charCodeAt(i);
+            hash = Math.imul(hash, 0x01000193);
+        }
+        return (hash >>> 0) % this.size;
+        /* eslint-enable no-bitwise */
+    }
+}

package/storages/request_list.d.ts

@@ -1,69 +1,14 @@
+import type { BaseHttpClient } from '@crawlee/http-client';
 import type { Dictionary } from '@crawlee/types';
-import { Configuration } from '../configuration.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import { type InternalSource, Request, type RequestOptions, type Source } from '../request.js';
+import type { Configuration } from '../configuration.js';
+import type { IProxyConfiguration } from '../proxy_configuration.js';
+import { Request, type RequestOptions, type Source } from '../request.js';
+import type { IRequestLoader, RequestLoaderStatus } from './request_loader.js';
+import type { IRequestManager } from './request_manager.js';
 /** @internal */
 export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
 /** @internal */
 export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
-/**
- * Represents a static list of URLs to crawl.
- */
-export interface IRequestList {
-    /**
-     * Returns the total number of unique requests present in the list.
-     */
-    length(): number;
-    /**
-     * Returns `true` if all requests were already handled and there are no more left.
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * Resolves to `true` if the next call to {@link IRequestList.fetchNextRequest} function
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the list is empty, there might be some pending requests currently being processed.
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * Returns number of handled requests.
-     */
-    handledCount(): number;
-    /**
-     * Persists the current state of the `IRequestList` into the default {@link KeyValueStore}.
-     * The state is persisted automatically in regular intervals, but calling this method manually
-     * is useful in cases where you want to have the most current state available after you pause
-     * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
-     * a server migration.
-     */
-    persistState(): Promise<void>;
-    /**
-     * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
-     * using the {@link RequestList.reclaimRequest} function, if there is any.
-     * Otherwise it gets the next request from sources.
-     *
-     * The function's `Promise` resolves to `null` if there are no more
-     * requests to process.
-     */
-    fetchNextRequest(): Promise<Request | null>;
-    /**
-     * Can be used to iterate over the `RequestList` instance in a `for await .. of` loop.
-     * Provides an alternative for the repeated use of `fetchNextRequest`.
-     */
-    [Symbol.asyncIterator](): AsyncGenerator<Request>;
-    /**
-     * Reclaims request to the list if its processing failed.
-     * The request will become available in the next `this.fetchNextRequest()`.
-     */
-    reclaimRequest(request: Request): Promise<void>;
-    /**
-     * Marks request as handled after successful processing.
-     */
-    markRequestHandled(request: Request): Promise<void>;
-    /**
-     * @internal
-     */
-    inProgress: Set<string>;
-}
 export interface RequestListOptions {
     /**
      * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
@@ -143,7 +88,7 @@ export interface RequestListOptions {
      * Takes advantage of the internal address rotation and authentication process.
      * If undefined, the `requestsFromUrl` requests will be made without proxy.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    proxyConfiguration?: IProxyConfiguration;
     /**
      * Identifies the key in the default key-value store under which `RequestList` periodically stores its
      * state (i.e. which URLs were crawled and which not).
@@ -182,7 +127,8 @@ export interface RequestListOptions {
      * ```
      *
      * Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList`
-     * is to use the `stateKeyPrefix` parameter instead.
+     * is to use the `stateKeyPrefix` parameter instead. When both are set, a record persisted under
+     * {@link RequestListOptions.persistStateKey} takes precedence over this option.
      */
     state?: RequestListState;
     /**
@@ -202,7 +148,13 @@ export interface RequestListOptions {
      */
     keepDuplicateUrls?: boolean;
     /** @internal */
-    config?: Configuration;
+    configuration?: Configuration;
+    /**
+     * The HTTP client to be used to download `requestsFromUrl` URLs.
+     *
+     * If not specified the `RequestList` will use the default HTTP client.
+     */
+    httpClient?: BaseHttpClient;
 }
 /**
  * Represents a static list of URLs to crawl.
@@ -226,8 +178,8 @@ export interface RequestListOptions {
  * > In practical terms, such a combination can be useful when there is a large number of initial URLs,
  * > but more URLs would be added dynamically by the crawler.
  *
- * `RequestList` has an internal state where it stores information about which requests were already handled,
- * which are in progress and which were reclaimed. The state may be automatically persisted to the default
+ * `RequestList` has an internal state where it stores information about which requests were already handled
+ * and which are in progress. The state may be automatically persisted to the default
  * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
  * event that is periodically emitted by {@link EventManager}.
@@ -265,49 +217,24 @@ export interface RequestListOptions {
  * ```
  * @category Sources
  */
-export declare class RequestList implements IRequestList {
-    private log;
+export declare class RequestList implements IRequestLoader {
+    #private;
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
      * @internal
      */
-    requests: (Request | RequestOptions)[];
-    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    private nextIndex;
-    /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
-    private uniqueKeyToIndex;
+    readonly requests: (Request | RequestOptions)[];
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
-    inProgress: Set<string>;
-    /**
-     * Set of `uniqueKey`s of requests for which reclaimRequest() was called.
-     * @internal
-     */
-    reclaimed: Set<string>;
-    /**
-     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
-     * @internal
-     */
-    isStatePersisted: boolean;
+    get inProgress(): Set<string>;
     /**
      * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
      * @internal
      */
     areRequestsPersisted: boolean;
-    private isLoading;
-    private isInitialized;
-    private persistStateKey?;
-    private persistRequestsKey?;
-    private initialState?;
-    private store?;
-    private keepDuplicateUrls;
-    private sources;
-    private sourcesFunction?;
-    private proxyConfiguration?;
-    private events;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
@@ -324,33 +251,30 @@ export declare class RequestList implements IRequestList {
      * This needs to be done in a memory efficient way. We should update the input
      * to a Stream once apify-client supports streams.
      */
-    protected _addPersistedRequests(persistedRequests: Buffer): Promise<void>;
+    private addPersistedRequests;
     /**
      * Add Requests from both options.sources and options.sourcesFunction.
      * This function is called only when persisted sources were not loaded.
      * We need to avoid keeping both sources and requests in memory
      * to reduce memory footprint with very large sources.
      */
-    protected _addRequestsFromSources(): Promise<void>;
+    private addRequestsFromSources;
     /**
      * @inheritDoc
      */
     persistState(): Promise<void>;
+    /**
+     * Removes the `PERSIST_STATE` event listener registered during initialization and persists
+     * the current state one last time. Call this when you are done with the `RequestList` to avoid
+     * leaking the listener (and the requests it retains) on the shared event manager.
+     */
+    teardown(): Promise<void>;
     /**
      * Unlike persistState(), this is used only internally, since the sources
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
-    protected _persistRequests(): Promise<void>;
-    /**
-     * Restores RequestList state from a state object.
-     */
-    protected _restoreState(state?: RequestListState): void;
-    /**
-     * Attempts to load state and requests using the `RequestList` configuration
-     * and returns a tuple of [state, requests] where each may be null if not loaded.
-     */
-    protected _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]>;
+    private persistRequests;
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
@@ -359,11 +283,7 @@ export declare class RequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    isEmpty(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * @inheritDoc
      */
@@ -376,47 +296,53 @@ export declare class RequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    markRequestHandled(request: Request): Promise<void>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(request: Request): Promise<void>;
+    markRequestAsHandled(request: Request): Promise<void>;
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]): Promise<void>;
-    protected _getPersistedState<T>(key: string): Promise<T>;
+    private addFetchedRequests;
+    private getPersistedState;
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
+    private fetchRequestsFromUrl;
     /**
      * Adds given request.
      * If the `source` parameter is a string or plain object and not an instance
      * of a `Request`, then the function creates a `Request` instance.
      */
-    protected _addRequest(source: RequestListSource): void;
+    private addRequest;
     /**
      * Helper function that validates unique key.
      * Throws an error if uniqueKey is not a non-empty string.
      */
-    protected _ensureUniqueKeyValid(uniqueKey: string): void;
+    private ensureUniqueKeyValid;
     /**
-     * Checks that request is not reclaimed and throws an error if so.
+     * Checks that a request is currently being processed and throws an error if not.
      */
-    protected _ensureInProgressAndNotReclaimed(uniqueKey: string): void;
+    private ensureInProgress;
     /**
      * Throws an error if request list wasn't initialized.
      */
-    protected _ensureIsInitialized(): void;
+    private ensureIsInitialized;
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
-    length(): number;
+    getTotalCount(): Promise<number>;
+    /**
+     * Returns the number of pending requests in the `RequestList`.
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
     /**
      * @inheritDoc
      */
-    handledCount(): number;
+    getHandledCount(): Promise<number>;
     /**
      * Opens a request list and returns a promise resolving to an instance
      * of the {@link RequestList} class that is already initialized.
@@ -474,7 +400,7 @@ export declare class RequestList implements IRequestList {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _downloadListOfUrls;
+    private downloadListOfUrls;
 }
 /**
  * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.
@@ -503,4 +429,3 @@ export interface RequestListState {
 type RequestListSource = string | Source;
 export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;
 export {};
-//# sourceMappingURL=request_list.d.ts.map