@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/memory-storage/resource-clients/key-value-store.js

@@ -0,0 +1,201 @@
+import { randomUUID } from 'node:crypto';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
+import { isStream, toBuffer } from '../utils.js';
+import { BaseClient } from './common/base-client.js';
+import mime from 'mime-types';
+const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+const keySchema = z.string();
+const inputRecordSchema = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.null(),
+        z.string(),
+        z.number(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // only checks the value is an actual object, not null, nor array
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
+/**
+ * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
+ * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
+ */
+const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
+export class KeyValueStoreBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    #keyValueEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toKeyValueStoreInfo();
+    }
+    async drop() {
+        const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
+            oldBackend.#keyValueEntries.clear();
+        }
+    }
+    async purge() {
+        this.#keyValueEntries.clear();
+        this.updateTimestamps(true);
+    }
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    async purgeExceptInput() {
+        for (const key of this.#keyValueEntries.keys()) {
+            if (key !== KEY_VALUE_STORE_INPUT_KEY) {
+                this.#keyValueEntries.delete(key);
+            }
+        }
+        this.updateTimestamps(true);
+    }
+    async listKeys(options = {}) {
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
+        const items = [];
+        for (const record of this.#keyValueEntries.values()) {
+            const size = Buffer.byteLength(record.value);
+            items.push({
+                key: record.key,
+                size,
+                contentType: record.contentType ?? 'application/octet-stream',
+            });
+        }
+        // Lexically sort to emulate API.
+        items.sort((a, b) => a.key.localeCompare(b.key));
+        let filteredItems = items.filter((item) => !prefix || item.key.startsWith(prefix));
+        if (exclusiveStartKey) {
+            const keyPos = filteredItems.findIndex((item) => item.key === exclusiveStartKey);
+            if (keyPos === -1) {
+                throw new Error(`exclusiveStartKey "${exclusiveStartKey}" was not found in the key-value store. ` +
+                    `This is likely a bug — the key may have been deleted between paginated listKeys calls.`);
+            }
+            filteredItems = filteredItems.slice(keyPos + 1);
+        }
+        const isTruncated = limit !== undefined && filteredItems.length > limit;
+        const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
+        const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
+        this.updateTimestamps(false);
+        return {
+            items: pageItems,
+            count: pageItems.length,
+            limit: limit ?? pageItems.length,
+            exclusiveStartKey,
+            isTruncated,
+            nextExclusiveStartKey,
+        };
+    }
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    async getPublicUrl(key) {
+        parseArgument(key, keySchema);
+        return undefined;
+    }
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    async recordExists(key) {
+        parseArgument(key, keySchema);
+        return this.#keyValueEntries.has(key);
+    }
+    async getValue(key) {
+        parseArgument(key, keySchema);
+        const entry = this.#keyValueEntries.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        // Return raw bytes + verbatim content type. Parsing is the frontend's job (see the
+        // KeyValueStore codec). The mime fallback reconstructs the content type for on-disk records.
+        const record = {
+            key: entry.key,
+            value: entry.value,
+            // mime.contentType returns `false` for unknown extensions; fall back to undefined so the
+            // frontend treats it as "no content type" rather than a bogus value.
+            contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
+        };
+        this.updateTimestamps(false);
+        return record;
+    }
+    async setValue(record) {
+        parseArgument(record, inputRecordSchema);
+        const { key } = record;
+        let { value } = record;
+        // The frontend (KeyValueStore codec) serializes the value and resolves its content type
+        // before it reaches the backend. We only need it here for on-disk extension bookkeeping.
+        const contentType = record.contentType ?? 'application/octet-stream';
+        const extension = mime.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+        // Draining a stream into a Buffer for storage is the backend's responsibility.
+        if (isStream(value)) {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(chunk);
+            }
+            value = Buffer.concat(chunks);
+        }
+        // This backend is a byte transport: it stores and returns raw bytes regardless of the input
+        // shape. Streams were drained above; encode strings to UTF-8 bytes and normalize
+        // ArrayBuffer / typed-array views to a Buffer over the same memory.
+        const normalizedValue = typeof value === 'string'
+            ? Buffer.from(value, 'utf-8')
+            : toBuffer(value);
+        const normalizedRecord = {
+            extension,
+            key,
+            value: normalizedValue,
+            contentType,
+        };
+        this.#keyValueEntries.set(key, normalizedRecord);
+        this.updateTimestamps(true);
+    }
+    async deleteValue(key) {
+        parseArgument(key, keySchema);
+        if (this.#keyValueEntries.has(key)) {
+            this.#keyValueEntries.delete(key);
+            this.updateTimestamps(true);
+        }
+    }
+    toKeyValueStoreInfo() {
+        return {
+            id: this.id,
+            name: this.name,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            modifiedAt: this.modifiedAt,
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}

package/memory-storage/resource-clients/request-queue.d.ts

@@ -0,0 +1,77 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface RequestQueueBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalRequest {
+    id: string;
+    orderNo: number | null;
+    url: string;
+    uniqueKey: string;
+    method: storage.RequestSchema['method'];
+    retryCount: number;
+    json: string;
+}
+export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    handledRequestCount: number;
+    pendingRequestCount: number;
+    private readonly storageBackend;
+    constructor(options: RequestQueueBackendOptions);
+    getMetadata(): Promise<storage.RequestQueueInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    private requestKeyIterator;
+    /**
+     * Scans the queue and returns the pending head — requests that are neither handled nor currently
+     * in progress — ordered by `orderNo`, deduplicated.
+     *
+     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
+     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
+     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
+     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
+     * crawler down while it is still handling the last requests.
+     *
+     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
+     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
+     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
+     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     */
+    private listPendingHead;
+    fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
+    addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
+    reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    /**
+     * Returns all pending (not yet handled, not currently in progress) requests in the queue, ordered
+     * the same way {@link fetchNextRequest} would hand them out. This does not mutate the queue,
+     * nothing is marked in progress.
+     */
+    listItems(): Promise<storage.UpdateRequestSchema[]>;
+    toRequestQueueInfo(): storage.RequestQueueInfo;
+    private updateTimestamps;
+    private jsonToRequest;
+    private createInternalRequest;
+    private calculateOrderNo;
+}

package/memory-storage/resource-clients/request-queue.js

@@ -0,0 +1,409 @@
+import { randomUUID } from 'node:crypto';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { AsyncQueue } from '@sapphire/async-queue';
+import { z } from 'zod';
+import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils.js';
+import { BaseClient } from './common/base-client.js';
+const uniqueKeySchema = z.string();
+export class RequestQueueBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    handledRequestCount = 0;
+    pendingRequestCount = 0;
+    /**
+     * Serializes every operation that reads-then-writes this backend's shared queue state — the
+     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
+     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
+     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
+     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
+     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
+     */
+    #queueStateMutex = new AsyncQueue();
+    #forefrontRequestIds = [];
+    /**
+     * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
+     * "in progress" and will not be handed out again by {@link fetchNextRequest}.
+     *
+     * Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
+     * process and is never shared with another consumer, so there is no need for an expiring,
+     * cross-process-visible lock — tracking in-progress requests in this set is enough.
+     */
+    #inProgressRequestIds = new Set();
+    #requests = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toRequestQueueInfo();
+    }
+    async drop() {
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
+        // operation cannot observe half-cleared state — e.g. a forefront id whose request has already been
+        // removed, which `listPendingHead` would then dereference as `undefined`.
+        await this.#queueStateMutex.wait();
+        try {
+            const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
+            if (storeIndex !== -1) {
+                const [oldBackend] = this.storageBackend.requestQueueBackendCache.splice(storeIndex, 1);
+                oldBackend.pendingRequestCount = 0;
+                // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
+                // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
+                // scan would resolve to a missing request and dereference.
+                oldBackend.#requests.clear();
+                oldBackend.#forefrontRequestIds = [];
+                oldBackend.#inProgressRequestIds.clear();
+            }
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    async purge() {
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
+        // operation cannot observe or repopulate half-cleared state across the `await` below.
+        await this.#queueStateMutex.wait();
+        try {
+            // Clear all in-memory state
+            this.#requests.clear();
+            this.#forefrontRequestIds = [];
+            this.#inProgressRequestIds.clear();
+            this.handledRequestCount = 0;
+            this.pendingRequestCount = 0;
+            this.updateTimestamps(true);
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    *requestKeyIterator() {
+        for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
+            yield this.#forefrontRequestIds[i];
+        }
+        for (const key of this.#requests.keys()) {
+            yield key;
+        }
+    }
+    /**
+     * Scans the queue and returns the pending head — requests that are neither handled nor currently
+     * in progress — ordered by `orderNo`, deduplicated.
+     *
+     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
+     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
+     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
+     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
+     * crawler down while it is still handling the last requests.
+     *
+     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
+     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
+     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
+     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     */
+    async listPendingHead(limit, detectInProgressRequests = false) {
+        const items = [];
+        let hasInProgressRequests = false;
+        // Tracks processed request IDs to avoid duplicates (request in both `forefrontRequestIds` and `requests`).
+        const seenRequestIds = new Set();
+        // Tracks handled request IDs from `forefrontRequestIds` to be removed.
+        const handledForefrontIds = new Set();
+        for (const requestId of this.requestKeyIterator()) {
+            // Once the requested page is filled we can stop — unless the caller asked us to detect
+            // in-progress requests and we have not yet seen one, in which case we must keep scanning.
+            if (items.length >= limit && (!detectInProgressRequests || hasInProgressRequests)) {
+                break;
+            }
+            if (seenRequestIds.has(requestId)) {
+                continue;
+            }
+            seenRequestIds.add(requestId);
+            const request = this.#requests.get(requestId);
+            // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
+            if (request.orderNo === null) {
+                if (this.#forefrontRequestIds.includes(requestId)) {
+                    handledForefrontIds.add(requestId);
+                }
+                continue;
+            }
+            // In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
+            // queue is not truly empty.
+            if (this.#inProgressRequestIds.has(requestId)) {
+                hasInProgressRequests = true;
+                continue;
+            }
+            if (items.length < limit) {
+                items.push(request);
+            }
+        }
+        this.#forefrontRequestIds = this.#forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
+        return {
+            items: items.sort((a, b) => a.orderNo - b.orderNo),
+            hasInProgressRequests: detectInProgressRequests ? hasInProgressRequests : undefined,
+        };
+    }
+    async fetchNextRequest() {
+        this.updateTimestamps(false);
+        await this.#queueStateMutex.wait();
+        try {
+            const { items: [head], } = await this.listPendingHead(1);
+            if (!head) {
+                return undefined;
+            }
+            // Mark the request as in progress so it is not handed out again until it is handled or
+            // reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
+            this.#inProgressRequestIds.add(head.id);
+            return this.jsonToRequest(head.json) ?? undefined;
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    async addBatchOfRequests(requests, options = {}) {
+        parseArgument(requests, schemas.storageRequestBatch);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
+        // shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
+        // concurrent operation interleaving at one of the `await` points below.
+        await this.#queueStateMutex.wait();
+        try {
+            const result = {
+                processedRequests: [],
+                unprocessedRequests: [],
+            };
+            for (const model of requests) {
+                const requestModel = this.createInternalRequest(model, options.forefront);
+                const existingRequestWithId = this.#requests.get(requestModel.id);
+                if (existingRequestWithId) {
+                    result.processedRequests.push({
+                        requestId: existingRequestWithId.id,
+                        uniqueKey: existingRequestWithId.uniqueKey,
+                        wasAlreadyHandled: existingRequestWithId.orderNo === null,
+                        wasAlreadyPresent: true,
+                    });
+                    continue;
+                }
+                this.#requests.set(requestModel.id, requestModel);
+                if (requestModel.orderNo) {
+                    this.pendingRequestCount += 1;
+                }
+                else {
+                    this.handledRequestCount += 1;
+                }
+                if (options.forefront) {
+                    this.#forefrontRequestIds.push(requestModel.id);
+                }
+                result.processedRequests.push({
+                    requestId: requestModel.id,
+                    uniqueKey: requestModel.uniqueKey,
+                    // We return wasAlreadyHandled: false even though the request may
+                    // have been added as handled, because that's how API behaves.
+                    wasAlreadyHandled: false,
+                    wasAlreadyPresent: false,
+                });
+            }
+            this.updateTimestamps(true);
+            return result;
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    async getRequest(uniqueKey) {
+        parseArgument(uniqueKey, uniqueKeySchema);
+        this.updateTimestamps(false);
+        const id = uniqueKeyToRequestId(uniqueKey);
+        const json = this.#requests.get(id)?.json;
+        return this.jsonToRequest(json);
+    }
+    async markRequestAsHandled(request) {
+        parseArgument(request, schemas.storageRequest);
+        this.updateTimestamps(false);
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
+        // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
+        // `await` points below.
+        await this.#queueStateMutex.wait();
+        try {
+            const id = uniqueKeyToRequestId(request.uniqueKey);
+            const existingRequest = this.#requests.get(id);
+            // The request must exist to be marked as handled. We intentionally do NOT require it to still
+            // be in progress: marking an already-released request handled must still succeed, otherwise
+            // the request could be handed out again and the queue would never finish.
+            if (!existingRequest) {
+                return undefined;
+            }
+            // A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
+            const wasAlreadyHandled = existingRequest.orderNo === null;
+            const handledAt = request.handledAt ?? new Date().toISOString();
+            const requestModel = this.createInternalRequest({ ...request, handledAt }, false);
+            this.#requests.set(id, requestModel);
+            // The request is no longer in progress for this client.
+            this.#inProgressRequestIds.delete(id);
+            if (!wasAlreadyHandled) {
+                this.pendingRequestCount -= 1;
+                this.handledRequestCount += 1;
+            }
+            this.updateTimestamps(true);
+            return {
+                requestId: id,
+                wasAlreadyHandled,
+                wasAlreadyPresent: true,
+            };
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    async reclaimRequest(request, options = {}) {
+        parseArgument(request, schemas.storageRequest);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        this.updateTimestamps(false);
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
+        // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
+        // across the `await` points below.
+        await this.#queueStateMutex.wait();
+        try {
+            const id = uniqueKeyToRequestId(request.uniqueKey);
+            const existingRequest = this.#requests.get(id);
+            // The request must exist and not already be handled to be reclaimed. As with
+            // `markRequestAsHandled`, we do NOT require it to still be in progress — returning an
+            // already-released request to the queue (e.g. to honor a `forefront` reorder) must still
+            // work, rather than have the reclaim silently dropped.
+            if (!existingRequest || existingRequest.orderNo === null) {
+                return undefined;
+            }
+            // Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
+            // (at the front if `forefront`).
+            const requestModel = this.createInternalRequest(request, options.forefront);
+            this.#requests.set(id, requestModel);
+            // The request is no longer in progress for this client.
+            this.#inProgressRequestIds.delete(id);
+            if (options.forefront) {
+                this.#forefrontRequestIds.push(id);
+            }
+            this.updateTimestamps(true);
+            return {
+                requestId: id,
+                wasAlreadyHandled: false,
+                wasAlreadyPresent: true,
+            };
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    async isEmpty() {
+        this.updateTimestamps(false);
+        // "Empty" means there is nothing left to fetch right now — i.e. the next `fetchNextRequest`
+        // would return `null`. Requests that are currently in progress are intentionally NOT counted
+        // here: they are not fetchable, so the queue is empty from a consumer's point of view. Whether
+        // those in-progress requests mean crawling is not yet done is a separate question, answered by
+        // `isFinished`.
+        //
+        // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
+        // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
+        await this.#queueStateMutex.wait();
+        try {
+            const { items } = await this.listPendingHead(1);
+            return items.length === 0;
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    async isFinished() {
+        this.updateTimestamps(false);
+        // The queue is finished only when there is nothing left to fetch AND nothing currently in
+        // progress. Counting in-progress requests is what allows a crawler with concurrency to keep
+        // waiting while it still holds the last requests, instead of finishing prematurely.
+        //
+        // Detecting in-progress requests requires a full scan, hence the `detectInProgressRequests`
+        // flag — unlike `fetchNextRequest`/`isEmpty`, which only need the head and can stop early.
+        //
+        // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
+        // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
+        await this.#queueStateMutex.wait();
+        try {
+            const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
+            return items.length === 0 && !hasInProgressRequests;
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    /**
+     * Returns all pending (not yet handled, not currently in progress) requests in the queue, ordered
+     * the same way {@link fetchNextRequest} would hand them out. This does not mutate the queue,
+     * nothing is marked in progress.
+     */
+    async listItems() {
+        this.updateTimestamps(false);
+        // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
+        // mutex to avoid racing a concurrent mutator at its `await` points.
+        await this.#queueStateMutex.wait();
+        try {
+            const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
+            return items.map((request) => this.jsonToRequest(request.json));
+        }
+        finally {
+            this.#queueStateMutex.shift();
+        }
+    }
+    toRequestQueueInfo() {
+        return {
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            handledRequestCount: this.handledRequestCount,
+            id: this.id,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+            pendingRequestCount: this.pendingRequestCount,
+            totalRequestCount: this.#requests.size,
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+    jsonToRequest(requestJson) {
+        if (!requestJson)
+            return undefined;
+        const request = JSON.parse(requestJson);
+        return purgeNullsFromObject(request);
+    }
+    createInternalRequest(request, forefront) {
+        const orderNo = this.calculateOrderNo(request, forefront);
+        const id = uniqueKeyToRequestId(request.uniqueKey);
+        if (request.id && request.id !== id) {
+            throw new Error('Request ID does not match its uniqueKey.');
+        }
+        const json = JSON.stringify({ ...request, id });
+        return {
+            id,
+            json,
+            method: request.method,
+            orderNo,
+            retryCount: request.retryCount ?? 0,
+            uniqueKey: request.uniqueKey,
+            url: request.url,
+        };
+    }
+    calculateOrderNo(request, forefront) {
+        if (request.handledAt)
+            return null;
+        const timestamp = Date.now();
+        return forefront ? -timestamp : timestamp;
+    }
+}