@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/memory-storage/memory-storage.js

@@ -0,0 +1,130 @@
+import { randomUUID } from 'node:crypto';
+import { DatasetBackend } from './resource-clients/dataset.js';
+import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
+import { RequestQueueBackend } from './resource-clients/request-queue.js';
+/** The alias the default (unnamed) storage is opened under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+export class MemoryStorageBackend {
+    logger;
+    /**
+     * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
+     * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
+     */
+    #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
+    keyValueStoreBackendCache = [];
+    datasetBackendCache = [];
+    requestQueueBackendCache = [];
+    constructor(options = {}) {
+        this.logger = options.logger;
+    }
+    /**
+     * Return a per-instance unique cache key so that distinct `MemoryStorageBackend` instances get separate
+     * cache partitions in the storage backend cache.
+     */
+    getStorageBackendCacheKey() {
+        return this.#instanceCacheKey;
+    }
+    static #resolveStorageKey(options) {
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in the storage frontends, so that a backend used
+        // directly lands on the very storage the frontends would have opened.
+        const alias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        // `alias` covers the identifier-less case, so one of the three is always set.
+        const rawKey = alias ?? options.name ?? options.id;
+        // Normalize the internal __default__ alias to the user-facing 'default' name.
+        const cacheKey = rawKey === DEFAULT_STORAGE_ALIAS ? 'default' : rawKey;
+        return { isAlias: alias !== undefined, cacheKey };
+    }
+    async createDatasetBackend(options = {}) {
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
+        }
+        const newStore = new DatasetBackend({
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
+            storageBackend: this,
+        });
+        this.datasetBackendCache.push(newStore);
+        return newStore;
+    }
+    async createKeyValueStoreBackend(options = {}) {
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
+        }
+        const newStore = new KeyValueStoreBackend({
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
+            storageBackend: this,
+        });
+        this.keyValueStoreBackendCache.push(newStore);
+        return newStore;
+    }
+    async createRequestQueueBackend(options = {}) {
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+            queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
+        }
+        const newStore = new RequestQueueBackend({
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
+            storageBackend: this,
+        });
+        this.requestQueueBackendCache.push(newStore);
+        return newStore;
+    }
+    async storageExists(id, type) {
+        let backends;
+        switch (type) {
+            case 'Dataset':
+                backends = this.datasetBackendCache;
+                break;
+            case 'KeyValueStore':
+                backends = this.keyValueStoreBackendCache;
+                break;
+            case 'RequestQueue':
+                backends = this.requestQueueBackendCache;
+                break;
+            default:
+                return false;
+        }
+        // In-memory storage only knows about backends in its cache.
+        return backends.some((store) => store.id === id);
+    }
+    /**
+     * Cleans up the run-scoped storages before the run starts. For the in-memory storage this simply
+     * resets the in-memory state of the cached backends.
+     */
+    async purge() {
+        // `#resolveStorageKey` leaves `name` unset for the default and alias-keyed storages, which is what
+        // marks them as run-scoped. `'default'` is the exception — it collapses onto the default storage.
+        const isRunScoped = (store) => store.name === undefined || store.name === 'default';
+        const isDefault = (store) => store.name === 'default' || store.cacheKey === 'default';
+        const purgeRunScoped = async (cache, purgeStore) => {
+            await Promise.all(cache.filter(isRunScoped).map(async (store) => purgeStore(store)));
+        };
+        await Promise.all([
+            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
+            purgeRunScoped(this.keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
+            purgeRunScoped(this.datasetBackendCache, async (store) => store.purge()),
+            purgeRunScoped(this.requestQueueBackendCache, async (store) => store.purge()),
+        ]);
+    }
+    /**
+     * This method should be called at the end of the process. The in-memory storage holds no resources
+     * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
+     */
+    async teardown() {
+        // Nothing to tear down for in-memory storage.
+    }
+}

package/memory-storage/resource-clients/common/base-client.d.ts

@@ -0,0 +1,4 @@
+export declare class BaseClient {
+    id: string;
+    constructor(id: string);
+}

package/memory-storage/resource-clients/common/base-client.js

@@ -0,0 +1,6 @@
+export class BaseClient {
+    id;
+    constructor(id) {
+        this.id = id;
+    }
+}

package/memory-storage/resource-clients/dataset.d.ts

@@ -0,0 +1,40 @@
+import type * as storage from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface DatasetBackendOptions {
+    id?: string;
+    name?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    itemCount: number;
+    private readonly storageBackend;
+    constructor(options: DatasetBackendOptions);
+    getMetadata(): Promise<storage.DatasetInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;
+    private getDataPage;
+    pushData(items: Data[]): Promise<void>;
+    toDatasetInfo(): storage.DatasetInfo;
+    private generateLocalEntryName;
+    private getStartAndEndIndexes;
+    private updateTimestamps;
+}

package/memory-storage/resource-clients/dataset.js

@@ -0,0 +1,114 @@
+import { randomUUID } from 'node:crypto';
+import { s } from '@sapphire/shapeshift';
+import { BaseClient } from './common/base-client.js';
+/**
+ * This is what API returns in the x-apify-pagination-limit
+ * header when no limit query parameter is used.
+ */
+const LIST_ITEMS_LIMIT = 999_999_999_999;
+/**
+ * Number of characters of the dataset item entry names.
+ * E.g.: 000000019 - 9 digits
+ */
+const LOCAL_ENTRY_NAME_DIGITS = 9;
+export class DatasetBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    itemCount = 0;
+    #datasetEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toDatasetInfo();
+    }
+    async drop() {
+        const storeIndex = this.storageBackend.datasetBackendCache.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
+            oldBackend.itemCount = 0;
+            oldBackend.#datasetEntries.clear();
+        }
+    }
+    async purge() {
+        this.itemCount = 0;
+        this.#datasetEntries.clear();
+        this.updateTimestamps(true);
+    }
+    getData(options = {}) {
+        const { desc, limit, offset } = s
+            .object({
+            desc: s.boolean().optional(),
+            limit: s.number().int().optional(),
+            offset: s.number().int().optional(),
+        })
+            .parse(options);
+        return this.getDataPage({
+            desc,
+            offset: offset ?? 0,
+            limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
+        });
+    }
+    async getDataPage(options = {}) {
+        const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
+        const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
+        const items = [];
+        for (let idx = start; idx < end; idx++) {
+            const entryNumber = this.generateLocalEntryName(idx);
+            items.push(this.#datasetEntries.get(entryNumber));
+        }
+        this.updateTimestamps(false);
+        return {
+            count: items.length,
+            desc: desc ?? false,
+            items: desc ? items.reverse() : items,
+            limit,
+            offset,
+            total: this.itemCount,
+        };
+    }
+    async pushData(items) {
+        for (const entry of items) {
+            const idx = this.generateLocalEntryName(++this.itemCount);
+            this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+        }
+        this.updateTimestamps(true);
+    }
+    toDatasetInfo() {
+        return {
+            id: this.id,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            itemCount: this.itemCount,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+        };
+    }
+    generateLocalEntryName(idx) {
+        return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
+    }
+    getStartAndEndIndexes(offset, limit = this.itemCount) {
+        const start = offset + 1;
+        const end = Math.min(offset + limit, this.itemCount) + 1;
+        return [start, end];
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -0,0 +1,63 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface KeyValueStoreBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalKeyRecord {
+    key: string;
+    value: Buffer;
+    contentType?: string;
+    extension: string;
+}
+export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    private readonly storageBackend;
+    constructor(options: KeyValueStoreBackendOptions);
+    getMetadata(): Promise<storage.KeyValueStoreInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    purgeExceptInput(): Promise<void>;
+    listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    getPublicUrl(key: string): Promise<string | undefined>;
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    recordExists(key: string): Promise<boolean>;
+    getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
+    setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
+    deleteValue(key: string): Promise<void>;
+    toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
+    private updateTimestamps;
+}

package/memory-storage/resource-clients/key-value-store.js

@@ -0,0 +1,204 @@
+import { randomUUID } from 'node:crypto';
+import { s } from '@sapphire/shapeshift';
+import { isStream, toBuffer } from '../utils.js';
+import { BaseClient } from './common/base-client.js';
+import mime from 'mime-types';
+const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+/**
+ * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
+ * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
+ */
+const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
+export class KeyValueStoreBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    #keyValueEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toKeyValueStoreInfo();
+    }
+    async drop() {
+        const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
+            oldBackend.#keyValueEntries.clear();
+        }
+    }
+    async purge() {
+        this.#keyValueEntries.clear();
+        this.updateTimestamps(true);
+    }
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    async purgeExceptInput() {
+        for (const key of this.#keyValueEntries.keys()) {
+            if (key !== KEY_VALUE_STORE_INPUT_KEY) {
+                this.#keyValueEntries.delete(key);
+            }
+        }
+        this.updateTimestamps(true);
+    }
+    async listKeys(options = {}) {
+        const { prefix, exclusiveStartKey, limit } = s
+            .object({
+            prefix: s.string().optional(),
+            exclusiveStartKey: s.string().optional(),
+            limit: s.number().int().greaterThan(0).optional(),
+        })
+            .parse(options);
+        const items = [];
+        for (const record of this.#keyValueEntries.values()) {
+            const size = Buffer.byteLength(record.value);
+            items.push({
+                key: record.key,
+                size,
+                contentType: record.contentType ?? 'application/octet-stream',
+            });
+        }
+        // Lexically sort to emulate API.
+        items.sort((a, b) => a.key.localeCompare(b.key));
+        let filteredItems = items.filter((item) => !prefix || item.key.startsWith(prefix));
+        if (exclusiveStartKey) {
+            const keyPos = filteredItems.findIndex((item) => item.key === exclusiveStartKey);
+            if (keyPos === -1) {
+                throw new Error(`exclusiveStartKey "${exclusiveStartKey}" was not found in the key-value store. ` +
+                    `This is likely a bug — the key may have been deleted between paginated listKeys calls.`);
+            }
+            filteredItems = filteredItems.slice(keyPos + 1);
+        }
+        const isTruncated = limit !== undefined && filteredItems.length > limit;
+        const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
+        const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
+        this.updateTimestamps(false);
+        return {
+            items: pageItems,
+            count: pageItems.length,
+            limit: limit ?? pageItems.length,
+            exclusiveStartKey,
+            isTruncated,
+            nextExclusiveStartKey,
+        };
+    }
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    async getPublicUrl(key) {
+        s.string().parse(key);
+        return undefined;
+    }
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    async recordExists(key) {
+        s.string().parse(key);
+        return this.#keyValueEntries.has(key);
+    }
+    async getValue(key) {
+        s.string().parse(key);
+        const entry = this.#keyValueEntries.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        // Return raw bytes + verbatim content type. Parsing is the frontend's job (see the
+        // KeyValueStore codec). The mime fallback reconstructs the content type for on-disk records.
+        const record = {
+            key: entry.key,
+            value: entry.value,
+            // mime.contentType returns `false` for unknown extensions; fall back to undefined so the
+            // frontend treats it as "no content type" rather than a bogus value.
+            contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
+        };
+        this.updateTimestamps(false);
+        return record;
+    }
+    async setValue(record) {
+        s.object({
+            key: s.string().lengthGreaterThan(0),
+            value: s.union([
+                s.null(),
+                s.string(),
+                s.number(),
+                s.instance(Buffer),
+                s.instance(ArrayBuffer),
+                s.typedArray(),
+                // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
+                s.object({}).setValidationEnabled(false),
+            ]),
+            contentType: s.string().lengthGreaterThan(0).optional(),
+        }).parse(record);
+        const { key } = record;
+        let { value } = record;
+        // The frontend (KeyValueStore codec) serializes the value and resolves its content type
+        // before it reaches the backend. We only need it here for on-disk extension bookkeeping.
+        const contentType = record.contentType ?? 'application/octet-stream';
+        const extension = mime.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+        // Draining a stream into a Buffer for storage is the backend's responsibility.
+        if (isStream(value)) {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(chunk);
+            }
+            value = Buffer.concat(chunks);
+        }
+        // This backend is a byte transport: it stores and returns raw bytes regardless of the input
+        // shape. Streams were drained above; encode strings to UTF-8 bytes and normalize
+        // ArrayBuffer / typed-array views to a Buffer over the same memory.
+        const normalizedValue = typeof value === 'string'
+            ? Buffer.from(value, 'utf-8')
+            : toBuffer(value);
+        const normalizedRecord = {
+            extension,
+            key,
+            value: normalizedValue,
+            contentType,
+        };
+        this.#keyValueEntries.set(key, normalizedRecord);
+        this.updateTimestamps(true);
+    }
+    async deleteValue(key) {
+        s.string().parse(key);
+        if (this.#keyValueEntries.has(key)) {
+            this.#keyValueEntries.delete(key);
+            this.updateTimestamps(true);
+        }
+    }
+    toKeyValueStoreInfo() {
+        return {
+            id: this.id,
+            name: this.name,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            modifiedAt: this.modifiedAt,
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}

package/memory-storage/resource-clients/request-queue.d.ts

@@ -0,0 +1,77 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface RequestQueueBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalRequest {
+    id: string;
+    orderNo: number | null;
+    url: string;
+    uniqueKey: string;
+    method: storage.RequestSchema['method'];
+    retryCount: number;
+    json: string;
+}
+export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    handledRequestCount: number;
+    pendingRequestCount: number;
+    private readonly storageBackend;
+    constructor(options: RequestQueueBackendOptions);
+    getMetadata(): Promise<storage.RequestQueueInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    private requestKeyIterator;
+    /**
+     * Scans the queue and returns the pending head — requests that are neither handled nor currently
+     * in progress — ordered by `orderNo`, deduplicated.
+     *
+     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
+     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
+     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
+     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
+     * crawler down while it is still handling the last requests.
+     *
+     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
+     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
+     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
+     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     */
+    private listPendingHead;
+    fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
+    addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
+    reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    /**
+     * Returns all pending (not yet handled, not currently in progress) requests in the queue, ordered
+     * the same way {@link fetchNextRequest} would hand them out. This does not mutate the queue,
+     * nothing is marked in progress.
+     */
+    listItems(): Promise<storage.UpdateRequestSchema[]>;
+    toRequestQueueInfo(): storage.RequestQueueInfo;
+    private updateTimestamps;
+    private jsonToRequest;
+    private createInternalRequest;
+    private calculateOrderNo;
+}