@crawlee/core 4.0.0-beta.66 → 4.0.0-beta.67

package/memory-storage/resource-clients/common/base-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-client.js","sourceRoot":"","sources":["../../../../src/memory-storage/resource-clients/common/base-client.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,UAAU;IACnB,EAAE,CAAS;IAEX,YAAY,EAAU;QAClB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;CACJ"}

package/memory-storage/resource-clients/dataset.d.ts

@@ -0,0 +1,41 @@
+import type * as storage from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface DatasetBackendOptions {
+    id?: string;
+    name?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    itemCount: number;
+    private readonly datasetEntries;
+    private readonly storageBackend;
+    constructor(options: DatasetBackendOptions);
+    getMetadata(): Promise<storage.DatasetInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;
+    private getDataPage;
+    pushData(items: Data[]): Promise<void>;
+    toDatasetInfo(): storage.DatasetInfo;
+    private generateLocalEntryName;
+    private getStartAndEndIndexes;
+    private updateTimestamps;
+}
+//# sourceMappingURL=dataset.d.ts.map

package/memory-storage/resource-clients/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../../src/memory-storage/resource-clients/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAcrD,MAAM,WAAW,qBAAqB;IAClC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,oBAAoB,CAAC;CACxC;AAED,qBAAa,cAAc,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,CAC5D,SAAQ,UACR,YAAW,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC;IAEvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,SAAS,SAAK;IAEd,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA2B;IAC1D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;gBAE1C,OAAO,EAAE,qBAAqB;IAOpC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;IAK3C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B,OAAO,CAAC,OAAO,GAAE,OAAO,CAAC,yBAA8B,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAgBhF,WAAW;IA2BnB,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAS5C,aAAa,IAAI,OAAO,CAAC,WAAW;IAWpC,OAAO,CAAC,sBAAsB;IAI9B,OAAO,CAAC,qBAAqB;IAM7B,OAAO,CAAC,gBAAgB;CAO3B"}

package/memory-storage/resource-clients/dataset.js

@@ -0,0 +1,114 @@
+import { randomUUID } from 'node:crypto';
+import { s } from '@sapphire/shapeshift';
+import { BaseClient } from './common/base-client.js';
+/**
+ * This is what API returns in the x-apify-pagination-limit
+ * header when no limit query parameter is used.
+ */
+const LIST_ITEMS_LIMIT = 999_999_999_999;
+/**
+ * Number of characters of the dataset item entry names.
+ * E.g.: 000000019 - 9 digits
+ */
+const LOCAL_ENTRY_NAME_DIGITS = 9;
+export class DatasetBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    itemCount = 0;
+    datasetEntries = new Map();
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toDatasetInfo();
+    }
+    async drop() {
+        const storeIndex = this.storageBackend.datasetBackendCache.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
+            oldBackend.itemCount = 0;
+            oldBackend.datasetEntries.clear();
+        }
+    }
+    async purge() {
+        this.itemCount = 0;
+        this.datasetEntries.clear();
+        this.updateTimestamps(true);
+    }
+    getData(options = {}) {
+        const { desc, limit, offset } = s
+            .object({
+            desc: s.boolean().optional(),
+            limit: s.number().int().optional(),
+            offset: s.number().int().optional(),
+        })
+            .parse(options);
+        return this.getDataPage({
+            desc,
+            offset: offset ?? 0,
+            limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
+        });
+    }
+    async getDataPage(options = {}) {
+        const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
+        const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
+        const items = [];
+        for (let idx = start; idx < end; idx++) {
+            const entryNumber = this.generateLocalEntryName(idx);
+            items.push(this.datasetEntries.get(entryNumber));
+        }
+        this.updateTimestamps(false);
+        return {
+            count: items.length,
+            desc: desc ?? false,
+            items: desc ? items.reverse() : items,
+            limit,
+            offset,
+            total: this.itemCount,
+        };
+    }
+    async pushData(items) {
+        for (const entry of items) {
+            const idx = this.generateLocalEntryName(++this.itemCount);
+            this.datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+        }
+        this.updateTimestamps(true);
+    }
+    toDatasetInfo() {
+        return {
+            id: this.id,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            itemCount: this.itemCount,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+        };
+    }
+    generateLocalEntryName(idx) {
+        return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
+    }
+    getStartAndEndIndexes(offset, limit = this.itemCount) {
+        const start = offset + 1;
+        const end = Math.min(offset + limit, this.itemCount) + 1;
+        return [start, end];
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}
+//# sourceMappingURL=dataset.js.map

package/memory-storage/resource-clients/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../../src/memory-storage/resource-clients/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAGzC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD;;;GAGG;AACH,MAAM,gBAAgB,GAAG,eAAe,CAAC;AAEzC;;;GAGG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAclC,MAAM,OAAO,cACT,SAAQ,UAAU;IAGlB,IAAI,CAAU;IACd;;;OAGG;IACH,QAAQ,CAAS;IACjB,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,SAAS,GAAG,CAAC,CAAC;IAEG,cAAc,GAAG,IAAI,GAAG,EAAgB,CAAC;IACzC,cAAc,CAAuB;IAEtD,YAAY,OAA8B;QACtC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,WAAW;QACb,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtG,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACnF,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC;YACzB,UAAU,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QACtC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,UAA6C,EAAE;QACnD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC;aAC5B,MAAM,CAAC;YACJ,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC5B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACtC,CAAC;aACD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEpB,OAAO,IAAI,CAAC,WAAW,CAAC;YACpB,IAAI;YACJ,MAAM,EAAE,MAAM,IAAI,CAAC;YACnB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,gBAAgB,EAAE,gBAAgB,CAAC;SAC/D,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,UAA6C,EAAE;QACrE,MAAM,EAAE,KAAK,GAAG,gBAAgB,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;QAE/D,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAC3C,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAC5D,KAAK,CACR,CAAC;QAEF,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC,CAAC;QACtD,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,IAAI,KAAK;YACnB,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK;YACrC,KAAK;YACL,MAAM;YACN,KAAK,EAAE,IAAI,CAAC,SAAS;SACxB,CAAC;IACN,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa;QACxB,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAS,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,aAAa;QACT,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;SAClB,CAAC;IACN,CAAC;IAEO,sBAAsB,CAAC,GAAW;QACtC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAChE,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAU,CAAC;IACjC,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;CACJ"}

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -0,0 +1,64 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface KeyValueStoreBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalKeyRecord {
+    key: string;
+    value: Buffer;
+    contentType?: string;
+    extension: string;
+}
+export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    private readonly keyValueEntries;
+    private readonly storageBackend;
+    constructor(options: KeyValueStoreBackendOptions);
+    getMetadata(): Promise<storage.KeyValueStoreInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    purgeExceptInput(): Promise<void>;
+    listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    getPublicUrl(key: string): Promise<string | undefined>;
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    recordExists(key: string): Promise<boolean>;
+    getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
+    setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
+    deleteValue(key: string): Promise<void>;
+    toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
+    private updateTimestamps;
+}
+//# sourceMappingURL=key-value-store.d.ts.map

package/memory-storage/resource-clients/key-value-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store.d.ts","sourceRoot":"","sources":["../../../src/memory-storage/resource-clients/key-value-store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAWrD,MAAM,WAAW,2BAA2B;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,oBAAoB,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,oBAAqB,SAAQ,UAAW,YAAW,OAAO,CAAC,oBAAoB;IACxF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IAExB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAwC;IACxE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;gBAE1C,OAAO,EAAE,2BAA2B;IAO1C,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAKjD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IASrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B;;;;;OAKG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAUjC,QAAQ,CAAC,OAAO,GAAE,OAAO,CAAC,4BAAiC,GAAG,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC;IAoDhH;;;;OAIG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAM5D;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAM3C,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAwBvE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqDjE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS7C,mBAAmB,IAAI,OAAO,CAAC,iBAAiB;IAUhD,OAAO,CAAC,gBAAgB;CAO3B"}

package/memory-storage/resource-clients/key-value-store.js

@@ -0,0 +1,204 @@
+import { randomUUID } from 'node:crypto';
+import { s } from '@sapphire/shapeshift';
+import { isStream, toBuffer } from '../utils.js';
+import { BaseClient } from './common/base-client.js';
+import mime from 'mime-types';
+const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+/**
+ * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
+ * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
+ */
+const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
+export class KeyValueStoreBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    keyValueEntries = new Map();
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toKeyValueStoreInfo();
+    }
+    async drop() {
+        const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
+            oldBackend.keyValueEntries.clear();
+        }
+    }
+    async purge() {
+        this.keyValueEntries.clear();
+        this.updateTimestamps(true);
+    }
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    async purgeExceptInput() {
+        for (const key of this.keyValueEntries.keys()) {
+            if (key !== KEY_VALUE_STORE_INPUT_KEY) {
+                this.keyValueEntries.delete(key);
+            }
+        }
+        this.updateTimestamps(true);
+    }
+    async listKeys(options = {}) {
+        const { prefix, exclusiveStartKey, limit } = s
+            .object({
+            prefix: s.string().optional(),
+            exclusiveStartKey: s.string().optional(),
+            limit: s.number().int().greaterThan(0).optional(),
+        })
+            .parse(options);
+        const items = [];
+        for (const record of this.keyValueEntries.values()) {
+            const size = Buffer.byteLength(record.value);
+            items.push({
+                key: record.key,
+                size,
+                contentType: record.contentType ?? 'application/octet-stream',
+            });
+        }
+        // Lexically sort to emulate API.
+        items.sort((a, b) => a.key.localeCompare(b.key));
+        let filteredItems = items.filter((item) => !prefix || item.key.startsWith(prefix));
+        if (exclusiveStartKey) {
+            const keyPos = filteredItems.findIndex((item) => item.key === exclusiveStartKey);
+            if (keyPos === -1) {
+                throw new Error(`exclusiveStartKey "${exclusiveStartKey}" was not found in the key-value store. ` +
+                    `This is likely a bug — the key may have been deleted between paginated listKeys calls.`);
+            }
+            filteredItems = filteredItems.slice(keyPos + 1);
+        }
+        const isTruncated = limit !== undefined && filteredItems.length > limit;
+        const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
+        const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
+        this.updateTimestamps(false);
+        return {
+            items: pageItems,
+            count: pageItems.length,
+            limit: limit ?? pageItems.length,
+            exclusiveStartKey,
+            isTruncated,
+            nextExclusiveStartKey,
+        };
+    }
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    async getPublicUrl(key) {
+        s.string().parse(key);
+        return undefined;
+    }
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    async recordExists(key) {
+        s.string().parse(key);
+        return this.keyValueEntries.has(key);
+    }
+    async getValue(key) {
+        s.string().parse(key);
+        const entry = this.keyValueEntries.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        // Return raw bytes + verbatim content type. Parsing is the frontend's job (see the
+        // KeyValueStore codec). The mime fallback reconstructs the content type for on-disk records.
+        const record = {
+            key: entry.key,
+            value: entry.value,
+            // mime.contentType returns `false` for unknown extensions; fall back to undefined so the
+            // frontend treats it as "no content type" rather than a bogus value.
+            contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
+        };
+        this.updateTimestamps(false);
+        return record;
+    }
+    async setValue(record) {
+        s.object({
+            key: s.string().lengthGreaterThan(0),
+            value: s.union([
+                s.null(),
+                s.string(),
+                s.number(),
+                s.instance(Buffer),
+                s.instance(ArrayBuffer),
+                s.typedArray(),
+                // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
+                s.object({}).setValidationEnabled(false),
+            ]),
+            contentType: s.string().lengthGreaterThan(0).optional(),
+        }).parse(record);
+        const { key } = record;
+        let { value } = record;
+        // The frontend (KeyValueStore codec) serializes the value and resolves its content type
+        // before it reaches the backend. We only need it here for on-disk extension bookkeeping.
+        const contentType = record.contentType ?? 'application/octet-stream';
+        const extension = mime.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+        // Draining a stream into a Buffer for storage is the backend's responsibility.
+        if (isStream(value)) {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(chunk);
+            }
+            value = Buffer.concat(chunks);
+        }
+        // This backend is a byte transport: it stores and returns raw bytes regardless of the input
+        // shape. Streams were drained above; encode strings to UTF-8 bytes and normalize
+        // ArrayBuffer / typed-array views to a Buffer over the same memory.
+        const normalizedValue = typeof value === 'string'
+            ? Buffer.from(value, 'utf-8')
+            : toBuffer(value);
+        const _record = {
+            extension,
+            key,
+            value: normalizedValue,
+            contentType,
+        };
+        this.keyValueEntries.set(key, _record);
+        this.updateTimestamps(true);
+    }
+    async deleteValue(key) {
+        s.string().parse(key);
+        if (this.keyValueEntries.has(key)) {
+            this.keyValueEntries.delete(key);
+            this.updateTimestamps(true);
+        }
+    }
+    toKeyValueStoreInfo() {
+        return {
+            id: this.id,
+            name: this.name,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            modifiedAt: this.modifiedAt,
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}
+//# sourceMappingURL=key-value-store.js.map

package/memory-storage/resource-clients/key-value-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store.js","sourceRoot":"","sources":["../../../src/memory-storage/resource-clients/key-value-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAGzC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,IAAI,MAAM,YAAY,CAAC;AAE9B,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAE3C;;;GAGG;AACH,MAAM,yBAAyB,GAAG,OAAO,CAAC;AAqB1C,MAAM,OAAO,oBAAqB,SAAQ,UAAU;IAChD,IAAI,CAAU;IACd;;;OAGG;IACH,QAAQ,CAAS;IACjB,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IAEP,eAAe,GAAG,IAAI,GAAG,EAA6B,CAAC;IACvD,cAAc,CAAuB;IAEtD,YAAY,OAAoC;QAC5C,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,WAAW;QACb,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,mBAAmB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5G,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,yBAAyB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACzF,UAAU,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACvC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,gBAAgB;QAClB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,GAAG,KAAK,yBAAyB,EAAE,CAAC;gBACpC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAgD,EAAE;QAC7D,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,CAAC;aACzC,MAAM,CAAC;YACJ,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC7B,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEpB,MAAM,KAAK,GAAoC,EAAE,CAAC;QAElD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC;gBACP,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI;gBACJ,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,0BAA0B;aAChE,CAAC,CAAC;QACP,CAAC;QAED,iCAAiC;QACjC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjD,IAAI,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnF,IAAI,iBAAiB,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC;YACjF,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CACX,sBAAsB,iBAAiB,0CAA0C;oBAC7E,wFAAwF,CAC/F,CAAC;YACN,CAAC;YACD,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,WAAW,GAAG,KAAK,KAAK,SAAS,IAAI,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;QACxE,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;QAC9E,MAAM,qBAAqB,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAE5F,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,OAAO;YACH,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,SAAS,CAAC,MAAM;YACvB,KAAK,EAAE,KAAK,IAAI,SAAS,CAAC,MAAM;YAChC,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACxB,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtB,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,GAAW;QACtB,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,mFAAmF;QACnF,6FAA6F;QAC7F,MAAM,MAAM,GAAgC;YACxC,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,yFAAyF;YACzF,qEAAqE;YACrE,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;SACrF,CAAC;QAEF,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAwC;QACnD,CAAC,CAAC,MAAM,CAAC;YACL,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACpC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;gBACX,CAAC,CAAC,IAAI,EAAE;gBACR,CAAC,CAAC,MAAM,EAAE;gBACV,CAAC,CAAC,MAAM,EAAE;gBACV,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAClB,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACvB,CAAC,CAAC,UAAU,EAAE;gBACd,iHAAiH;gBACjH,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC;aAC3C,CAAC;YACF,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;SAC1D,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEjB,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;QACvB,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QACvB,wFAAwF;QACxF,yFAAyF;QACzF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,0BAA0B,CAAC;QAErE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,4BAA4B,CAAC;QAE9E,+EAA+E;QAC/E,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAClB,MAAM,MAAM,GAAG,EAAE,CAAC;YAClB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YACD,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,4FAA4F;QAC5F,iFAAiF;QACjF,oEAAoE;QACpE,MAAM,eAAe,GACjB,OAAO,KAAK,KAAK,QAAQ;YACrB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC;YAC7B,CAAC,CAAC,QAAQ,CAAC,KAA+C,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAG;YACZ,SAAS;YACT,GAAG;YACH,KAAK,EAAE,eAAe;YACtB,WAAW;SACc,CAAC;QAE9B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAEvC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW;QACzB,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,mBAAmB;QACf,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;CACJ"}

package/memory-storage/resource-clients/request-queue.d.ts

@@ -0,0 +1,97 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface RequestQueueBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalRequest {
+    id: string;
+    orderNo: number | null;
+    url: string;
+    uniqueKey: string;
+    method: storage.RequestSchema['method'];
+    retryCount: number;
+    json: string;
+}
+export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    handledRequestCount: number;
+    pendingRequestCount: number;
+    /**
+     * Serializes every operation that reads-then-writes this backend's shared queue state — the
+     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
+     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
+     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
+     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
+     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
+     */
+    private readonly queueStateMutex;
+    private forefrontRequestIds;
+    /**
+     * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
+     * "in progress" and will not be handed out again by {@link fetchNextRequest}.
+     *
+     * Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
+     * process and is never shared with another consumer, so there is no need for an expiring,
+     * cross-process-visible lock — tracking in-progress requests in this set is enough.
+     */
+    private readonly inProgressRequestIds;
+    private readonly requests;
+    private readonly storageBackend;
+    constructor(options: RequestQueueBackendOptions);
+    getMetadata(): Promise<storage.RequestQueueInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    private requestKeyIterator;
+    /**
+     * Scans the queue and returns the pending head — requests that are neither handled nor currently
+     * in progress — ordered by `orderNo`, deduplicated.
+     *
+     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
+     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
+     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
+     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
+     * crawler down while it is still handling the last requests.
+     *
+     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
+     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
+     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
+     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     */
+    private listPendingHead;
+    fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
+    addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
+    reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    /**
+     * Returns all pending (not yet handled, not currently in progress) requests in the queue, ordered
+     * the same way {@link fetchNextRequest} would hand them out. This does not mutate the queue,
+     * nothing is marked in progress.
+     */
+    listItems(): Promise<storage.UpdateRequestSchema[]>;
+    toRequestQueueInfo(): storage.RequestQueueInfo;
+    private updateTimestamps;
+    private _jsonToRequest;
+    private _createInternalRequest;
+    private _calculateOrderNo;
+}
+//# sourceMappingURL=request-queue.d.ts.map

package/memory-storage/resource-clients/request-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue.d.ts","sourceRoot":"","sources":["../../../src/memory-storage/resource-clients/request-queue.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAqBrD,MAAM,WAAW,0BAA0B;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,oBAAoB,CAAC;CACxC;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,mBAAoB,SAAQ,UAAW,YAAW,OAAO,CAAC,mBAAmB;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,mBAAmB,SAAK;IACxB,mBAAmB,SAAK;IACxB;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAoB;IACpD,OAAO,CAAC,mBAAmB,CAAgB;IAE3C;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAqB;IAE1D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;gBAE1C,OAAO,EAAE,0BAA0B;IAOzC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAKhD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB5B,OAAO,CAAE,kBAAkB;IAU3B;;;;;;;;;;;;;;OAcG;YACW,eAAe;IAuDvB,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAwBpE,kBAAkB,CACpB,QAAQ,EAAE,OAAO,CAAC,aAAa,EAAE,EACjC,OAAO,GAAE,OAAO,CAAC,4BAAiC,GACnD,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC;IA6DpC,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAQ/E,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAiD3G,cAAc,CAChB,OAAO,EAAE,OAAO,CAAC,mBAAmB,EACpC,OAAO,GAAE,OAAO,CAAC,4BAAiC,GACnD,OAAO,CAAC,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAgD5C,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAqB3B,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAsBpC;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAezD,kBAAkB,IAAI,OAAO,CAAC,gBAAgB;IAa9C,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,iBAAiB;CAO5B"}