@crawlee/fs-storage 4.0.0-beta.67

package/resource-clients/dataset.js

@@ -0,0 +1,76 @@
+import { s } from '@sapphire/shapeshift';
+import { CachedIdClient } from './cached-id-client.js';
+/**
+ * `getData` options accepted by the high-level `Dataset` frontend but not supported by the native
+ * file-system backend (it can only paginate raw items by `offset`/`limit`/`desc`). They are silently
+ * ignored, so we warn once if a caller passes any of them.
+ *
+ * Implementing these in the native client is tracked in
+ * https://github.com/apify/crawlee-storage/issues/8.
+ */
+const UNSUPPORTED_GET_DATA_OPTIONS = ['clean', 'fields', 'omit', 'skipHidden', 'skipEmpty'];
+/**
+ * A file-system dataset backend backed by the native `@crawlee/fs-storage-native` Rust extension.
+ *
+ * This class is a thin adapter: it forwards each operation to the native client (which owns the
+ * on-disk format, timestamps and item counting) and converts results into the shapes expected by
+ * the `@crawlee/types` interfaces.
+ */
+export class DatasetBackend extends CachedIdClient {
+    name;
+    cacheKey;
+    nativeBackend;
+    logger;
+    constructor(options) {
+        super();
+        this.name = options.name;
+        this.cacheKey = options.cacheKey;
+        this.nativeBackend = options.nativeBackend;
+        this.logger = options.logger;
+    }
+    get datasetDirectory() {
+        return this.nativeBackend.pathToDataset;
+    }
+    static async create(options) {
+        const backend = new DatasetBackend(options);
+        backend._cachedId = (await options.nativeBackend.getMetadata()).id;
+        return backend;
+    }
+    async getMetadata() {
+        return this.nativeBackend.getMetadata();
+    }
+    async drop() {
+        await this.nativeBackend.dropStorage();
+    }
+    async purge() {
+        await this.nativeBackend.purge();
+    }
+    async pushData(items) {
+        await this.nativeBackend.pushData(items);
+    }
+    async getData(options = {}) {
+        const passedOptions = options;
+        const ignored = UNSUPPORTED_GET_DATA_OPTIONS.filter((key) => passedOptions[key] !== undefined);
+        if (ignored.length > 0) {
+            this.logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
+                `and were ignored. Only "offset", "limit" and "desc" are honored.`);
+        }
+        const { desc, limit, offset } = s
+            .object({
+            desc: s.boolean().optional(),
+            limit: s.number().int().optional(),
+            offset: s.number().int().optional(),
+        })
+            .parse(options);
+        const page = await this.nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
+        return {
+            count: page.count,
+            desc: page.desc,
+            items: page.items,
+            limit: page.limit,
+            offset: page.offset,
+            total: page.total,
+        };
+    }
+}
+//# sourceMappingURL=dataset.js.map

package/resource-clients/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAIzC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;;;;GAOG;AACH,MAAM,4BAA4B,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,CAAU,CAAC;AAcrG;;;;;;GAMG;AACH,MAAM,OAAO,cACT,SAAQ,cAAc;IAGb,IAAI,CAAU;IACd,QAAQ,CAAS;IAET,aAAa,CAAiC;IAC9C,MAAM,CAAiB;IAExC,YAAY,OAA8B;QACtC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,gBAAgB;QAChB,OAAO,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC;IAC5C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CACf,OAA8B;QAE9B,MAAM,OAAO,GAAG,IAAI,cAAc,CAAO,OAAO,CAAC,CAAC;QAClD,OAAO,CAAC,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,WAAW;QACb,OAAO,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa;QACxB,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,UAA6C,EAAE;QACzD,MAAM,aAAa,GAAG,OAAkC,CAAC;QACzD,MAAM,OAAO,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;QAC/F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAClB,sBAAsB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iDAAiD;gBACrF,kEAAkE,CACzE,CAAC;QACN,CAAC;QAED,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC;aAC5B,MAAM,CAAC;YACJ,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC5B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACtC,CAAC;aACD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEpB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,KAAK,EAAE,KAAK,CAAC,CAAC;QAExF,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAe;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;SACpB,CAAC;IACN,CAAC;CACJ"}

package/resource-clients/key-value-store.d.ts

@@ -0,0 +1,82 @@
+import type * as storage from '@crawlee/types';
+import type { CrawleeLogger } from '@crawlee/types';
+import type { FileSystemKeyValueStoreClient as NativeFileSystemKeyValueStoreBackend } from '@crawlee/fs-storage-native';
+import { CachedIdClient } from './cached-id-client.js';
+export interface KeyValueStoreBackendOptions {
+    /** The user-facing storage name, or `undefined` for unnamed (alias / default) storages. */
+    name?: string;
+    /**
+     * The key used for cache lookup in {@link FileSystemStorageBackend}. For named storages this equals
+     * the name; for alias (unnamed) storages it is the alias string. Falls back to the storage id.
+     */
+    cacheKey: string;
+    nativeBackend: NativeFileSystemKeyValueStoreBackend;
+    logger?: CrawleeLogger;
+}
+/**
+ * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
+ * extension.
+ *
+ * This adapter is a plain byte transport: values are written and read verbatim as `Buffer`s with a
+ * content type carried alongside them. Serializing arbitrary values into bytes and parsing them back
+ * is the {@link KeyValueStore} frontend codec's job, not this backend's.
+ */
+export declare class KeyValueStoreBackend extends CachedIdClient implements storage.KeyValueStoreBackend {
+    readonly name?: string;
+    readonly cacheKey: string;
+    private readonly nativeBackend;
+    constructor(options: KeyValueStoreBackendOptions);
+    get keyValueStoreDirectory(): string;
+    static create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;
+    getMetadata(): Promise<storage.KeyValueStoreInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    /**
+     * Remove every record from the store except the run input. Used by
+     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
+     * while preserving the run's input, matching the historical file-system storage behavior.
+     *
+     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
+     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     */
+    purgeExceptInput(): Promise<void>;
+    listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
+    /**
+     * Generates a public `file://` URL for accessing a specific record in the key-value store.
+     *
+     * Returns `undefined` if the record does not exist.
+     * @param key The key of the record to generate the public URL for.
+     */
+    getPublicUrl(key: string): Promise<string | undefined>;
+    /**
+     * Tests whether a record with the given key exists without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` otherwise.
+     */
+    recordExists(key: string): Promise<boolean>;
+    getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
+    setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
+    deleteValue(key: string): Promise<void>;
+    /**
+     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
+     * key is checked against its tracked record; the run-input keys additionally fall back to
+     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
+     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
+     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
+     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
+     */
+    private resolveExistingKey;
+    /**
+     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
+     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
+     *
+     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
+     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
+     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
+     *   the tracked record first, then the bare file at that exact name (a single empty-extension
+     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
+     */
+    private bareFallbacksFor;
+}
+//# sourceMappingURL=key-value-store.d.ts.map

package/resource-clients/key-value-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store.d.ts","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EACR,6BAA6B,IAAI,oCAAoC,EAExE,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAoDvD,MAAM,WAAW,2BAA2B;IACxC,2FAA2F;IAC3F,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,oCAAoC,CAAC;IACpD,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;GAOG;AACH,qBAAa,oBAAqB,SAAQ,cAAe,YAAW,OAAO,CAAC,oBAAoB;IAC5F,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuC;gBAEzD,OAAO,EAAE,2BAA2B;IAOhD,IAAI,sBAAsB,IAAI,MAAM,CAEnC;WAEY,MAAM,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAMlF,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAIjD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjC,QAAQ,CAAC,OAAO,GAAE,OAAO,CAAC,4BAAiC,GAAG,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC;IAuChH;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAa5D;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAK3C,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAmBvE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC;IA4CjE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK7C;;;;;;;OAOG;YACW,kBAAkB;IAahC;;;;;;;;;OASG;IAEH,OAAO,CAAC,gBAAgB;CAU3B"}

package/resource-clients/key-value-store.js

@@ -0,0 +1,240 @@
+import { Readable } from 'node:stream';
+import { s } from '@sapphire/shapeshift';
+import { isStream } from '../utils.js';
+import { CachedIdClient } from './cached-id-client.js';
+/**
+ * Out-of-band ("bare") value-file fallbacks tried when the {@link ALLOWED_BARE_FILES} lookup misses the tracked
+ * record, so a lookup for `INPUT` also matches a hand-placed `INPUT.json`/`.txt`/`.bin`. Passed to the
+ * native `resolveValue`/`resolveExistingKey`, which do the probing and re-keying.
+ *
+ * Each entry declares the content type to report on a match — the native client does no MIME
+ * inference. An empty `contentType` is its sentinel for "keep the synthesized
+ * `application/octet-stream`", used for the extensionless key and `.bin`.
+ */
+const BARE_FILE_FALLBACKS = [
+    { extension: '', contentType: '' },
+    { extension: '.json', contentType: 'application/json; charset=utf-8' },
+    { extension: '.txt', contentType: 'text/plain; charset=utf-8' },
+    { extension: '.bin', contentType: '' },
+];
+const ALLOWED_BARE_FILES = ['INPUT'];
+/**
+ * The out-of-band ("bare") files to surface from the native `listKeys`, derived from
+ * {@link ALLOWED_BARE_FILES} × {@link BARE_FILE_FALLBACKS}. Each native {@link ListBareFallback}
+ * `name` is the literal on-disk filename to probe (e.g. `INPUT.json`), and the native lists a match
+ * under that same `name` — which is exactly the key we return, so a listed bare file round-trips
+ * through `getValue`/`recordExists` (see {@link BARE_FILE_CONTENT_TYPES}).
+ */
+const LIST_BARE_FALLBACKS = ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })));
+/**
+ * Lookup from a bare file's literal on-disk name (e.g. `INPUT.json`) to the content type to report
+ * for it, used to read a listed bare key back directly (`getValue('INPUT.json')`). The empty-extension
+ * entry (`INPUT`) is intentionally excluded: an extensionless `INPUT` lookup goes through the
+ * `resolveValue` fallback probing instead, which already covers the extensionless file.
+ */
+const BARE_FILE_CONTENT_TYPES = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.filter(({ extension }) => extension !== '').map(({ extension, contentType }) => [`${key}${extension}`, contentType])));
+/** Maps a bare file's on-disk name (e.g. `INPUT.json`) to its logical key (e.g. `INPUT`), for dedup. */
+const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key])));
+/**
+ * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
+ * extension.
+ *
+ * This adapter is a plain byte transport: values are written and read verbatim as `Buffer`s with a
+ * content type carried alongside them. Serializing arbitrary values into bytes and parsing them back
+ * is the {@link KeyValueStore} frontend codec's job, not this backend's.
+ */
+export class KeyValueStoreBackend extends CachedIdClient {
+    name;
+    cacheKey;
+    nativeBackend;
+    constructor(options) {
+        super();
+        this.name = options.name;
+        this.cacheKey = options.cacheKey;
+        this.nativeBackend = options.nativeBackend;
+    }
+    get keyValueStoreDirectory() {
+        return this.nativeBackend.pathToKvs;
+    }
+    static async create(options) {
+        const backend = new KeyValueStoreBackend(options);
+        backend._cachedId = (await options.nativeBackend.getMetadata()).id;
+        return backend;
+    }
+    async getMetadata() {
+        return this.nativeBackend.getMetadata();
+    }
+    async drop() {
+        await this.nativeBackend.dropStorage();
+    }
+    async purge() {
+        await this.nativeBackend.purge();
+    }
+    /**
+     * Remove every record from the store except the run input. Used by
+     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
+     * while preserving the run's input, matching the historical file-system storage behavior.
+     *
+     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
+     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     */
+    async purgeExceptInput() {
+        await this.nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
+    }
+    async listKeys(options = {}) {
+        const { prefix, exclusiveStartKey, limit } = s
+            .object({
+            prefix: s.string().optional(),
+            exclusiveStartKey: s.string().optional(),
+            limit: s.number().int().greaterThan(0).optional(),
+        })
+            .parse(options);
+        // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`)
+        // are enumerated alongside tracked records, under their actual on-disk name. The native reads
+        // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
+        // The native `listKeys` already returns a self-describing page (items + pagination cursors)
+        // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
+        const page = await this.nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
+        const presentKeys = new Set(page.items.map((record) => record.key));
+        // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
+        // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
+        // logical key itself (`INPUT`): it shadows the extension-bearing bare variants (`INPUT.json`
+        // etc.) for the same logical key, so drop those. The extensionless bare file *is* the logical
+        // key, so it is never a separate duplicate.
+        const items = page.items.filter((record) => {
+            const logicalKey = BARE_FILE_LOGICAL_KEYS.get(record.key);
+            const isExtensionBearingBareFile = logicalKey !== undefined && logicalKey !== record.key;
+            return !(isExtensionBearingBareFile && presentKeys.has(logicalKey));
+        });
+        return {
+            items,
+            count: items.length,
+            limit: page.limit,
+            exclusiveStartKey: page.exclusiveStartKey,
+            isTruncated: page.isTruncated,
+            nextExclusiveStartKey: page.nextExclusiveStartKey,
+        };
+    }
+    /**
+     * Generates a public `file://` URL for accessing a specific record in the key-value store.
+     *
+     * Returns `undefined` if the record does not exist.
+     * @param key The key of the record to generate the public URL for.
+     */
+    async getPublicUrl(key) {
+        s.string().parse(key);
+        // The native `getPublicUrl` stats the encoded path but does not probe bare-file extensions,
+        // so we resolve the on-disk key first (handling e.g. `INPUT` -> `INPUT.json`) and normalize
+        // the native `null`-for-missing result to the historical `undefined` contract.
+        const resolvedKey = await this.resolveExistingKey(key);
+        if (resolvedKey === undefined) {
+            return undefined;
+        }
+        return (await this.nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
+    }
+    /**
+     * Tests whether a record with the given key exists without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` otherwise.
+     */
+    async recordExists(key) {
+        s.string().parse(key);
+        return (await this.resolveExistingKey(key)) !== undefined;
+    }
+    async getValue(key) {
+        s.string().parse(key);
+        const fallbacks = this.bareFallbacksFor(key);
+        const record = fallbacks
+            ? await this.nativeBackend.resolveValue(key, fallbacks)
+            : await this.nativeBackend.getValue(key);
+        if (record) {
+            return {
+                key: record.key,
+                value: record.value,
+                contentType: record.contentType,
+            };
+        }
+        return undefined;
+    }
+    async setValue(record) {
+        // By the time a value reaches the backend the frontend (KeyValueStore codec) has already
+        // serialized it: non-bytes become a `string`, everything else is a `Buffer`/typed array or a
+        // stream. So we only accept those shapes here — there is no JSON inference or `String(value)`
+        // coercion left to do.
+        s.object({
+            key: s.string().lengthGreaterThan(0),
+            value: s.union([
+                s.string(),
+                s.instance(Buffer),
+                s.instance(ArrayBuffer),
+                s.typedArray(),
+                // A stream is an object; disabling validation makes shapeshift only check it is a
+                // non-null, non-array object (the stream guard below does the real check).
+                s.object({}).setValidationEnabled(false),
+            ]),
+            contentType: s.string().lengthGreaterThan(0).optional(),
+        }).parse(record);
+        const { key, value } = record;
+        // The frontend resolves the content type before it reaches the backend; this backend is a plain
+        // byte transport and does not infer content types.
+        const contentType = record.contentType ?? 'application/octet-stream';
+        // Stream the value straight to disk without buffering it all into memory. The native client
+        // consumes a Web `ReadableStream`, so convert the Node `Readable` we get from the frontend.
+        if (isStream(value)) {
+            const webStream = Readable.toWeb(value);
+            await this.nativeBackend.setValueStream(key, webStream, contentType);
+            return;
+        }
+        // Normalize the remaining (already-serialized) value into a Buffer for the native client.
+        const buffer = Buffer.isBuffer(value)
+            ? value
+            : value instanceof ArrayBuffer
+                ? Buffer.from(value)
+                : ArrayBuffer.isView(value)
+                    ? Buffer.from(value.buffer, value.byteOffset, value.byteLength)
+                    : Buffer.from(value);
+        await this.nativeBackend.setValue(key, buffer, contentType);
+    }
+    async deleteValue(key) {
+        s.string().parse(key);
+        await this.nativeBackend.deleteValue(key);
+    }
+    /**
+     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
+     * key is checked against its tracked record; the run-input keys additionally fall back to
+     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
+     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
+     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
+     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
+     */
+    async resolveExistingKey(key) {
+        const fallbacks = this.bareFallbacksFor(key);
+        if (fallbacks) {
+            return ((await this.nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
+        }
+        return (await this.nativeBackend.recordExists(key)) ? key : undefined;
+    }
+    /**
+     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
+     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
+     *
+     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
+     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
+     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
+     *   the tracked record first, then the bare file at that exact name (a single empty-extension
+     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
+     */
+    // eslint-disable-next-line class-methods-use-this
+    bareFallbacksFor(key) {
+        if (ALLOWED_BARE_FILES.includes(key)) {
+            return BARE_FILE_FALLBACKS;
+        }
+        const contentType = BARE_FILE_CONTENT_TYPES.get(key);
+        if (contentType !== undefined) {
+            return [{ extension: '', contentType }];
+        }
+        return undefined;
+    }
+}
+//# sourceMappingURL=key-value-store.js.map

package/resource-clients/key-value-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store.js","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIvC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAMzC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;;;;;GAQG;AACH,MAAM,mBAAmB,GAAiD;IACtE,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;IAClC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,iCAAiC,EAAE;IACtE,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC/D,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE;CACzC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,CAAC;AAErC;;;;;;GAMG;AACH,MAAM,mBAAmB,GAAuB,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/E,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,GAAG,SAAS,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CACzG,CAAC;AAEF;;;;;GAKG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CACnC,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/B,mBAAmB,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC,GAAG,CAC/D,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,SAAS,EAAE,EAAE,WAAW,CAAU,CAC/E,CACJ,CACJ,CAAC;AAEF,wGAAwG;AACxG,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAClC,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/B,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,SAAS,EAAE,EAAE,GAAG,CAAU,CAAC,CACnF,CACJ,CAAC;AAcF;;;;;;;GAOG;AACH,MAAM,OAAO,oBAAqB,SAAQ,cAAc;IAC3C,IAAI,CAAU;IACd,QAAQ,CAAS;IAET,aAAa,CAAuC;IAErE,YAAY,OAAoC;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,CAAC;IAED,IAAI,sBAAsB;QACtB,OAAO,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAoC;QACpD,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAClD,OAAO,CAAC,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,WAAW;QACb,OAAO,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,QAAQ,SAAS,EAAE,CAAC,CAAC,CAAC;IACxG,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAgD,EAAE;QAC7D,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,CAAC;aACzC,MAAM,CAAC;YACJ,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC7B,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEpB,4FAA4F;QAC5F,8FAA8F;QAC9F,0FAA0F;QAC1F,4FAA4F;QAC5F,0FAA0F;QAC1F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,iBAAiB,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAC;QAEtG,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAEpE,8FAA8F;QAC9F,0FAA0F;QAC1F,6FAA6F;QAC7F,8FAA8F;QAC9F,4CAA4C;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;YACvC,MAAM,UAAU,GAAG,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1D,MAAM,0BAA0B,GAAG,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,MAAM,CAAC,GAAG,CAAC;YACzF,OAAO,CAAC,CAAC,0BAA0B,IAAI,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,OAAO;YACH,KAAK;YACL,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;SACpD,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtB,4FAA4F;QAC5F,4FAA4F;QAC5F,+EAA+E;QAC/E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACvD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;IAC7E,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACtB,OAAO,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,GAAW;QACtB,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,SAAS;YACpB,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC;YACvD,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAE7C,IAAI,MAAM,EAAE,CAAC;YACT,OAAO;gBACH,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,WAAW,EAAE,MAAM,CAAC,WAAW;aAClC,CAAC;QACN,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAwC;QACnD,yFAAyF;QACzF,6FAA6F;QAC7F,8FAA8F;QAC9F,uBAAuB;QACvB,CAAC,CAAC,MAAM,CAAC;YACL,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACpC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;gBACX,CAAC,CAAC,MAAM,EAAE;gBACV,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAClB,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACvB,CAAC,CAAC,UAAU,EAAE;gBACd,kFAAkF;gBAClF,2EAA2E;gBAC3E,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC;aAC3C,CAAC;YACF,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;SAC1D,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEjB,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC9B,gGAAgG;QAChG,mDAAmD;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,0BAA0B,CAAC;QAErE,4FAA4F;QAC5F,4FAA4F;QAC5F,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAiB,CAA+B,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;YACrE,OAAO;QACX,CAAC;QAED,0FAA0F;QAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACjC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,KAAK,YAAY,WAAW;gBAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;gBACpB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC;oBACzB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC;oBAC/D,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAe,CAAC,CAAC;QAEvC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW;QACzB,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACtB,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,kBAAkB,CAAC,GAAW;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,SAAS,EAAE,CAAC;YACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,kBAAkB,CACxC,GAAG,EACH,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAC9C,CAAC,IAAI,SAAS,CAClB,CAAC;QACN,CAAC;QACD,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,CAAC;IAED;;;;;;;;;OASG;IACH,kDAAkD;IAC1C,gBAAgB,CAAC,GAAW;QAChC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,mBAAmB,CAAC;QAC/B,CAAC;QACD,MAAM,WAAW,GAAG,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}

package/resource-clients/request-queue.d.ts

@@ -0,0 +1,52 @@
+import type * as storage from '@crawlee/types';
+import type { CrawleeLogger } from '@crawlee/types';
+import type { FileSystemRequestQueueClient as NativeFileSystemRequestQueueBackend } from '@crawlee/fs-storage-native';
+import { CachedIdClient } from './cached-id-client.js';
+export interface RequestQueueBackendOptions {
+    /** The user-facing storage name, or `undefined` for unnamed (alias / default) storages. */
+    name?: string;
+    /**
+     * The key used for cache lookup in {@link FileSystemStorageBackend}. For named storages this equals
+     * the name; for alias (unnamed) storages it is the alias string. Falls back to the storage id.
+     */
+    cacheKey: string;
+    nativeBackend: NativeFileSystemRequestQueueBackend;
+    logger?: CrawleeLogger;
+}
+/**
+ * A file-system request queue backend backed by the native `@crawlee/fs-storage-native` Rust
+ * extension.
+ *
+ * Request ordering, in-progress locking and state persistence are all owned by the native client.
+ * This adapter forwards each operation and converts result shapes to the `@crawlee/types` interfaces.
+ */
+export declare class RequestQueueBackend extends CachedIdClient implements storage.RequestQueueBackend {
+    readonly name?: string;
+    readonly cacheKey: string;
+    private readonly nativeBackend;
+    constructor(options: RequestQueueBackendOptions);
+    get requestQueueDirectory(): string;
+    static create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend>;
+    /**
+     * Tells the native client how long (in seconds) a fetched request stays locked before it becomes
+     * available again.
+     */
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    getMetadata(): Promise<storage.RequestQueueInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
+    fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
+    reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    /**
+     * Persist the native client's in-memory state to disk. Called by
+     * {@link FileSystemStorageBackend.teardown} so that fetched-but-unhandled requests are not stuck
+     * for the next consumer of the same on-disk queue.
+     */
+    persistState(): Promise<void>;
+}
+//# sourceMappingURL=request-queue.d.ts.map

package/resource-clients/request-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue.d.ts","sourceRoot":"","sources":["../../src/resource-clients/request-queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,4BAA4B,IAAI,mCAAmC,EAAE,MAAM,4BAA4B,CAAC;AAEtH,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAoCvD,MAAM,WAAW,0BAA0B;IACvC,2FAA2F;IAC3F,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,mCAAmC,CAAC;IACnD,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,qBAAa,mBAAoB,SAAQ,cAAe,YAAW,OAAO,CAAC,mBAAmB;IAC1F,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAsC;gBAExD,OAAO,EAAE,0BAA0B;IAO/C,IAAI,qBAAqB,IAAI,MAAM,CAElC;WAEY,MAAM,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAMtF;;;OAGG;IACG,oCAAoC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIjE,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAIhD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,kBAAkB,CACpB,QAAQ,EAAE,OAAO,CAAC,aAAa,EAAE,EACjC,OAAO,GAAE,OAAO,CAAC,4BAAiC,GACnD,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC;IAkBpC,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAQ/E,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAIpE,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAK3G,cAAc,CAChB,OAAO,EAAE,OAAO,CAAC,mBAAmB,EACpC,OAAO,GAAE,OAAO,CAAC,4BAAiC,GACnD,OAAO,CAAC,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAQ5C,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAI3B,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAIpC;;;;OAIG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;CAGtC"}

package/resource-clients/request-queue.js

@@ -0,0 +1,120 @@
+import { s } from '@sapphire/shapeshift';
+import { CachedIdClient } from './cached-id-client.js';
+/**
+ * Convert a request (either a Crawlee `Request` instance or a plain schema object) into a plain object
+ * whose properties are all enumerable.
+ *
+ * Crawlee's `Request` stores internal metadata (crawl depth, enqueue strategy, session id, ...) in a
+ * *non-enumerable* `userData.__crawlee` bag. The native `@crawlee/fs-storage-native` client reads
+ * request properties directly over the N-API boundary, which only exposes enumerable own properties
+ * and does not honor `toJSON`. Passing a `Request` straight through would therefore silently drop the
+ * `__crawlee` metadata, resetting `crawlDepth` to 0 on the next `fetchNextRequest` (breaking e.g.
+ * `maxCrawlDepth` and enqueue-strategy handling). Round-tripping through JSON invokes the request's
+ * `toJSON`, flattening everything into enumerable properties the native client can persist.
+ */
+function plainifyRequest(request) {
+    return JSON.parse(JSON.stringify(request));
+}
+const requestShape = s
+    .object({
+    id: s.string(),
+    url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
+    uniqueKey: s.string(),
+    method: s.string().optional(),
+    retryCount: s.number().int().optional(),
+    handledAt: s.union([s.string(), s.date().valid()]).optional(),
+})
+    .passthrough();
+const requestShapeWithoutId = requestShape.omit(['id']);
+const batchRequestShapeWithoutId = requestShapeWithoutId.array();
+const requestOptionsShape = s.object({
+    forefront: s.boolean().optional(),
+});
+/**
+ * A file-system request queue backend backed by the native `@crawlee/fs-storage-native` Rust
+ * extension.
+ *
+ * Request ordering, in-progress locking and state persistence are all owned by the native client.
+ * This adapter forwards each operation and converts result shapes to the `@crawlee/types` interfaces.
+ */
+export class RequestQueueBackend extends CachedIdClient {
+    name;
+    cacheKey;
+    nativeBackend;
+    constructor(options) {
+        super();
+        this.name = options.name;
+        this.cacheKey = options.cacheKey;
+        this.nativeBackend = options.nativeBackend;
+    }
+    get requestQueueDirectory() {
+        return this.nativeBackend.pathToRq;
+    }
+    static async create(options) {
+        const backend = new RequestQueueBackend(options);
+        backend._cachedId = (await options.nativeBackend.getMetadata()).id;
+        return backend;
+    }
+    /**
+     * Tells the native client how long (in seconds) a fetched request stays locked before it becomes
+     * available again.
+     */
+    async setExpectedRequestProcessingTimeSecs(secs) {
+        await this.nativeBackend.setExpectedRequestProcessingTime(secs);
+    }
+    async getMetadata() {
+        return this.nativeBackend.getMetadata();
+    }
+    async drop() {
+        await this.nativeBackend.dropStorage();
+    }
+    async purge() {
+        await this.nativeBackend.purge();
+    }
+    async addBatchOfRequests(requests, options = {}) {
+        batchRequestShapeWithoutId.parse(requests);
+        requestOptionsShape.parse(options);
+        const response = await this.nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
+        // `processedRequests` is structurally identical between the native and `storage` types, so it
+        // passes through unchanged. `unprocessedRequests` only differs in that the native `method` is
+        // a plain `string`, hence the cast to the narrower `AllowedHttpMethods` union.
+        return {
+            processedRequests: response.processedRequests,
+            unprocessedRequests: response.unprocessedRequests,
+        };
+    }
+    async getRequest(uniqueKey) {
+        s.string().parse(uniqueKey);
+        // The native client tags requests with an internal `orderNo`; it's harmless to leak, so we
+        // hand the request back as-is rather than copying it just to drop one undeclared property.
+        // The native client already returns `undefined` for a missing request, matching this contract.
+        return (await this.nativeBackend.getRequest(uniqueKey));
+    }
+    async fetchNextRequest() {
+        return (await this.nativeBackend.fetchNextRequest());
+    }
+    async markRequestAsHandled(request) {
+        requestShape.parse(request);
+        return (await this.nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
+    }
+    async reclaimRequest(request, options = {}) {
+        requestShape.parse(request);
+        requestOptionsShape.parse(options);
+        return ((await this.nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ?? undefined);
+    }
+    async isEmpty() {
+        return this.nativeBackend.isEmpty();
+    }
+    async isFinished() {
+        return this.nativeBackend.isFinished();
+    }
+    /**
+     * Persist the native client's in-memory state to disk. Called by
+     * {@link FileSystemStorageBackend.teardown} so that fetched-but-unhandled requests are not stuck
+     * for the next consumer of the same on-disk queue.
+     */
+    async persistState() {
+        await this.nativeBackend.persistState();
+    }
+}
+//# sourceMappingURL=request-queue.js.map

package/resource-clients/request-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue.js","sourceRoot":"","sources":["../../src/resource-clients/request-queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAIzC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;;;;;;;;;;GAWG;AACH,SAAS,eAAe,CAAC,OAAgB;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAA4B,CAAC;AAC1E,CAAC;AAED,MAAM,YAAY,GAAG,CAAC;KACjB,MAAM,CAAC;IACJ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,gBAAgB,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC9D,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CAChE,CAAC;KACD,WAAW,EAAE,CAAC;AAEnB,MAAM,qBAAqB,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACxD,MAAM,0BAA0B,GAAG,qBAAqB,CAAC,KAAK,EAAE,CAAC;AAEjE,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC;AAcH;;;;;;GAMG;AACH,MAAM,OAAO,mBAAoB,SAAQ,cAAc;IAC1C,IAAI,CAAU;IACd,QAAQ,CAAS;IAET,aAAa,CAAsC;IAEpE,YAAY,OAAmC;QAC3C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,CAAC;IAED,IAAI,qBAAqB;QACrB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAmC;QACnD,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oCAAoC,CAAC,IAAY;QACnD,MAAM,IAAI,CAAC,aAAa,CAAC,gCAAgC,CAAC,IAAI,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,WAAW;QACb,OAAO,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,kBAAkB,CACpB,QAAiC,EACjC,UAAgD,EAAE;QAElD,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,kBAAkB,CACxD,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,EACnD,OAAO,CAAC,SAAS,IAAI,KAAK,CAC7B,CAAC;QAEF,8FAA8F;QAC9F,8FAA8F;QAC9F,+EAA+E;QAC/E,OAAO;YACH,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;YAC7C,mBAAmB,EAAE,QAAQ,CAAC,mBAA4E;SAC7G,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAC9B,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC5B,2FAA2F;QAC3F,2FAA2F;QAC3F,+FAA+F;QAC/F,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,CAA4C,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,gBAAgB;QAClB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAA4C,CAAC;IACpG,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,OAAoC;QAC3D,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;IAClG,CAAC;IAED,KAAK,CAAC,cAAc,CAChB,OAAoC,EACpC,UAAgD,EAAE;QAElD,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,IAAI,SAAS,CAC/G,CAAC;IACN,CAAC;IAED,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY;QACd,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,CAAC;IAC5C,CAAC;CACJ"}

package/utils.d.ts

@@ -0,0 +1,2 @@
+export declare function isStream(value: any): boolean;
+//# sourceMappingURL=utils.d.ts.map

package/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,QAAQ,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAM5C"}

package/utils.js

@@ -0,0 +1,6 @@
+export function isStream(value) {
+    return (typeof value === 'object' &&
+        value &&
+        ['on', 'pipe'].every((key) => key in value && typeof value[key] === 'function'));
+}
+//# sourceMappingURL=utils.js.map

package/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,QAAQ,CAAC,KAAU;IAC/B,OAAO,CACH,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK;QACL,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAClF,CAAC;AACN,CAAC"}