@crawlee/fs-storage 4.0.0-beta.98 → 4.0.0-rc.0

package/file-system-storage.d.ts

@@ -38,6 +38,7 @@ export interface FileSystemStorageOptions {
  * `teardown` can operate over them), and exposing them through the `@crawlee/types` interfaces.
  */
 export declare class FileSystemStorageBackend implements storage.StorageBackend {
+    #private;
     readonly localDataDirectory: string;
     readonly datasetsDirectory: string;
     readonly keyValueStoresDirectory: string;
@@ -54,27 +55,13 @@ export declare class FileSystemStorageBackend implements storage.StorageBackend
      * partitions, by including the storage directory in the cache key.
      */
     getStorageBackendCacheKey(): string;
-    private static resolveStorageKey;
     createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
     createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
     createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;
     storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     /**
-     * Resolve the real `id` of the on-disk storage identified by `entryNameOrId` under `baseDirectory`,
-     * or `undefined` if none matches. The storage's real id lives in its directory's
-     * `__metadata__.json`; the directory itself is named after the storage's `name ?? id`. So this
-     * first tries the directory named exactly `entryNameOrId` (reading its metadata id), then falls
-     * back to scanning sibling directories for one whose metadata id equals `entryNameOrId` (the case
-     * of a storage opened by name and later looked up by its auto-assigned id).
-     */
-    private static resolveStorageIdOnDisk;
-    /** Read the `id` field from a storage directory's `__metadata__.json`, or `undefined` if absent. */
-    private static readMetadataId;
-    /**
-     * Cleans up the default storages before the run starts:
-     *  - the default dataset;
-     *  - all records from the default key-value store, except for the "INPUT" key;
-     *  - the default request queue.
+     * Cleans up the run-scoped storages before the run starts, sweeping the storage directories so that
+     * leftovers from a previous process are caught too.
      */
     purge(): Promise<void>;
     /**

package/file-system-storage.js

@@ -1,10 +1,28 @@
 import { opendir, readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
-import { s } from '@sapphire/shapeshift';
-import { FileSystemDatasetClient as NativeDatasetBackend, FileSystemKeyValueStoreClient as NativeKeyValueStoreBackend, FileSystemRequestQueueClient as NativeRequestQueueBackend, } from '@crawlee/fs-storage-native';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { DatasetBackend } from './resource-clients/dataset.js';
 import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
+const fileSystemStorageOptionsSchema = z.object({
+    localDataDirectory: z.string(),
+    requestQueueAccess: z.enum(['single', 'shared']).default('single'),
+    logger: schemas.logger.optional(),
+});
+// The native package throws at load time on platforms without a published binary (e.g.
+// linux musl), and `@crawlee/core` imports this module eagerly via its service locator.
+// Load it lazily so merely importing `@crawlee/fs-storage` stays safe everywhere and the
+// native binding is only loaded when a file-system storage is actually used.
+let nativeModule;
+async function importNativeModule() {
+    nativeModule ??= import('@crawlee/fs-storage-native');
+    return nativeModule;
+}
+/** The alias `@crawlee/core` opens the default (unnamed) storage under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
+const DEFAULT_STORAGE_DIRECTORY = 'default';
 /**
  * A file-system storage backend backed by the native `@crawlee/fs-storage-native` Rust extension.
  *
@@ -24,13 +42,10 @@ export class FileSystemStorageBackend {
     datasetBackendCache = [];
     requestQueueBackendCache = [];
     constructor(options) {
-        s.object({
-            localDataDirectory: s.string(),
-            requestQueueAccess: s.enum(['single', 'shared']).optional(),
-        }).parse(options);
-        this.logger = options.logger;
-        this.requestQueueAccess = options.requestQueueAccess ?? 'single';
-        this.localDataDirectory = options.localDataDirectory;
+        const { logger, requestQueueAccess, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
+        this.logger = logger;
+        this.requestQueueAccess = requestQueueAccess;
+        this.localDataDirectory = localDataDirectory;
         this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
         this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
         this.requestQueuesDirectory = resolve(this.localDataDirectory, 'request_queues');
@@ -43,27 +58,31 @@ export class FileSystemStorageBackend {
     getStorageBackendCacheKey() {
         return `FileSystemStorageBackend:${resolve(this.localDataDirectory)}`;
     }
-    static resolveStorageKey(options) {
-        const isAlias = 'alias' in options && !!options.alias;
-        const rawKey = isAlias ? options.alias : (options.name ?? options.id);
-        // Normalize the internal __default__ alias to the user-facing 'default' name.
-        const cacheKey = rawKey === '__default__' ? 'default' : rawKey;
-        return { id: options.id, name: options.name, alias: options.alias, cacheKey };
+    static #resolveStorageKey(options) {
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in @crawlee/core, so that a backend used directly
+        // lands on the very storage the frontends would have opened.
+        const requestedAlias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        // `__default__` is an internal sentinel and must not escape onto disk: the default storage lives
+        // in `default`, which is what the docs, the project templates and every pre-existing local
+        // `storage/` directory expect. Normalizing here keeps the cache key and the directory in step.
+        const alias = requestedAlias === DEFAULT_STORAGE_ALIAS ? DEFAULT_STORAGE_DIRECTORY : requestedAlias;
+        // `alias` covers the identifier-less case, so one of the three is always set.
+        const cacheKey = alias ?? options.name ?? options.id;
+        return { id: options.id, name: options.name, alias, cacheKey };
     }
     async createDatasetBackend(options = {}) {
-        const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
-                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
+        const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
-        const nativeBackend = await NativeDatasetBackend.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemDatasetClient.open(id, name, alias, this.localDataDirectory);
         const newStore = await DatasetBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
-            cacheKey: cacheKey ?? '',
+            cacheKey,
             nativeBackend,
             logger: this.logger,
         });
@@ -71,19 +90,17 @@ export class FileSystemStorageBackend {
         return newStore;
     }
     async createKeyValueStoreBackend(options = {}) {
-        const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
-                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
+        const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
-        const nativeBackend = await NativeKeyValueStoreBackend.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory);
         const newStore = await KeyValueStoreBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
-            cacheKey: cacheKey ?? '',
+            cacheKey,
             nativeBackend,
             logger: this.logger,
         });
@@ -91,21 +108,19 @@ export class FileSystemStorageBackend {
         return newStore;
     }
     async createRequestQueueBackend(options = {}) {
-        const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
-                queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
+        const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+            queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
-        const nativeBackend = await NativeRequestQueueBackend.open(id, name, alias, this.localDataDirectory, 
+        const nativeBackend = await (await importNativeModule()).FileSystemRequestQueueClient.open(id, name, alias, this.localDataDirectory, 
         // useTestClock — always real wall-clock outside of native tests.
         undefined, this.requestQueueAccess);
         const newStore = await RequestQueueBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
-            cacheKey: cacheKey ?? '',
+            cacheKey,
             nativeBackend,
             logger: this.logger,
         });
@@ -144,7 +159,7 @@ export class FileSystemStorageBackend {
         // has a matching directory. We therefore read the real id from the metadata and only report
         // existence when it equals the queried string. This matches upstream PR #3800/#3808 and
         // prevents a named storage from being re-resolved as `{ id: name }` on a subsequent run.
-        const resolvedId = await FileSystemStorageBackend.resolveStorageIdOnDisk(baseDir, id);
+        const resolvedId = await FileSystemStorageBackend.#resolveStorageIdOnDisk(baseDir, id);
         return resolvedId === id;
     }
     /**
@@ -155,10 +170,10 @@ export class FileSystemStorageBackend {
      * back to scanning sibling directories for one whose metadata id equals `entryNameOrId` (the case
      * of a storage opened by name and later looked up by its auto-assigned id).
      */
-    static async resolveStorageIdOnDisk(baseDirectory, entryNameOrId) {
+    static async #resolveStorageIdOnDisk(baseDirectory, entryNameOrId) {
         // Directory named exactly after the string: return its real (metadata) id, which may differ
         // from the string when the string is a name rather than an id.
-        const directId = await FileSystemStorageBackend.readMetadataId(resolve(baseDirectory, entryNameOrId));
+        const directId = (await FileSystemStorageBackend.#readMetadata(resolve(baseDirectory, entryNameOrId)))?.id;
         if (directId !== undefined) {
             return directId;
         }
@@ -174,49 +189,91 @@ export class FileSystemStorageBackend {
             if (!directory.isDirectory()) {
                 continue;
             }
-            const metadataId = await FileSystemStorageBackend.readMetadataId(resolve(baseDirectory, directory.name));
+            const metadataId = (await FileSystemStorageBackend.#readMetadata(resolve(baseDirectory, directory.name)))
+                ?.id;
             if (metadataId === entryNameOrId) {
                 return metadataId;
             }
         }
         return undefined;
     }
-    /** Read the `id` field from a storage directory's `__metadata__.json`, or `undefined` if absent. */
-    static async readMetadataId(storageDirectory) {
+    /** Read a storage directory's `__metadata__.json`, or `undefined` if there is none to read. */
+    static async #readMetadata(storageDirectory) {
         try {
             const fileContent = await readFile(resolve(storageDirectory, '__metadata__.json'), 'utf8');
-            return JSON.parse(fileContent).id;
+            return JSON.parse(fileContent);
         }
         catch {
-            // Directory missing, or no/unreadable metadata file — no id to report.
+            // Directory missing, or no/unreadable metadata file — nothing to report.
             return undefined;
         }
     }
     /**
-     * Cleans up the default storages before the run starts:
-     *  - the default dataset;
-     *  - all records from the default key-value store, except for the "INPUT" key;
-     *  - the default request queue.
+     * Cleans up the run-scoped storages before the run starts, sweeping the storage directories so that
+     * leftovers from a previous process are caught too.
      */
     async purge() {
-        // Resolve the default stores up front so leftover on-disk records are purged even when the
-        // store has not been opened in this process yet (e.g. a fresh run over a pre-existing
-        // directory). Opening caches the backend, so the subsequent purge operates on a real backend.
-        // The default store is opened via the internal `__default__` alias (see resolveStorageIdentifier
-        // in @crawlee/core), which resolves to the `default` cache key — match that here so we purge the
-        // very backend the default open would return rather than creating a divergent one.
-        const [defaultKeyValueStore, defaultDataset, defaultRequestQueue] = await Promise.all([
-            this.createKeyValueStoreBackend({ alias: '__default__' }),
-            this.createDatasetBackend({ alias: '__default__' }),
-            this.createRequestQueueBackend({ alias: '__default__' }),
-        ]);
         await Promise.all([
-            // Preserve the run input (INPUT) when purging the default key-value store.
-            defaultKeyValueStore.purgeExceptInput(),
-            defaultDataset.purge(),
-            defaultRequestQueue.purge(),
+            this.#purgeRunScopedStorages(this.keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }), 
+            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
+            async (store, isDefault) => (isDefault ? store.purgeExceptInput() : store.purge())),
+            this.#purgeRunScopedStorages(this.datasetsDirectory, async (alias) => this.createDatasetBackend({ alias }), async (store) => store.purge()),
+            this.#purgeRunScopedStorages(this.requestQueuesDirectory, async (alias) => this.createRequestQueueBackend({ alias }), async (store) => store.purge()),
         ]);
     }
+    /**
+     * Purge every run-scoped storage under `storagesDirectory`, whether or not it has been opened in this
+     * process yet. Storages are opened rather than emptied on disk directly, so that one already open
+     * under the same name or alias is purged through the backend the run is using, not a second one.
+     */
+    async #purgeRunScopedStorages(storagesDirectory, open, purgeStorage) {
+        // The default storage is listed unconditionally, so that a run over an empty directory still ends
+        // up with it opened (and cached) exactly as it was before. Deduplicating by cache key then keeps
+        // it to a single open: every run after the first also finds its `default` directory on disk, and
+        // opening the same storage twice concurrently would race two backends onto one directory.
+        const aliasesByCacheKey = new Map();
+        for (const alias of [
+            DEFAULT_STORAGE_ALIAS,
+            ...(await FileSystemStorageBackend.#listUnnamedStorages(storagesDirectory)),
+        ]) {
+            const { cacheKey } = FileSystemStorageBackend.#resolveStorageKey({ alias });
+            if (!aliasesByCacheKey.has(cacheKey)) {
+                aliasesByCacheKey.set(cacheKey, alias);
+            }
+        }
+        await Promise.all(Array.from(aliasesByCacheKey, async ([cacheKey, alias]) => {
+            await purgeStorage(await open(alias), cacheKey === DEFAULT_STORAGE_DIRECTORY);
+        }));
+    }
+    /**
+     * The directory names of the on-disk storages under `storagesDirectory` that Crawlee created without
+     * a name — the default storage and every alias-keyed one. Since the directory is named after the
+     * storage's `name ?? alias`, the name is read from the metadata rather than guessed.
+     *
+     * Two kinds of directory are left out, as purging them would destroy data this process never wrote:
+     * one without a readable `__metadata__.json` (not written by Crawlee — a hand-placed input directory,
+     * say), and one named after its own id, which is reachable only by `{ id }` and so is not run-scoped.
+     */
+    static async #listUnnamedStorages(storagesDirectory) {
+        let directories;
+        try {
+            directories = await opendir(storagesDirectory);
+        }
+        catch {
+            return [];
+        }
+        const unnamed = [];
+        for await (const directory of directories) {
+            if (!directory.isDirectory()) {
+                continue;
+            }
+            const metadata = await FileSystemStorageBackend.#readMetadata(resolve(storagesDirectory, directory.name));
+            if (metadata !== undefined && typeof metadata.name !== 'string' && metadata.id !== directory.name) {
+                unnamed.push(directory.name);
+            }
+        }
+        return unnamed;
+    }
     /**
      * This method should be called at the end of the process, to ensure all data is saved.
      *

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.98",
+    "version": "4.0.0-rc.0",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,9 @@
     },
     "dependencies": {
         "@crawlee/fs-storage-native": "0.1.5-beta.18",
-        "@crawlee/types": "4.0.0-beta.98",
-        "@sapphire/shapeshift": "^4.0.0"
+        "@crawlee/types": "4.0.0-rc.0",
+        "@crawlee/utils": "4.0.0-rc.0",
+        "zod": "^4.4.3"
     },
     "lerna": {
         "command": {
@@ -53,5 +54,5 @@
             }
         }
     },
-    "gitHead": "3b8cd86b13e253ab5fc71e631e12a68f7465cee5"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }

package/resource-clients/dataset.d.ts

@@ -21,10 +21,9 @@ export interface DatasetBackendOptions {
  * the `@crawlee/types` interfaces.
  */
 export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends CachedIdClient implements storage.DatasetBackend<Data> {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
-    private readonly logger?;
     constructor(options: DatasetBackendOptions);
     get datasetDirectory(): string;
     static create<Data extends Dictionary = Dictionary>(options: DatasetBackendOptions): Promise<DatasetBackend<Data>>;

package/resource-clients/dataset.js

@@ -1,4 +1,4 @@
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { CachedIdClient } from './cached-id-client.js';
 /**
  * `getData` options accepted by the high-level `Dataset` frontend but not supported by the native
@@ -19,17 +19,17 @@ const UNSUPPORTED_GET_DATA_OPTIONS = ['clean', 'fields', 'omit', 'skipHidden', '
 export class DatasetBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
-    logger;
+    #nativeBackend;
+    #logger;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
-        this.logger = options.logger;
+        this.#nativeBackend = options.nativeBackend;
+        this.#logger = options.logger;
     }
     get datasetDirectory() {
-        return this.nativeBackend.pathToDataset;
+        return this.#nativeBackend.pathToDataset;
     }
     static async create(options) {
         const backend = new DatasetBackend(options);
@@ -37,32 +37,26 @@ export class DatasetBackend extends CachedIdClient {
         return backend;
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     async pushData(items) {
-        await this.nativeBackend.pushData(items);
+        await this.#nativeBackend.pushData(items);
     }
     async getData(options = {}) {
         const passedOptions = options;
         const ignored = UNSUPPORTED_GET_DATA_OPTIONS.filter((key) => passedOptions[key] !== undefined);
         if (ignored.length > 0) {
-            this.logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
+            this.#logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
                 `and were ignored. Only "offset", "limit" and "desc" are honored.`);
         }
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
-        const page = await this.nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
+        const page = await this.#nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
         return {
             count: page.count,
             desc: page.desc,

package/resource-clients/key-value-store.d.ts

@@ -22,9 +22,9 @@ export interface KeyValueStoreBackendOptions {
  * is the {@link KeyValueStore} frontend codec's job, not this backend's.
  */
 export declare class KeyValueStoreBackend extends CachedIdClient implements storage.KeyValueStoreBackend {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
     constructor(options: KeyValueStoreBackendOptions);
     get keyValueStoreDirectory(): string;
     static create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;

package/resource-clients/key-value-store.js

@@ -1,5 +1,6 @@
 import { Readable } from 'node:stream';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { isStream } from '../utils.js';
 import { CachedIdClient } from './cached-id-client.js';
 /**
@@ -18,6 +19,20 @@ const BARE_FILE_FALLBACKS = [
     { extension: '.bin', contentType: '' },
 ];
 const ALLOWED_BARE_FILES = ['INPUT'];
+const keySchema = z.string();
+const inputRecordShape = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.string(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // A stream is an object; this only checks it is a non-null, non-array object
+        // (the stream guard in `setValue` does the real check).
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
 /**
  * The out-of-band ("bare") files to surface from the native `listKeys`, derived from
  * {@link ALLOWED_BARE_FILES} × {@link BARE_FILE_FALLBACKS}. Each native {@link ListBareFallback}
@@ -46,15 +61,15 @@ const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_
 export class KeyValueStoreBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
+    #nativeBackend;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
+        this.#nativeBackend = options.nativeBackend;
     }
     get keyValueStoreDirectory() {
-        return this.nativeBackend.pathToKvs;
+        return this.#nativeBackend.pathToKvs;
     }
     static async create(options) {
         const backend = new KeyValueStoreBackend(options);
@@ -62,13 +77,13 @@ export class KeyValueStoreBackend extends CachedIdClient {
         return backend;
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     /**
      * Remove every record from the store except the run input. Used by
@@ -79,22 +94,16 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
      */
     async purgeExceptInput() {
-        await this.nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
+        await this.#nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
     }
     async listKeys(options = {}) {
-        const { prefix, exclusiveStartKey, limit } = s
-            .object({
-            prefix: s.string().optional(),
-            exclusiveStartKey: s.string().optional(),
-            limit: s.number().int().greaterThan(0).optional(),
-        })
-            .parse(options);
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`)
         // are enumerated alongside tracked records, under their actual on-disk name. The native reads
         // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
         // The native `listKeys` already returns a self-describing page (items + pagination cursors)
         // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
-        const page = await this.nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
+        const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
         const presentKeys = new Set(page.items.map((record) => record.key));
         // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
         // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
@@ -122,7 +131,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         // The native `getPublicUrl` stats the encoded path but does not probe bare-file extensions,
         // so we resolve the on-disk key first (handling e.g. `INPUT` -> `INPUT.json`) and normalize
         // the native `null`-for-missing result to the historical `undefined` contract.
@@ -130,7 +139,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         if (resolvedKey === undefined) {
             return undefined;
         }
-        return (await this.nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
+        return (await this.#nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
     }
     /**
      * Tests whether a record with the given key exists without retrieving its value.
@@ -139,15 +148,15 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * @returns `true` if the record exists, `false` otherwise.
      */
     async recordExists(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         return (await this.resolveExistingKey(key)) !== undefined;
     }
     async getValue(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         const fallbacks = this.bareFallbacksFor(key);
         const record = fallbacks
-            ? await this.nativeBackend.resolveValue(key, fallbacks)
-            : await this.nativeBackend.getValue(key);
+            ? await this.#nativeBackend.resolveValue(key, fallbacks)
+            : await this.#nativeBackend.getValue(key);
         if (record) {
             return {
                 key: record.key,
@@ -162,19 +171,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // serialized it: non-bytes become a `string`, everything else is a `Buffer`/typed array or a
         // stream. So we only accept those shapes here — there is no JSON inference or `String(value)`
         // coercion left to do.
-        s.object({
-            key: s.string().lengthGreaterThan(0),
-            value: s.union([
-                s.string(),
-                s.instance(Buffer),
-                s.instance(ArrayBuffer),
-                s.typedArray(),
-                // A stream is an object; disabling validation makes shapeshift only check it is a
-                // non-null, non-array object (the stream guard below does the real check).
-                s.object({}).setValidationEnabled(false),
-            ]),
-            contentType: s.string().lengthGreaterThan(0).optional(),
-        }).parse(record);
+        parseArgument(record, inputRecordShape);
         const { key, value } = record;
         // The frontend resolves the content type before it reaches the backend; this backend is a plain
         // byte transport and does not infer content types.
@@ -183,7 +180,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // consumes a Web `ReadableStream`, so convert the Node `Readable` we get from the frontend.
         if (isStream(value)) {
             const webStream = Readable.toWeb(value);
-            await this.nativeBackend.setValueStream(key, webStream, contentType);
+            await this.#nativeBackend.setValueStream(key, webStream, contentType);
             return;
         }
         // Normalize the remaining (already-serialized) value into a Buffer for the native client.
@@ -194,11 +191,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
                 : ArrayBuffer.isView(value)
                     ? Buffer.from(value.buffer, value.byteOffset, value.byteLength)
                     : Buffer.from(value);
-        await this.nativeBackend.setValue(key, buffer, contentType);
+        await this.#nativeBackend.setValue(key, buffer, contentType);
     }
     async deleteValue(key) {
-        s.string().parse(key);
-        await this.nativeBackend.deleteValue(key);
+        parseArgument(key, keySchema);
+        await this.#nativeBackend.deleteValue(key);
     }
     /**
      * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
@@ -211,9 +208,9 @@ export class KeyValueStoreBackend extends CachedIdClient {
     async resolveExistingKey(key) {
         const fallbacks = this.bareFallbacksFor(key);
         if (fallbacks) {
-            return ((await this.nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
+            return ((await this.#nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
         }
-        return (await this.nativeBackend.recordExists(key)) ? key : undefined;
+        return (await this.#nativeBackend.recordExists(key)) ? key : undefined;
     }
     /**
      * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or

package/resource-clients/request-queue.d.ts

@@ -21,9 +21,9 @@ export interface RequestQueueBackendOptions {
  * This adapter forwards each operation and converts result shapes to the `@crawlee/types` interfaces.
  */
 export declare class RequestQueueBackend extends CachedIdClient implements storage.RequestQueueBackend {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
     constructor(options: RequestQueueBackendOptions);
     get requestQueueDirectory(): string;
     static create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend>;

package/resource-clients/request-queue.js

@@ -1,5 +1,7 @@
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { CachedIdClient } from './cached-id-client.js';
+const uniqueKeySchema = z.string();
 /**
  * Convert a request (either a Crawlee `Request` instance or a plain schema object) into a plain object
  * whose properties are all enumerable.
@@ -15,21 +17,6 @@ import { CachedIdClient } from './cached-id-client.js';
 function plainifyRequest(request) {
     return JSON.parse(JSON.stringify(request));
 }
-const requestShape = s
-    .object({
-    id: s.string(),
-    url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
-    uniqueKey: s.string(),
-    method: s.string().optional(),
-    retryCount: s.number().int().optional(),
-    handledAt: s.union([s.string(), s.date().valid()]).optional(),
-})
-    .passthrough();
-const requestShapeWithoutId = requestShape.omit(['id']);
-const batchRequestShapeWithoutId = requestShapeWithoutId.array();
-const requestOptionsShape = s.object({
-    forefront: s.boolean().optional(),
-});
 /**
  * A file-system request queue backend backed by the native `@crawlee/fs-storage-native` Rust
  * extension.
@@ -40,15 +27,15 @@ const requestOptionsShape = s.object({
 export class RequestQueueBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
+    #nativeBackend;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
+        this.#nativeBackend = options.nativeBackend;
     }
     get requestQueueDirectory() {
-        return this.nativeBackend.pathToRq;
+        return this.#nativeBackend.pathToRq;
     }
     static async create(options) {
         const backend = new RequestQueueBackend(options);
@@ -60,21 +47,21 @@ export class RequestQueueBackend extends CachedIdClient {
      * available again.
      */
     async setExpectedRequestProcessingTimeSecs(secs) {
-        await this.nativeBackend.setExpectedRequestProcessingTime(secs);
+        await this.#nativeBackend.setExpectedRequestProcessingTime(secs);
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     async addBatchOfRequests(requests, options = {}) {
-        batchRequestShapeWithoutId.parse(requests);
-        requestOptionsShape.parse(options);
-        const response = await this.nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
+        parseArgument(requests, schemas.storageRequestBatch);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        const response = await this.#nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
         // `processedRequests` is structurally identical between the native and `storage` types, so it
         // passes through unchanged. `unprocessedRequests` only differs in that the native `method` is
         // a plain `string`, hence the cast to the narrower `AllowedHttpMethods` union.
@@ -84,29 +71,30 @@ export class RequestQueueBackend extends CachedIdClient {
         };
     }
     async getRequest(uniqueKey) {
-        s.string().parse(uniqueKey);
+        parseArgument(uniqueKey, uniqueKeySchema);
         // The native client tags requests with an internal `orderNo`; it's harmless to leak, so we
         // hand the request back as-is rather than copying it just to drop one undeclared property.
         // The native client already returns `undefined` for a missing request, matching this contract.
-        return (await this.nativeBackend.getRequest(uniqueKey));
+        return (await this.#nativeBackend.getRequest(uniqueKey));
     }
     async fetchNextRequest() {
-        return (await this.nativeBackend.fetchNextRequest());
+        return (await this.#nativeBackend.fetchNextRequest());
     }
     async markRequestAsHandled(request) {
-        requestShape.parse(request);
-        return (await this.nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
+        parseArgument(request, schemas.storageRequest);
+        return (await this.#nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
     }
     async reclaimRequest(request, options = {}) {
-        requestShape.parse(request);
-        requestOptionsShape.parse(options);
-        return ((await this.nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ?? undefined);
+        parseArgument(request, schemas.storageRequest);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        return ((await this.#nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ??
+            undefined);
     }
     async isEmpty() {
-        return this.nativeBackend.isEmpty();
+        return this.#nativeBackend.isEmpty();
     }
     async isFinished() {
-        return this.nativeBackend.isFinished();
+        return this.#nativeBackend.isFinished();
     }
     /**
      * Persist the native client's in-memory state to disk. Called by
@@ -114,6 +102,6 @@ export class RequestQueueBackend extends CachedIdClient {
      * for the next consumer of the same on-disk queue.
      */
     async persistState() {
-        await this.nativeBackend.persistState();
+        await this.#nativeBackend.persistState();
     }
 }