@crawlee/fs-storage 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/file-system-storage.d.ts

@@ -1,8 +1,16 @@
 import type * as storage from '@crawlee/types';
 import type { CrawleeLogger } from '@crawlee/types';
-import { DatasetBackend } from './resource-clients/dataset.js';
-import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
-import { RequestQueueBackend } from './resource-clients/request-queue.js';
+import type { AdoptionCandidate } from '@crawlee/fs-storage-native';
+/** What the key-value store hooks of {@link FileSystemStorageBackend} learn about the store at hand. */
+export interface KeyValueStoreHookOptions {
+    /** Whether the store is the run's default key-value store. */
+    isDefaultStore: boolean;
+}
+/** A file-system key-value store backend as seen by {@link FileSystemStorageBackend.purgeKeyValueStore}. */
+export interface PurgeableKeyValueStoreBackend extends storage.KeyValueStoreBackend {
+    /** Remove every record from the store except the given keys. */
+    purgeExcept(keys: string[]): Promise<void>;
+}
 export interface FileSystemStorageOptions {
     /**
      * Path to directory where the data will be saved.
@@ -10,6 +18,8 @@ export interface FileSystemStorageOptions {
     localDataDirectory: string;
     /**
      * Optional logger for FileSystemStorageBackend warnings.
+     *
+     * @internal
      */
     logger?: CrawleeLogger;
     /**
@@ -38,15 +48,7 @@ export interface FileSystemStorageOptions {
  * `teardown` can operate over them), and exposing them through the `@crawlee/types` interfaces.
  */
 export declare class FileSystemStorageBackend implements storage.StorageBackend {
-    readonly localDataDirectory: string;
-    readonly datasetsDirectory: string;
-    readonly keyValueStoresDirectory: string;
-    readonly requestQueuesDirectory: string;
-    readonly logger?: CrawleeLogger;
-    readonly requestQueueAccess: 'single' | 'shared';
-    readonly keyValueStoreBackendCache: KeyValueStoreBackend[];
-    readonly datasetBackendCache: DatasetBackend[];
-    readonly requestQueueBackendCache: RequestQueueBackend[];
+    #private;
     constructor(options: FileSystemStorageOptions);
     /**
      * Return a cache key that includes the resolved storage directory, so that two
@@ -54,27 +56,33 @@ export declare class FileSystemStorageBackend implements storage.StorageBackend
      * partitions, by including the storage directory in the cache key.
      */
     getStorageBackendCacheKey(): string;
-    private static resolveStorageKey;
     createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
     createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
     createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;
-    storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     /**
-     * Resolve the real `id` of the on-disk storage identified by `entryNameOrId` under `baseDirectory`,
-     * or `undefined` if none matches. The storage's real id lives in its directory's
-     * `__metadata__.json`; the directory itself is named after the storage's `name ?? id`. So this
-     * first tries the directory named exactly `entryNameOrId` (reading its metadata id), then falls
-     * back to scanning sibling directories for one whose metadata id equals `entryNameOrId` (the case
-     * of a storage opened by name and later looked up by its auto-assigned id).
+     * What the native `open` may turn into records: value files sitting in the store directory with no
+     * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     * Each becomes a record keyed by its filename, with the content type derived from the extension alone.
+     *
+     * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify
+     * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way.
+     *
+     * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
+     * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles)
+     * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the
+     * spared keys.
+     *
+     */
+    protected keyValueStoreAdoptionCandidates(_options: KeyValueStoreHookOptions): AdoptionCandidate[];
+    /**
+     * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the
+     * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input.
      */
-    private static resolveStorageIdOnDisk;
-    /** Read the `id` field from a storage directory's `__metadata__.json`, or `undefined` if absent. */
-    private static readMetadataId;
+    protected purgeKeyValueStore(store: PurgeableKeyValueStoreBackend, _options: KeyValueStoreHookOptions): Promise<void>;
+    storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     /**
-     * Cleans up the default storages before the run starts:
-     *  - the default dataset;
-     *  - all records from the default key-value store, except for the "INPUT" key;
-     *  - the default request queue.
+     * Cleans up the run-scoped storages before the run starts, sweeping the storage directories so that
+     * leftovers from a previous process are caught too.
      */
     purge(): Promise<void>;
     /**

package/file-system-storage.js

@@ -1,10 +1,35 @@
 import { opendir, readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
-import { s } from '@sapphire/shapeshift';
-import { FileSystemDatasetClient as NativeDatasetBackend, FileSystemKeyValueStoreClient as NativeKeyValueStoreBackend, FileSystemRequestQueueClient as NativeRequestQueueBackend, } from '@crawlee/fs-storage-native';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { DatasetBackend } from './resource-clients/dataset.js';
 import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
+const fileSystemStorageOptionsSchema = z.object({
+    localDataDirectory: z.string(),
+    requestQueueAccess: z.enum(['single', 'shared']).default('single'),
+    logger: schemas.logger.optional(),
+});
+// The native package throws at load time on platforms without a published binary (e.g.
+// linux musl), and `@crawlee/core` imports this module eagerly via its service locator.
+// Load it lazily so merely importing `@crawlee/fs-storage` stays safe everywhere and the
+// native binding is only loaded when a file-system storage is actually used.
+let nativeModule;
+async function importNativeModule() {
+    nativeModule ??= import('@crawlee/fs-storage-native');
+    return nativeModule;
+}
+/** The alias `@crawlee/core` opens the default (unnamed) storage under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
+const DEFAULT_STORAGE_DIRECTORY = 'default';
+/**
+ * Content types declared for adopted files. The native client infers nothing from extensions, and
+ * neither do we beyond this: a `.json` file is JSON, anything else is bytes for the
+ * {@link KeyValueStore} frontend to make sense of.
+ */
+const ADOPTED_JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
+const ADOPTED_BINARY_CONTENT_TYPE = 'application/octet-stream';
 /**
  * A file-system storage backend backed by the native `@crawlee/fs-storage-native` Rust extension.
  *
@@ -14,26 +39,23 @@ import { RequestQueueBackend } from './resource-clients/request-queue.js';
  * `teardown` can operate over them), and exposing them through the `@crawlee/types` interfaces.
  */
 export class FileSystemStorageBackend {
-    localDataDirectory;
-    datasetsDirectory;
-    keyValueStoresDirectory;
-    requestQueuesDirectory;
-    logger;
-    requestQueueAccess;
-    keyValueStoreBackendCache = [];
-    datasetBackendCache = [];
-    requestQueueBackendCache = [];
+    #localDataDirectory;
+    #datasetsDirectory;
+    #keyValueStoresDirectory;
+    #requestQueuesDirectory;
+    #logger;
+    #requestQueueAccess;
+    #keyValueStoreBackendCache = [];
+    #datasetBackendCache = [];
+    #requestQueueBackendCache = [];
     constructor(options) {
-        s.object({
-            localDataDirectory: s.string(),
-            requestQueueAccess: s.enum(['single', 'shared']).optional(),
-        }).parse(options);
-        this.logger = options.logger;
-        this.requestQueueAccess = options.requestQueueAccess ?? 'single';
-        this.localDataDirectory = options.localDataDirectory;
-        this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
-        this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
-        this.requestQueuesDirectory = resolve(this.localDataDirectory, 'request_queues');
+        const { logger, requestQueueAccess, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
+        this.#logger = logger;
+        this.#requestQueueAccess = requestQueueAccess;
+        this.#localDataDirectory = localDataDirectory;
+        this.#datasetsDirectory = resolve(this.#localDataDirectory, 'datasets');
+        this.#keyValueStoresDirectory = resolve(this.#localDataDirectory, 'key_value_stores');
+        this.#requestQueuesDirectory = resolve(this.#localDataDirectory, 'request_queues');
     }
     /**
      * Return a cache key that includes the resolved storage directory, so that two
@@ -41,92 +63,125 @@ export class FileSystemStorageBackend {
      * partitions, by including the storage directory in the cache key.
      */
     getStorageBackendCacheKey() {
-        return `FileSystemStorageBackend:${resolve(this.localDataDirectory)}`;
+        return `FileSystemStorageBackend:${resolve(this.#localDataDirectory)}`;
     }
-    static resolveStorageKey(options) {
-        const isAlias = 'alias' in options && !!options.alias;
-        const rawKey = isAlias ? options.alias : (options.name ?? options.id);
-        // Normalize the internal __default__ alias to the user-facing 'default' name.
-        const cacheKey = rawKey === '__default__' ? 'default' : rawKey;
-        return { id: options.id, name: options.name, alias: options.alias, cacheKey };
+    static #resolveStorageKey(options) {
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in @crawlee/core, so that a backend used directly
+        // lands on the very storage the frontends would have opened.
+        const requestedAlias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        // `__default__` is an internal sentinel and must not escape onto disk: the default storage lives
+        // in `default`, which is what the docs, the project templates and every pre-existing local
+        // `storage/` directory expect. Normalizing here keeps the cache key and the directory in step.
+        const alias = requestedAlias === DEFAULT_STORAGE_ALIAS ? DEFAULT_STORAGE_DIRECTORY : requestedAlias;
+        // `alias` covers the identifier-less case, so one of the three is always set.
+        const cacheKey = alias ?? options.name ?? options.id;
+        return { id: options.id, name: options.name, alias, cacheKey };
     }
     async createDatasetBackend(options = {}) {
-        const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
-                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
+        const found = this.#datasetBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
-        const nativeBackend = await NativeDatasetBackend.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemDatasetClient.open(id, name, alias, this.#localDataDirectory);
         const newStore = await DatasetBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
-            cacheKey: cacheKey ?? '',
+            cacheKey,
             nativeBackend,
-            logger: this.logger,
+            logger: this.#logger,
         });
-        this.datasetBackendCache.push(newStore);
+        this.#datasetBackendCache.push(newStore);
         return newStore;
     }
     async createKeyValueStoreBackend(options = {}) {
-        const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
-                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
+        const found = this.#keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
-        const nativeBackend = await NativeKeyValueStoreBackend.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.#localDataDirectory, 
+        // useTestClock — always real wall-clock outside of native tests.
+        undefined, this.keyValueStoreAdoptionCandidates({ isDefaultStore: cacheKey === DEFAULT_STORAGE_DIRECTORY }));
         const newStore = await KeyValueStoreBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
-            cacheKey: cacheKey ?? '',
+            cacheKey,
             nativeBackend,
-            logger: this.logger,
+            logger: this.#logger,
         });
-        this.keyValueStoreBackendCache.push(newStore);
+        this.#keyValueStoreBackendCache.push(newStore);
         return newStore;
     }
     async createRequestQueueBackend(options = {}) {
-        const { id, name, alias, cacheKey } = FileSystemStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
-                queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
+        const found = this.#requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+            queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
-        const nativeBackend = await NativeRequestQueueBackend.open(id, name, alias, this.localDataDirectory, 
+        const nativeBackend = await (await importNativeModule()).FileSystemRequestQueueClient.open(id, name, alias, this.#localDataDirectory, 
         // useTestClock — always real wall-clock outside of native tests.
-        undefined, this.requestQueueAccess);
+        undefined, this.#requestQueueAccess);
         const newStore = await RequestQueueBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
-            cacheKey: cacheKey ?? '',
+            cacheKey,
             nativeBackend,
-            logger: this.logger,
+            logger: this.#logger,
         });
-        this.requestQueueBackendCache.push(newStore);
+        this.#requestQueueBackendCache.push(newStore);
         return newStore;
     }
+    /**
+     * What the native `open` may turn into records: value files sitting in the store directory with no
+     * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     * Each becomes a record keyed by its filename, with the content type derived from the extension alone.
+     *
+     * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify
+     * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way.
+     *
+     * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
+     * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles)
+     * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the
+     * spared keys.
+     *
+     */
+    keyValueStoreAdoptionCandidates(_options) {
+        return [
+            {
+                files: [
+                    { filename: '*.json', contentType: ADOPTED_JSON_CONTENT_TYPE },
+                    { filename: '*', contentType: ADOPTED_BINARY_CONTENT_TYPE },
+                ],
+            },
+        ];
+    }
+    /**
+     * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the
+     * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input.
+     */
+    async purgeKeyValueStore(store, _options) {
+        await store.purge();
+    }
     async storageExists(id, type) {
         let backends;
         let baseDir;
         switch (type) {
             case 'Dataset':
-                backends = this.datasetBackendCache;
-                baseDir = this.datasetsDirectory;
+                backends = this.#datasetBackendCache;
+                baseDir = this.#datasetsDirectory;
                 break;
             case 'KeyValueStore':
-                backends = this.keyValueStoreBackendCache;
-                baseDir = this.keyValueStoresDirectory;
+                backends = this.#keyValueStoreBackendCache;
+                baseDir = this.#keyValueStoresDirectory;
                 break;
             case 'RequestQueue':
-                backends = this.requestQueueBackendCache;
-                baseDir = this.requestQueuesDirectory;
+                backends = this.#requestQueueBackendCache;
+                baseDir = this.#requestQueuesDirectory;
                 break;
             default:
                 return false;
@@ -144,7 +199,7 @@ export class FileSystemStorageBackend {
         // has a matching directory. We therefore read the real id from the metadata and only report
         // existence when it equals the queried string. This matches upstream PR #3800/#3808 and
         // prevents a named storage from being re-resolved as `{ id: name }` on a subsequent run.
-        const resolvedId = await FileSystemStorageBackend.resolveStorageIdOnDisk(baseDir, id);
+        const resolvedId = await FileSystemStorageBackend.#resolveStorageIdOnDisk(baseDir, id);
         return resolvedId === id;
     }
     /**
@@ -155,10 +210,10 @@ export class FileSystemStorageBackend {
      * back to scanning sibling directories for one whose metadata id equals `entryNameOrId` (the case
      * of a storage opened by name and later looked up by its auto-assigned id).
      */
-    static async resolveStorageIdOnDisk(baseDirectory, entryNameOrId) {
+    static async #resolveStorageIdOnDisk(baseDirectory, entryNameOrId) {
         // Directory named exactly after the string: return its real (metadata) id, which may differ
         // from the string when the string is a name rather than an id.
-        const directId = await FileSystemStorageBackend.readMetadataId(resolve(baseDirectory, entryNameOrId));
+        const directId = (await FileSystemStorageBackend.#readMetadata(resolve(baseDirectory, entryNameOrId)))?.id;
         if (directId !== undefined) {
             return directId;
         }
@@ -174,49 +229,89 @@ export class FileSystemStorageBackend {
             if (!directory.isDirectory()) {
                 continue;
             }
-            const metadataId = await FileSystemStorageBackend.readMetadataId(resolve(baseDirectory, directory.name));
+            const metadataId = (await FileSystemStorageBackend.#readMetadata(resolve(baseDirectory, directory.name)))
+                ?.id;
             if (metadataId === entryNameOrId) {
                 return metadataId;
             }
         }
         return undefined;
     }
-    /** Read the `id` field from a storage directory's `__metadata__.json`, or `undefined` if absent. */
-    static async readMetadataId(storageDirectory) {
+    /** Read a storage directory's `__metadata__.json`, or `undefined` if there is none to read. */
+    static async #readMetadata(storageDirectory) {
         try {
             const fileContent = await readFile(resolve(storageDirectory, '__metadata__.json'), 'utf8');
-            return JSON.parse(fileContent).id;
+            return JSON.parse(fileContent);
         }
         catch {
-            // Directory missing, or no/unreadable metadata file — no id to report.
+            // Directory missing, or no/unreadable metadata file — nothing to report.
             return undefined;
         }
     }
     /**
-     * Cleans up the default storages before the run starts:
-     *  - the default dataset;
-     *  - all records from the default key-value store, except for the "INPUT" key;
-     *  - the default request queue.
+     * Cleans up the run-scoped storages before the run starts, sweeping the storage directories so that
+     * leftovers from a previous process are caught too.
      */
     async purge() {
-        // Resolve the default stores up front so leftover on-disk records are purged even when the
-        // store has not been opened in this process yet (e.g. a fresh run over a pre-existing
-        // directory). Opening caches the backend, so the subsequent purge operates on a real backend.
-        // The default store is opened via the internal `__default__` alias (see resolveStorageIdentifier
-        // in @crawlee/core), which resolves to the `default` cache key — match that here so we purge the
-        // very backend the default open would return rather than creating a divergent one.
-        const [defaultKeyValueStore, defaultDataset, defaultRequestQueue] = await Promise.all([
-            this.createKeyValueStoreBackend({ alias: '__default__' }),
-            this.createDatasetBackend({ alias: '__default__' }),
-            this.createRequestQueueBackend({ alias: '__default__' }),
-        ]);
         await Promise.all([
-            // Preserve the run input (INPUT) when purging the default key-value store.
-            defaultKeyValueStore.purgeExceptInput(),
-            defaultDataset.purge(),
-            defaultRequestQueue.purge(),
+            this.#purgeRunScopedStorages(this.#keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }), async (store, isDefault) => this.purgeKeyValueStore(store, { isDefaultStore: isDefault })),
+            this.#purgeRunScopedStorages(this.#datasetsDirectory, async (alias) => this.createDatasetBackend({ alias }), async (store) => store.purge()),
+            this.#purgeRunScopedStorages(this.#requestQueuesDirectory, async (alias) => this.createRequestQueueBackend({ alias }), async (store) => store.purge()),
         ]);
     }
+    /**
+     * Purge every run-scoped storage under `storagesDirectory`, whether or not it has been opened in this
+     * process yet. Storages are opened rather than emptied on disk directly, so that one already open
+     * under the same name or alias is purged through the backend the run is using, not a second one.
+     */
+    async #purgeRunScopedStorages(storagesDirectory, open, purgeStorage) {
+        // The default storage is listed unconditionally, so that a run over an empty directory still ends
+        // up with it opened (and cached) exactly as it was before. Deduplicating by cache key then keeps
+        // it to a single open: every run after the first also finds its `default` directory on disk, and
+        // opening the same storage twice concurrently would race two backends onto one directory.
+        const aliasesByCacheKey = new Map();
+        for (const alias of [
+            DEFAULT_STORAGE_ALIAS,
+            ...(await FileSystemStorageBackend.#listUnnamedStorages(storagesDirectory)),
+        ]) {
+            const { cacheKey } = FileSystemStorageBackend.#resolveStorageKey({ alias });
+            if (!aliasesByCacheKey.has(cacheKey)) {
+                aliasesByCacheKey.set(cacheKey, alias);
+            }
+        }
+        await Promise.all(Array.from(aliasesByCacheKey, async ([cacheKey, alias]) => {
+            await purgeStorage(await open(alias), cacheKey === DEFAULT_STORAGE_DIRECTORY);
+        }));
+    }
+    /**
+     * The directory names of the on-disk storages under `storagesDirectory` that Crawlee created without
+     * a name — the default storage and every alias-keyed one. Since the directory is named after the
+     * storage's `name ?? alias`, the name is read from the metadata rather than guessed.
+     *
+     * Two kinds of directory are left out, as purging them would destroy data this process never wrote:
+     * one without a readable `__metadata__.json` (not written by Crawlee — a hand-placed input directory,
+     * say), and one named after its own id, which is reachable only by `{ id }` and so is not run-scoped.
+     */
+    static async #listUnnamedStorages(storagesDirectory) {
+        let directories;
+        try {
+            directories = await opendir(storagesDirectory);
+        }
+        catch {
+            return [];
+        }
+        const unnamed = [];
+        for await (const directory of directories) {
+            if (!directory.isDirectory()) {
+                continue;
+            }
+            const metadata = await FileSystemStorageBackend.#readMetadata(resolve(storagesDirectory, directory.name));
+            if (metadata !== undefined && typeof metadata.name !== 'string' && metadata.id !== directory.name) {
+                unnamed.push(directory.name);
+            }
+        }
+        return unnamed;
+    }
     /**
      * This method should be called at the end of the process, to ensure all data is saved.
      *
@@ -224,6 +319,6 @@ export class FileSystemStorageBackend {
      * are not stuck (until their lock expires) for the next consumer of the same on-disk queue.
      */
     async teardown() {
-        await Promise.all(this.requestQueueBackendCache.map(async (queue) => queue.persistState()));
+        await Promise.all(this.#requestQueueBackendCache.map(async (queue) => queue.persistState()));
     }
 }

package/index.d.ts

@@ -1 +1,2 @@
 export * from './file-system-storage.js';
+export type { AdoptionCandidate } from '@crawlee/fs-storage-native';

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -42,9 +42,10 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/fs-storage-native": "0.1.5-beta.18",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@sapphire/shapeshift": "^4.0.0"
+        "@crawlee/fs-storage-native": ">=0.2.2 <0.3",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -53,5 +54,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/resource-clients/dataset.d.ts

@@ -21,10 +21,9 @@ export interface DatasetBackendOptions {
  * the `@crawlee/types` interfaces.
  */
 export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends CachedIdClient implements storage.DatasetBackend<Data> {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
-    private readonly logger?;
     constructor(options: DatasetBackendOptions);
     get datasetDirectory(): string;
     static create<Data extends Dictionary = Dictionary>(options: DatasetBackendOptions): Promise<DatasetBackend<Data>>;

package/resource-clients/dataset.js

@@ -1,4 +1,4 @@
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { CachedIdClient } from './cached-id-client.js';
 /**
  * `getData` options accepted by the high-level `Dataset` frontend but not supported by the native
@@ -19,17 +19,17 @@ const UNSUPPORTED_GET_DATA_OPTIONS = ['clean', 'fields', 'omit', 'skipHidden', '
 export class DatasetBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
-    logger;
+    #nativeBackend;
+    #logger;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
-        this.logger = options.logger;
+        this.#nativeBackend = options.nativeBackend;
+        this.#logger = options.logger;
     }
     get datasetDirectory() {
-        return this.nativeBackend.pathToDataset;
+        return this.#nativeBackend.pathToDataset;
     }
     static async create(options) {
         const backend = new DatasetBackend(options);
@@ -37,32 +37,26 @@ export class DatasetBackend extends CachedIdClient {
         return backend;
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     async pushData(items) {
-        await this.nativeBackend.pushData(items);
+        await this.#nativeBackend.pushData(items);
     }
     async getData(options = {}) {
         const passedOptions = options;
         const ignored = UNSUPPORTED_GET_DATA_OPTIONS.filter((key) => passedOptions[key] !== undefined);
         if (ignored.length > 0) {
-            this.logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
+            this.#logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
                 `and were ignored. Only "offset", "limit" and "desc" are honored.`);
         }
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
-        const page = await this.nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
+        const page = await this.#nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
         return {
             count: page.count,
             desc: page.desc,

package/resource-clients/key-value-store.d.ts

@@ -22,9 +22,9 @@ export interface KeyValueStoreBackendOptions {
  * is the {@link KeyValueStore} frontend codec's job, not this backend's.
  */
 export declare class KeyValueStoreBackend extends CachedIdClient implements storage.KeyValueStoreBackend {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
     constructor(options: KeyValueStoreBackendOptions);
     get keyValueStoreDirectory(): string;
     static create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;
@@ -32,19 +32,16 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     drop(): Promise<void>;
     purge(): Promise<void>;
     /**
-     * Remove every record from the store except the run input. Used by
-     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input, matching the historical file-system storage behavior.
-     *
-     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
-     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend}
+     * subclasses that keep some of the default store's records across the purge on start.
      */
-    purgeExceptInput(): Promise<void>;
+    purgeExcept(keys: string[]): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.
      *
-     * Returns `undefined` if the record does not exist.
+     * Existence-agnostic per the {@link KeyValueStoreBackend} contract: the URL points at the file
+     * the key is bound to if there is a record, and at the file it would be written to otherwise.
      * @param key The key of the record to generate the public URL for.
      */
     getPublicUrl(key: string): Promise<string | undefined>;
@@ -58,24 +55,4 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
     setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
     deleteValue(key: string): Promise<void>;
-    /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
-     * key is checked against its tracked record; the run-input keys additionally fall back to
-     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
-     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
-     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
-     */
-    private resolveExistingKey;
-    /**
-     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
-     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
-     *
-     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
-     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
-     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
-     *   the tracked record first, then the bare file at that exact name (a single empty-extension
-     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
-     */
-    private bareFallbacksFor;
 }

package/resource-clients/key-value-store.js

@@ -1,40 +1,22 @@
 import { Readable } from 'node:stream';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { isStream } from '../utils.js';
 import { CachedIdClient } from './cached-id-client.js';
-/**
- * Out-of-band ("bare") value-file fallbacks tried when the {@link ALLOWED_BARE_FILES} lookup misses the tracked
- * record, so a lookup for `INPUT` also matches a hand-placed `INPUT.json`/`.txt`/`.bin`. Passed to the
- * native `resolveValue`/`resolveExistingKey`, which do the probing and re-keying.
- *
- * Each entry declares the content type to report on a match — the native client does no MIME
- * inference. An empty `contentType` is its sentinel for "keep the synthesized
- * `application/octet-stream`", used for the extensionless key and `.bin`.
- */
-const BARE_FILE_FALLBACKS = [
-    { extension: '', contentType: '' },
-    { extension: '.json', contentType: 'application/json; charset=utf-8' },
-    { extension: '.txt', contentType: 'text/plain; charset=utf-8' },
-    { extension: '.bin', contentType: '' },
-];
-const ALLOWED_BARE_FILES = ['INPUT'];
-/**
- * The out-of-band ("bare") files to surface from the native `listKeys`, derived from
- * {@link ALLOWED_BARE_FILES} × {@link BARE_FILE_FALLBACKS}. Each native {@link ListBareFallback}
- * `name` is the literal on-disk filename to probe (e.g. `INPUT.json`), and the native lists a match
- * under that same `name` — which is exactly the key we return, so a listed bare file round-trips
- * through `getValue`/`recordExists` (see {@link BARE_FILE_CONTENT_TYPES}).
- */
-const LIST_BARE_FALLBACKS = ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })));
-/**
- * Lookup from a bare file's literal on-disk name (e.g. `INPUT.json`) to the content type to report
- * for it, used to read a listed bare key back directly (`getValue('INPUT.json')`). The empty-extension
- * entry (`INPUT`) is intentionally excluded: an extensionless `INPUT` lookup goes through the
- * `resolveValue` fallback probing instead, which already covers the extensionless file.
- */
-const BARE_FILE_CONTENT_TYPES = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.filter(({ extension }) => extension !== '').map(({ extension, contentType }) => [`${key}${extension}`, contentType])));
-/** Maps a bare file's on-disk name (e.g. `INPUT.json`) to its logical key (e.g. `INPUT`), for dedup. */
-const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key])));
+const keySchema = z.string();
+const inputRecordShape = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.string(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // A stream is an object; this only checks it is a non-null, non-array object
+        // (the stream guard in `setValue` does the real check).
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
 /**
  * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
  * extension.
@@ -46,15 +28,15 @@ const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_
 export class KeyValueStoreBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
+    #nativeBackend;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
+        this.#nativeBackend = options.nativeBackend;
     }
     get keyValueStoreDirectory() {
-        return this.nativeBackend.pathToKvs;
+        return this.#nativeBackend.pathToKvs;
     }
     static async create(options) {
         const backend = new KeyValueStoreBackend(options);
@@ -62,75 +44,37 @@ export class KeyValueStoreBackend extends CachedIdClient {
         return backend;
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     /**
-     * Remove every record from the store except the run input. Used by
-     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input, matching the historical file-system storage behavior.
-     *
-     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
-     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend}
+     * subclasses that keep some of the default store's records across the purge on start.
      */
-    async purgeExceptInput() {
-        await this.nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
+    async purgeExcept(keys) {
+        await this.#nativeBackend.purge(keys);
     }
     async listKeys(options = {}) {
-        const { prefix, exclusiveStartKey, limit } = s
-            .object({
-            prefix: s.string().optional(),
-            exclusiveStartKey: s.string().optional(),
-            limit: s.number().int().greaterThan(0).optional(),
-        })
-            .parse(options);
-        // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`)
-        // are enumerated alongside tracked records, under their actual on-disk name. The native reads
-        // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         // The native `listKeys` already returns a self-describing page (items + pagination cursors)
-        // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
-        const page = await this.nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
-        const presentKeys = new Set(page.items.map((record) => record.key));
-        // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
-        // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
-        // logical key itself (`INPUT`): it shadows the extension-bearing bare variants (`INPUT.json`
-        // etc.) for the same logical key, so drop those. The extensionless bare file *is* the logical
-        // key, so it is never a separate duplicate.
-        const items = page.items.filter((record) => {
-            const logicalKey = BARE_FILE_LOGICAL_KEYS.get(record.key);
-            const isExtensionBearingBareFile = logicalKey !== undefined && logicalKey !== record.key;
-            return !(isExtensionBearingBareFile && presentKeys.has(logicalKey));
-        });
-        return {
-            items,
-            count: items.length,
-            limit: page.limit,
-            exclusiveStartKey: page.exclusiveStartKey,
-            isTruncated: page.isTruncated,
-            nextExclusiveStartKey: page.nextExclusiveStartKey,
-        };
+        // matching the `KeyValueStoreListKeysResult` contract.
+        return this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix);
     }
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.
      *
-     * Returns `undefined` if the record does not exist.
+     * Existence-agnostic per the {@link KeyValueStoreBackend} contract: the URL points at the file
+     * the key is bound to if there is a record, and at the file it would be written to otherwise.
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
-        s.string().parse(key);
-        // The native `getPublicUrl` stats the encoded path but does not probe bare-file extensions,
-        // so we resolve the on-disk key first (handling e.g. `INPUT` -> `INPUT.json`) and normalize
-        // the native `null`-for-missing result to the historical `undefined` contract.
-        const resolvedKey = await this.resolveExistingKey(key);
-        if (resolvedKey === undefined) {
-            return undefined;
-        }
-        return (await this.nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
+        parseArgument(key, keySchema);
+        return this.#nativeBackend.getPublicUrl(key);
     }
     /**
      * Tests whether a record with the given key exists without retrieving its value.
@@ -139,15 +83,12 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * @returns `true` if the record exists, `false` otherwise.
      */
     async recordExists(key) {
-        s.string().parse(key);
-        return (await this.resolveExistingKey(key)) !== undefined;
+        parseArgument(key, keySchema);
+        return this.#nativeBackend.recordExists(key);
     }
     async getValue(key) {
-        s.string().parse(key);
-        const fallbacks = this.bareFallbacksFor(key);
-        const record = fallbacks
-            ? await this.nativeBackend.resolveValue(key, fallbacks)
-            : await this.nativeBackend.getValue(key);
+        parseArgument(key, keySchema);
+        const record = await this.#nativeBackend.getValue(key);
         if (record) {
             return {
                 key: record.key,
@@ -162,19 +103,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // serialized it: non-bytes become a `string`, everything else is a `Buffer`/typed array or a
         // stream. So we only accept those shapes here — there is no JSON inference or `String(value)`
         // coercion left to do.
-        s.object({
-            key: s.string().lengthGreaterThan(0),
-            value: s.union([
-                s.string(),
-                s.instance(Buffer),
-                s.instance(ArrayBuffer),
-                s.typedArray(),
-                // A stream is an object; disabling validation makes shapeshift only check it is a
-                // non-null, non-array object (the stream guard below does the real check).
-                s.object({}).setValidationEnabled(false),
-            ]),
-            contentType: s.string().lengthGreaterThan(0).optional(),
-        }).parse(record);
+        parseArgument(record, inputRecordShape);
         const { key, value } = record;
         // The frontend resolves the content type before it reaches the backend; this backend is a plain
         // byte transport and does not infer content types.
@@ -183,7 +112,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // consumes a Web `ReadableStream`, so convert the Node `Readable` we get from the frontend.
         if (isStream(value)) {
             const webStream = Readable.toWeb(value);
-            await this.nativeBackend.setValueStream(key, webStream, contentType);
+            await this.#nativeBackend.setValueStream(key, webStream, contentType);
             return;
         }
         // Normalize the remaining (already-serialized) value into a Buffer for the native client.
@@ -194,46 +123,10 @@ export class KeyValueStoreBackend extends CachedIdClient {
                 : ArrayBuffer.isView(value)
                     ? Buffer.from(value.buffer, value.byteOffset, value.byteLength)
                     : Buffer.from(value);
-        await this.nativeBackend.setValue(key, buffer, contentType);
+        await this.#nativeBackend.setValue(key, buffer, contentType);
     }
     async deleteValue(key) {
-        s.string().parse(key);
-        await this.nativeBackend.deleteValue(key);
-    }
-    /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
-     * key is checked against its tracked record; the run-input keys additionally fall back to
-     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
-     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
-     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
-     */
-    async resolveExistingKey(key) {
-        const fallbacks = this.bareFallbacksFor(key);
-        if (fallbacks) {
-            return ((await this.nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
-        }
-        return (await this.nativeBackend.recordExists(key)) ? key : undefined;
-    }
-    /**
-     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
-     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
-     *
-     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
-     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
-     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
-     *   the tracked record first, then the bare file at that exact name (a single empty-extension
-     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
-     */
-    // eslint-disable-next-line class-methods-use-this
-    bareFallbacksFor(key) {
-        if (ALLOWED_BARE_FILES.includes(key)) {
-            return BARE_FILE_FALLBACKS;
-        }
-        const contentType = BARE_FILE_CONTENT_TYPES.get(key);
-        if (contentType !== undefined) {
-            return [{ extension: '', contentType }];
-        }
-        return undefined;
+        parseArgument(key, keySchema);
+        await this.#nativeBackend.deleteValue(key);
     }
 }

package/resource-clients/request-queue.d.ts

@@ -21,9 +21,9 @@ export interface RequestQueueBackendOptions {
  * This adapter forwards each operation and converts result shapes to the `@crawlee/types` interfaces.
  */
 export declare class RequestQueueBackend extends CachedIdClient implements storage.RequestQueueBackend {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
     constructor(options: RequestQueueBackendOptions);
     get requestQueueDirectory(): string;
     static create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend>;
@@ -32,6 +32,11 @@ export declare class RequestQueueBackend extends CachedIdClient implements stora
      * available again.
      */
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * @inheritdoc
+     * The lock is owned and extended by the native client; `false` means it no longer holds the request.
+     */
+    extendRequestProcessingTimeSecs(requestId: string, secs: number): Promise<boolean>;
     getMetadata(): Promise<storage.RequestQueueInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;

package/resource-clients/request-queue.js

@@ -1,5 +1,7 @@
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { CachedIdClient } from './cached-id-client.js';
+const uniqueKeySchema = z.string();
 /**
  * Convert a request (either a Crawlee `Request` instance or a plain schema object) into a plain object
  * whose properties are all enumerable.
@@ -15,21 +17,6 @@ import { CachedIdClient } from './cached-id-client.js';
 function plainifyRequest(request) {
     return JSON.parse(JSON.stringify(request));
 }
-const requestShape = s
-    .object({
-    id: s.string(),
-    url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
-    uniqueKey: s.string(),
-    method: s.string().optional(),
-    retryCount: s.number().int().optional(),
-    handledAt: s.union([s.string(), s.date().valid()]).optional(),
-})
-    .passthrough();
-const requestShapeWithoutId = requestShape.omit(['id']);
-const batchRequestShapeWithoutId = requestShapeWithoutId.array();
-const requestOptionsShape = s.object({
-    forefront: s.boolean().optional(),
-});
 /**
  * A file-system request queue backend backed by the native `@crawlee/fs-storage-native` Rust
  * extension.
@@ -40,15 +27,15 @@ const requestOptionsShape = s.object({
 export class RequestQueueBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
+    #nativeBackend;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
+        this.#nativeBackend = options.nativeBackend;
     }
     get requestQueueDirectory() {
-        return this.nativeBackend.pathToRq;
+        return this.#nativeBackend.pathToRq;
     }
     static async create(options) {
         const backend = new RequestQueueBackend(options);
@@ -60,21 +47,28 @@ export class RequestQueueBackend extends CachedIdClient {
      * available again.
      */
     async setExpectedRequestProcessingTimeSecs(secs) {
-        await this.nativeBackend.setExpectedRequestProcessingTime(secs);
+        await this.#nativeBackend.setExpectedRequestProcessingTime(secs);
+    }
+    /**
+     * @inheritdoc
+     * The lock is owned and extended by the native client; `false` means it no longer holds the request.
+     */
+    async extendRequestProcessingTimeSecs(requestId, secs) {
+        return this.#nativeBackend.prolongRequestLock(requestId, secs);
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     async addBatchOfRequests(requests, options = {}) {
-        batchRequestShapeWithoutId.parse(requests);
-        requestOptionsShape.parse(options);
-        const response = await this.nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
+        parseArgument(requests, schemas.storageRequestBatch);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        const response = await this.#nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
         // `processedRequests` is structurally identical between the native and `storage` types, so it
         // passes through unchanged. `unprocessedRequests` only differs in that the native `method` is
         // a plain `string`, hence the cast to the narrower `AllowedHttpMethods` union.
@@ -84,29 +78,30 @@ export class RequestQueueBackend extends CachedIdClient {
         };
     }
     async getRequest(uniqueKey) {
-        s.string().parse(uniqueKey);
+        parseArgument(uniqueKey, uniqueKeySchema);
         // The native client tags requests with an internal `orderNo`; it's harmless to leak, so we
         // hand the request back as-is rather than copying it just to drop one undeclared property.
         // The native client already returns `undefined` for a missing request, matching this contract.
-        return (await this.nativeBackend.getRequest(uniqueKey));
+        return (await this.#nativeBackend.getRequest(uniqueKey));
     }
     async fetchNextRequest() {
-        return (await this.nativeBackend.fetchNextRequest());
+        return (await this.#nativeBackend.fetchNextRequest());
     }
     async markRequestAsHandled(request) {
-        requestShape.parse(request);
-        return (await this.nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
+        parseArgument(request, schemas.storageRequest);
+        return (await this.#nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
     }
     async reclaimRequest(request, options = {}) {
-        requestShape.parse(request);
-        requestOptionsShape.parse(options);
-        return ((await this.nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ?? undefined);
+        parseArgument(request, schemas.storageRequest);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        return ((await this.#nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ??
+            undefined);
     }
     async isEmpty() {
-        return this.nativeBackend.isEmpty();
+        return this.#nativeBackend.isEmpty();
     }
     async isFinished() {
-        return this.nativeBackend.isFinished();
+        return this.#nativeBackend.isFinished();
     }
     /**
      * Persist the native client's in-memory state to disk. Called by
@@ -114,6 +109,6 @@ export class RequestQueueBackend extends CachedIdClient {
      * for the next consumer of the same on-disk queue.
      */
     async persistState() {
-        await this.nativeBackend.persistState();
+        await this.#nativeBackend.persistState();
     }
 }