@crawlee/fs-storage 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/file-system-storage.d.ts

@@ -1,8 +1,16 @@
 import type * as storage from '@crawlee/types';
 import type { CrawleeLogger } from '@crawlee/types';
-import { DatasetBackend } from './resource-clients/dataset.js';
-import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
-import { RequestQueueBackend } from './resource-clients/request-queue.js';
+import type { AdoptionCandidate } from '@crawlee/fs-storage-native';
+/** What the key-value store hooks of {@link FileSystemStorageBackend} learn about the store at hand. */
+export interface KeyValueStoreHookOptions {
+    /** Whether the store is the run's default key-value store. */
+    isDefaultStore: boolean;
+}
+/** A file-system key-value store backend as seen by {@link FileSystemStorageBackend.purgeKeyValueStore}. */
+export interface PurgeableKeyValueStoreBackend extends storage.KeyValueStoreBackend {
+    /** Remove every record from the store except the given keys. */
+    purgeExcept(keys: string[]): Promise<void>;
+}
 export interface FileSystemStorageOptions {
     /**
      * Path to directory where the data will be saved.
@@ -10,6 +18,8 @@ export interface FileSystemStorageOptions {
     localDataDirectory: string;
     /**
      * Optional logger for FileSystemStorageBackend warnings.
+     *
+     * @internal
      */
     logger?: CrawleeLogger;
     /**
@@ -39,15 +49,6 @@ export interface FileSystemStorageOptions {
  */
 export declare class FileSystemStorageBackend implements storage.StorageBackend {
     #private;
-    readonly localDataDirectory: string;
-    readonly datasetsDirectory: string;
-    readonly keyValueStoresDirectory: string;
-    readonly requestQueuesDirectory: string;
-    readonly logger?: CrawleeLogger;
-    readonly requestQueueAccess: 'single' | 'shared';
-    readonly keyValueStoreBackendCache: KeyValueStoreBackend[];
-    readonly datasetBackendCache: DatasetBackend[];
-    readonly requestQueueBackendCache: RequestQueueBackend[];
     constructor(options: FileSystemStorageOptions);
     /**
      * Return a cache key that includes the resolved storage directory, so that two
@@ -58,6 +59,26 @@ export declare class FileSystemStorageBackend implements storage.StorageBackend
     createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
     createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
     createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;
+    /**
+     * What the native `open` may turn into records: value files sitting in the store directory with no
+     * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     * Each becomes a record keyed by its filename, with the content type derived from the extension alone.
+     *
+     * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify
+     * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way.
+     *
+     * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
+     * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles)
+     * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the
+     * spared keys.
+     *
+     */
+    protected keyValueStoreAdoptionCandidates(_options: KeyValueStoreHookOptions): AdoptionCandidate[];
+    /**
+     * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the
+     * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input.
+     */
+    protected purgeKeyValueStore(store: PurgeableKeyValueStoreBackend, _options: KeyValueStoreHookOptions): Promise<void>;
     storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     /**
      * Cleans up the run-scoped storages before the run starts, sweeping the storage directories so that

package/file-system-storage.js

@@ -23,6 +23,13 @@ async function importNativeModule() {
 const DEFAULT_STORAGE_ALIAS = '__default__';
 /** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
 const DEFAULT_STORAGE_DIRECTORY = 'default';
+/**
+ * Content types declared for adopted files. The native client infers nothing from extensions, and
+ * neither do we beyond this: a `.json` file is JSON, anything else is bytes for the
+ * {@link KeyValueStore} frontend to make sense of.
+ */
+const ADOPTED_JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
+const ADOPTED_BINARY_CONTENT_TYPE = 'application/octet-stream';
 /**
  * A file-system storage backend backed by the native `@crawlee/fs-storage-native` Rust extension.
  *
@@ -32,23 +39,23 @@ const DEFAULT_STORAGE_DIRECTORY = 'default';
  * `teardown` can operate over them), and exposing them through the `@crawlee/types` interfaces.
  */
 export class FileSystemStorageBackend {
-    localDataDirectory;
-    datasetsDirectory;
-    keyValueStoresDirectory;
-    requestQueuesDirectory;
-    logger;
-    requestQueueAccess;
-    keyValueStoreBackendCache = [];
-    datasetBackendCache = [];
-    requestQueueBackendCache = [];
+    #localDataDirectory;
+    #datasetsDirectory;
+    #keyValueStoresDirectory;
+    #requestQueuesDirectory;
+    #logger;
+    #requestQueueAccess;
+    #keyValueStoreBackendCache = [];
+    #datasetBackendCache = [];
+    #requestQueueBackendCache = [];
     constructor(options) {
         const { logger, requestQueueAccess, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
-        this.logger = logger;
-        this.requestQueueAccess = requestQueueAccess;
-        this.localDataDirectory = localDataDirectory;
-        this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
-        this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
-        this.requestQueuesDirectory = resolve(this.localDataDirectory, 'request_queues');
+        this.#logger = logger;
+        this.#requestQueueAccess = requestQueueAccess;
+        this.#localDataDirectory = localDataDirectory;
+        this.#datasetsDirectory = resolve(this.#localDataDirectory, 'datasets');
+        this.#keyValueStoresDirectory = resolve(this.#localDataDirectory, 'key_value_stores');
+        this.#requestQueuesDirectory = resolve(this.#localDataDirectory, 'request_queues');
     }
     /**
      * Return a cache key that includes the resolved storage directory, so that two
@@ -56,7 +63,7 @@ export class FileSystemStorageBackend {
      * partitions, by including the storage directory in the cache key.
      */
     getStorageBackendCacheKey() {
-        return `FileSystemStorageBackend:${resolve(this.localDataDirectory)}`;
+        return `FileSystemStorageBackend:${resolve(this.#localDataDirectory)}`;
     }
     static #resolveStorageKey(options) {
         // No identifier at all means the default storage, which is opened under the reserved alias —
@@ -73,75 +80,108 @@ export class FileSystemStorageBackend {
     }
     async createDatasetBackend(options = {}) {
         const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
-        const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
+        const found = this.#datasetBackendCache.find((store) => store.id === cacheKey ||
             store.name?.toLowerCase() === cacheKey.toLowerCase() ||
             store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
         if (found) {
             return found;
         }
-        const nativeBackend = await (await importNativeModule()).FileSystemDatasetClient.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemDatasetClient.open(id, name, alias, this.#localDataDirectory);
         const newStore = await DatasetBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
             nativeBackend,
-            logger: this.logger,
+            logger: this.#logger,
         });
-        this.datasetBackendCache.push(newStore);
+        this.#datasetBackendCache.push(newStore);
         return newStore;
     }
     async createKeyValueStoreBackend(options = {}) {
         const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
-        const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+        const found = this.#keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
             store.name?.toLowerCase() === cacheKey.toLowerCase() ||
             store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
         if (found) {
             return found;
         }
-        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.#localDataDirectory, 
+        // useTestClock — always real wall-clock outside of native tests.
+        undefined, this.keyValueStoreAdoptionCandidates({ isDefaultStore: cacheKey === DEFAULT_STORAGE_DIRECTORY }));
         const newStore = await KeyValueStoreBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
             nativeBackend,
-            logger: this.logger,
+            logger: this.#logger,
         });
-        this.keyValueStoreBackendCache.push(newStore);
+        this.#keyValueStoreBackendCache.push(newStore);
         return newStore;
     }
     async createRequestQueueBackend(options = {}) {
         const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);
-        const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+        const found = this.#requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
             queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
             queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
         if (found) {
             return found;
         }
-        const nativeBackend = await (await importNativeModule()).FileSystemRequestQueueClient.open(id, name, alias, this.localDataDirectory, 
+        const nativeBackend = await (await importNativeModule()).FileSystemRequestQueueClient.open(id, name, alias, this.#localDataDirectory, 
         // useTestClock — always real wall-clock outside of native tests.
-        undefined, this.requestQueueAccess);
+        undefined, this.#requestQueueAccess);
         const newStore = await RequestQueueBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
             nativeBackend,
-            logger: this.logger,
+            logger: this.#logger,
         });
-        this.requestQueueBackendCache.push(newStore);
+        this.#requestQueueBackendCache.push(newStore);
         return newStore;
     }
+    /**
+     * What the native `open` may turn into records: value files sitting in the store directory with no
+     * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     * Each becomes a record keyed by its filename, with the content type derived from the extension alone.
+     *
+     * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify
+     * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way.
+     *
+     * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
+     * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles)
+     * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the
+     * spared keys.
+     *
+     */
+    keyValueStoreAdoptionCandidates(_options) {
+        return [
+            {
+                files: [
+                    { filename: '*.json', contentType: ADOPTED_JSON_CONTENT_TYPE },
+                    { filename: '*', contentType: ADOPTED_BINARY_CONTENT_TYPE },
+                ],
+            },
+        ];
+    }
+    /**
+     * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the
+     * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input.
+     */
+    async purgeKeyValueStore(store, _options) {
+        await store.purge();
+    }
     async storageExists(id, type) {
         let backends;
         let baseDir;
         switch (type) {
             case 'Dataset':
-                backends = this.datasetBackendCache;
-                baseDir = this.datasetsDirectory;
+                backends = this.#datasetBackendCache;
+                baseDir = this.#datasetsDirectory;
                 break;
             case 'KeyValueStore':
-                backends = this.keyValueStoreBackendCache;
-                baseDir = this.keyValueStoresDirectory;
+                backends = this.#keyValueStoreBackendCache;
+                baseDir = this.#keyValueStoresDirectory;
                 break;
             case 'RequestQueue':
-                backends = this.requestQueueBackendCache;
-                baseDir = this.requestQueuesDirectory;
+                backends = this.#requestQueueBackendCache;
+                baseDir = this.#requestQueuesDirectory;
                 break;
             default:
                 return false;
@@ -214,11 +254,9 @@ export class FileSystemStorageBackend {
      */
     async purge() {
         await Promise.all([
-            this.#purgeRunScopedStorages(this.keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }), 
-            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
-            async (store, isDefault) => (isDefault ? store.purgeExceptInput() : store.purge())),
-            this.#purgeRunScopedStorages(this.datasetsDirectory, async (alias) => this.createDatasetBackend({ alias }), async (store) => store.purge()),
-            this.#purgeRunScopedStorages(this.requestQueuesDirectory, async (alias) => this.createRequestQueueBackend({ alias }), async (store) => store.purge()),
+            this.#purgeRunScopedStorages(this.#keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }), async (store, isDefault) => this.purgeKeyValueStore(store, { isDefaultStore: isDefault })),
+            this.#purgeRunScopedStorages(this.#datasetsDirectory, async (alias) => this.createDatasetBackend({ alias }), async (store) => store.purge()),
+            this.#purgeRunScopedStorages(this.#requestQueuesDirectory, async (alias) => this.createRequestQueueBackend({ alias }), async (store) => store.purge()),
         ]);
     }
     /**
@@ -281,6 +319,6 @@ export class FileSystemStorageBackend {
      * are not stuck (until their lock expires) for the next consumer of the same on-disk queue.
      */
     async teardown() {
-        await Promise.all(this.requestQueueBackendCache.map(async (queue) => queue.persistState()));
+        await Promise.all(this.#requestQueueBackendCache.map(async (queue) => queue.persistState()));
     }
 }

package/index.d.ts

@@ -1 +1,2 @@
 export * from './file-system-storage.js';
+export type { AdoptionCandidate } from '@crawlee/fs-storage-native';

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -42,10 +42,10 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/fs-storage-native": "0.1.5-beta.18",
-        "@crawlee/types": "4.0.0-rc.0",
-        "@crawlee/utils": "4.0.0-rc.0",
-        "zod": "^4.4.3"
+        "@crawlee/fs-storage-native": ">=0.2.2 <0.3",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -54,5 +54,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/resource-clients/key-value-store.d.ts

@@ -32,19 +32,16 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     drop(): Promise<void>;
     purge(): Promise<void>;
     /**
-     * Remove every record from the store except the run input. Used by
-     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input, matching the historical file-system storage behavior.
-     *
-     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
-     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend}
+     * subclasses that keep some of the default store's records across the purge on start.
      */
-    purgeExceptInput(): Promise<void>;
+    purgeExcept(keys: string[]): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.
      *
-     * Returns `undefined` if the record does not exist.
+     * Existence-agnostic per the {@link KeyValueStoreBackend} contract: the URL points at the file
+     * the key is bound to if there is a record, and at the file it would be written to otherwise.
      * @param key The key of the record to generate the public URL for.
      */
     getPublicUrl(key: string): Promise<string | undefined>;
@@ -58,24 +55,4 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
     setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
     deleteValue(key: string): Promise<void>;
-    /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
-     * key is checked against its tracked record; the run-input keys additionally fall back to
-     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
-     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
-     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
-     */
-    private resolveExistingKey;
-    /**
-     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
-     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
-     *
-     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
-     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
-     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
-     *   the tracked record first, then the bare file at that exact name (a single empty-extension
-     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
-     */
-    private bareFallbacksFor;
 }

package/resource-clients/key-value-store.js

@@ -3,22 +3,6 @@ import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { isStream } from '../utils.js';
 import { CachedIdClient } from './cached-id-client.js';
-/**
- * Out-of-band ("bare") value-file fallbacks tried when the {@link ALLOWED_BARE_FILES} lookup misses the tracked
- * record, so a lookup for `INPUT` also matches a hand-placed `INPUT.json`/`.txt`/`.bin`. Passed to the
- * native `resolveValue`/`resolveExistingKey`, which do the probing and re-keying.
- *
- * Each entry declares the content type to report on a match — the native client does no MIME
- * inference. An empty `contentType` is its sentinel for "keep the synthesized
- * `application/octet-stream`", used for the extensionless key and `.bin`.
- */
-const BARE_FILE_FALLBACKS = [
-    { extension: '', contentType: '' },
-    { extension: '.json', contentType: 'application/json; charset=utf-8' },
-    { extension: '.txt', contentType: 'text/plain; charset=utf-8' },
-    { extension: '.bin', contentType: '' },
-];
-const ALLOWED_BARE_FILES = ['INPUT'];
 const keySchema = z.string();
 const inputRecordShape = z.object({
     key: z.string().min(1),
@@ -33,23 +17,6 @@ const inputRecordShape = z.object({
     ]),
     contentType: z.string().min(1).optional(),
 });
-/**
- * The out-of-band ("bare") files to surface from the native `listKeys`, derived from
- * {@link ALLOWED_BARE_FILES} × {@link BARE_FILE_FALLBACKS}. Each native {@link ListBareFallback}
- * `name` is the literal on-disk filename to probe (e.g. `INPUT.json`), and the native lists a match
- * under that same `name` — which is exactly the key we return, so a listed bare file round-trips
- * through `getValue`/`recordExists` (see {@link BARE_FILE_CONTENT_TYPES}).
- */
-const LIST_BARE_FALLBACKS = ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })));
-/**
- * Lookup from a bare file's literal on-disk name (e.g. `INPUT.json`) to the content type to report
- * for it, used to read a listed bare key back directly (`getValue('INPUT.json')`). The empty-extension
- * entry (`INPUT`) is intentionally excluded: an extensionless `INPUT` lookup goes through the
- * `resolveValue` fallback probing instead, which already covers the extensionless file.
- */
-const BARE_FILE_CONTENT_TYPES = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.filter(({ extension }) => extension !== '').map(({ extension, contentType }) => [`${key}${extension}`, contentType])));
-/** Maps a bare file's on-disk name (e.g. `INPUT.json`) to its logical key (e.g. `INPUT`), for dedup. */
-const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key])));
 /**
  * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
  * extension.
@@ -86,60 +53,28 @@ export class KeyValueStoreBackend extends CachedIdClient {
         await this.#nativeBackend.purge();
     }
     /**
-     * Remove every record from the store except the run input. Used by
-     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input, matching the historical file-system storage behavior.
-     *
-     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
-     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend}
+     * subclasses that keep some of the default store's records across the purge on start.
      */
-    async purgeExceptInput() {
-        await this.#nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
+    async purgeExcept(keys) {
+        await this.#nativeBackend.purge(keys);
     }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
-        // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`)
-        // are enumerated alongside tracked records, under their actual on-disk name. The native reads
-        // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
         // The native `listKeys` already returns a self-describing page (items + pagination cursors)
-        // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
-        const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
-        const presentKeys = new Set(page.items.map((record) => record.key));
-        // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
-        // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
-        // logical key itself (`INPUT`): it shadows the extension-bearing bare variants (`INPUT.json`
-        // etc.) for the same logical key, so drop those. The extensionless bare file *is* the logical
-        // key, so it is never a separate duplicate.
-        const items = page.items.filter((record) => {
-            const logicalKey = BARE_FILE_LOGICAL_KEYS.get(record.key);
-            const isExtensionBearingBareFile = logicalKey !== undefined && logicalKey !== record.key;
-            return !(isExtensionBearingBareFile && presentKeys.has(logicalKey));
-        });
-        return {
-            items,
-            count: items.length,
-            limit: page.limit,
-            exclusiveStartKey: page.exclusiveStartKey,
-            isTruncated: page.isTruncated,
-            nextExclusiveStartKey: page.nextExclusiveStartKey,
-        };
+        // matching the `KeyValueStoreListKeysResult` contract.
+        return this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix);
     }
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.
      *
-     * Returns `undefined` if the record does not exist.
+     * Existence-agnostic per the {@link KeyValueStoreBackend} contract: the URL points at the file
+     * the key is bound to if there is a record, and at the file it would be written to otherwise.
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
         parseArgument(key, keySchema);
-        // The native `getPublicUrl` stats the encoded path but does not probe bare-file extensions,
-        // so we resolve the on-disk key first (handling e.g. `INPUT` -> `INPUT.json`) and normalize
-        // the native `null`-for-missing result to the historical `undefined` contract.
-        const resolvedKey = await this.resolveExistingKey(key);
-        if (resolvedKey === undefined) {
-            return undefined;
-        }
-        return (await this.#nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
+        return this.#nativeBackend.getPublicUrl(key);
     }
     /**
      * Tests whether a record with the given key exists without retrieving its value.
@@ -149,14 +84,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
      */
     async recordExists(key) {
         parseArgument(key, keySchema);
-        return (await this.resolveExistingKey(key)) !== undefined;
+        return this.#nativeBackend.recordExists(key);
     }
     async getValue(key) {
         parseArgument(key, keySchema);
-        const fallbacks = this.bareFallbacksFor(key);
-        const record = fallbacks
-            ? await this.#nativeBackend.resolveValue(key, fallbacks)
-            : await this.#nativeBackend.getValue(key);
+        const record = await this.#nativeBackend.getValue(key);
         if (record) {
             return {
                 key: record.key,
@@ -197,40 +129,4 @@ export class KeyValueStoreBackend extends CachedIdClient {
         parseArgument(key, keySchema);
         await this.#nativeBackend.deleteValue(key);
     }
-    /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
-     * key is checked against its tracked record; the run-input keys additionally fall back to
-     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
-     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
-     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
-     */
-    async resolveExistingKey(key) {
-        const fallbacks = this.bareFallbacksFor(key);
-        if (fallbacks) {
-            return ((await this.#nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
-        }
-        return (await this.#nativeBackend.recordExists(key)) ? key : undefined;
-    }
-    /**
-     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
-     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
-     *
-     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
-     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
-     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
-     *   the tracked record first, then the bare file at that exact name (a single empty-extension
-     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
-     */
-    // eslint-disable-next-line class-methods-use-this
-    bareFallbacksFor(key) {
-        if (ALLOWED_BARE_FILES.includes(key)) {
-            return BARE_FILE_FALLBACKS;
-        }
-        const contentType = BARE_FILE_CONTENT_TYPES.get(key);
-        if (contentType !== undefined) {
-            return [{ extension: '', contentType }];
-        }
-        return undefined;
-    }
 }

package/resource-clients/request-queue.d.ts

@@ -32,6 +32,11 @@ export declare class RequestQueueBackend extends CachedIdClient implements stora
      * available again.
      */
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * @inheritdoc
+     * The lock is owned and extended by the native client; `false` means it no longer holds the request.
+     */
+    extendRequestProcessingTimeSecs(requestId: string, secs: number): Promise<boolean>;
     getMetadata(): Promise<storage.RequestQueueInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;

package/resource-clients/request-queue.js

@@ -49,6 +49,13 @@ export class RequestQueueBackend extends CachedIdClient {
     async setExpectedRequestProcessingTimeSecs(secs) {
         await this.#nativeBackend.setExpectedRequestProcessingTime(secs);
     }
+    /**
+     * @inheritdoc
+     * The lock is owned and extended by the native client; `false` means it no longer holds the request.
+     */
+    async extendRequestProcessingTimeSecs(requestId, secs) {
+        return this.#nativeBackend.prolongRequestLock(requestId, secs);
+    }
     async getMetadata() {
         return this.#nativeBackend.getMetadata();
     }