@crawlee/fs-storage 4.0.0-beta.178 → 4.0.0-beta.180

package/file-system-storage.d.ts

@@ -28,11 +28,11 @@ export interface FileSystemStorageOptions {
     /**
      * The key the run input is read from — Crawlee's `inputKey` (`CRAWLEE_INPUT_KEY`).
      *
-     * Like the conventional `INPUT`, this key may live in the default key-value store as a bare value
+     * Like the conventional `INPUT`, this key may arrive in the default key-value store as a bare value
      * file with no metadata sidecar (e.g. the Apify CLI writes the effective input to `__CLI_INPUT.json`
-     * and points the run at that key). It is therefore readable out-of-band (`<key>`, `<key>.json`) and
-     * preserved when the default store is purged, exactly like `INPUT`,
-     * which is always kept regardless of this setting.
+     * and points the run at that key). Such a file is adopted into a record under this key when the
+     * store is opened, and the key is preserved when the default store is purged — exactly like
+     * `INPUT`, which is always treated as an input key regardless of this setting.
      *
      * @default 'INPUT'
      */

package/file-system-storage.js

@@ -24,6 +24,15 @@ async function importNativeModule() {
 const DEFAULT_STORAGE_ALIAS = '__default__';
 /** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
 const DEFAULT_STORAGE_DIRECTORY = 'default';
+/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */
+const DEFAULT_INPUT_KEY = 'INPUT';
+/**
+ * Content types declared for adopted files. The native client infers nothing from extensions, and
+ * neither do we beyond this: a `.json` file is JSON, anything else is bytes for the
+ * {@link KeyValueStore} frontend to make sense of.
+ */
+const ADOPTED_JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
+const ADOPTED_BINARY_CONTENT_TYPE = 'application/octet-stream';
 /**
  * A file-system storage backend backed by the native `@crawlee/fs-storage-native` Rust extension.
  *
@@ -39,7 +48,8 @@ export class FileSystemStorageBackend {
     requestQueuesDirectory;
     logger;
     requestQueueAccess;
-    #inputKey;
+    /** `INPUT` plus the configured `inputKey`, deduplicated. */
+    #inputKeys;
     #keyValueStoreBackendCache = [];
     #datasetBackendCache = [];
     #requestQueueBackendCache = [];
@@ -47,7 +57,7 @@ export class FileSystemStorageBackend {
         const { logger, requestQueueAccess, inputKey, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
         this.logger = logger;
         this.requestQueueAccess = requestQueueAccess;
-        this.#inputKey = inputKey;
+        this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, inputKey])];
         this.localDataDirectory = localDataDirectory;
         this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
         this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
@@ -100,13 +110,15 @@ export class FileSystemStorageBackend {
         if (found) {
             return found;
         }
-        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory, 
+        // useTestClock — always real wall-clock outside of native tests.
+        undefined, this.#adoptionCandidates(cacheKey === DEFAULT_STORAGE_DIRECTORY));
         const newStore = await KeyValueStoreBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
             nativeBackend,
             logger: this.logger,
-            inputKey: this.#inputKey,
+            inputKeys: this.#inputKeys,
         });
         this.#keyValueStoreBackendCache.push(newStore);
         return newStore;
@@ -131,6 +143,38 @@ export class FileSystemStorageBackend {
         this.#requestQueueBackendCache.push(newStore);
         return newStore;
     }
+    /**
+     * What the native `open` may turn into records: value files sitting in the store directory with no
+     * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     *
+     * Each run-input key claims a bare `<key>` or `<key>.json` first — that is the layout the Apify CLI
+     * and the templates produce, and the key must end up being `INPUT` rather than `INPUT.json`. Only
+     * the default store holds the run input, so elsewhere an `INPUT.json` is just a file named
+     * `INPUT.json`, adopted by the trailing sweep like every other one.
+     *
+     * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
+     * store — purged on start unless its key is a run-input key.
+     */
+    #adoptionCandidates(isDefaultStore) {
+        const inputCandidates = isDefaultStore
+            ? this.#inputKeys.map((key) => ({
+                key,
+                files: [
+                    { filename: key, contentType: ADOPTED_BINARY_CONTENT_TYPE },
+                    { filename: `${key}.json`, contentType: ADOPTED_JSON_CONTENT_TYPE },
+                ],
+            }))
+            : [];
+        return [
+            ...inputCandidates,
+            {
+                files: [
+                    { filename: '*.json', contentType: ADOPTED_JSON_CONTENT_TYPE },
+                    { filename: '*', contentType: ADOPTED_BINARY_CONTENT_TYPE },
+                ],
+            },
+        ];
+    }
     async storageExists(id, type) {
         let backends;
         let baseDir;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.178",
+    "version": "4.0.0-beta.180",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -42,9 +42,9 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/fs-storage-native": ">=0.2.1-beta.0 <0.3",
-        "@crawlee/types": "4.0.0-beta.178",
-        "@crawlee/utils": "4.0.0-beta.178",
+        "@crawlee/fs-storage-native": ">=0.2.1 <0.3",
+        "@crawlee/types": "4.0.0-beta.180",
+        "@crawlee/utils": "4.0.0-beta.180",
         "zod": "^4.5.4"
     },
     "lerna": {
@@ -54,5 +54,5 @@
             }
         }
     },
-    "gitHead": "7beea593ac56d23b678466bc28390d527dbc3d8b"
+    "gitHead": "4adb7b0a96ee16824ee53d18f437961d9287f977"
 }

package/resource-clients/key-value-store.d.ts

@@ -12,8 +12,11 @@ export interface KeyValueStoreBackendOptions {
     cacheKey: string;
     nativeBackend: NativeFileSystemKeyValueStoreBackend;
     logger?: CrawleeLogger;
-    /** The configured run-input key, see `FileSystemStorageOptions.inputKey`. Treated like `INPUT`. */
-    inputKey?: string;
+    /**
+     * The run-input keys of this store, deduplicated — `INPUT` plus `FileSystemStorageOptions.inputKey`.
+     * All that is left of the run-input special case: the keys {@link purgeExceptInput} spares.
+     */
+    inputKeys: string[];
 }
 /**
  * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
@@ -36,18 +39,15 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     /**
      * Remove every record from the store except the run input. Used by
      * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input. The native keep-list matches exact filenames, so every extension
-     * variant of every input key is listed.
+     * while preserving the run's input.
      */
     purgeExceptInput(): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.
      *
-     * The native `getPublicUrl` derives the URL from the key without probing bare-file extensions, so
-     * an `INPUT` that lives on disk as a hand-placed `INPUT.json` is resolved first. Nothing on disk
-     * means nothing to resolve — the requested key is used as-is, and the URL is the one the record will
-     * have once written.
+     * Existence-agnostic per the {@link KeyValueStoreBackend} contract: the URL points at the file
+     * the key is bound to if there is a record, and at the file it would be written to otherwise.
      * @param key The key of the record to generate the public URL for.
      */
     getPublicUrl(key: string): Promise<string | undefined>;
@@ -61,10 +61,4 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
     setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
     deleteValue(key: string): Promise<void>;
-    /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Run-input
-     * keys fall back to bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists.
-     */
-    private resolveExistingKey;
 }

package/resource-clients/key-value-store.js

@@ -3,21 +3,6 @@ import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { isStream } from '../utils.js';
 import { CachedIdClient } from './cached-id-client.js';
-/**
- * Out-of-band ("bare") value-file fallbacks tried when a run-input lookup misses the tracked record, so a
- * lookup for `INPUT` also matches a hand-placed `INPUT.json`. Passed to the native
- * `resolveValue`/`resolveExistingKey`, which do the probing and re-keying.
- *
- * Each entry declares the content type to report on a match — the native client does no MIME
- * inference. An empty `contentType` is its sentinel for "keep the synthesized
- * `application/octet-stream`", used for the extensionless key.
- */
-const BARE_FILE_FALLBACKS = [
-    { extension: '', contentType: '' },
-    { extension: '.json', contentType: 'application/json; charset=utf-8' },
-];
-/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */
-const DEFAULT_INPUT_KEY = 'INPUT';
 const keySchema = z.string();
 const inputRecordShape = z.object({
     key: z.string().min(1),
@@ -44,20 +29,14 @@ export class KeyValueStoreBackend extends CachedIdClient {
     name;
     cacheKey;
     #nativeBackend;
-    /** `INPUT` plus the configured `inputKey`, deduplicated. */
+    /** See {@link KeyValueStoreBackendOptions.inputKeys}. */
     #inputKeys;
-    /** Bare files the native `listKeys` should surface, under their on-disk name (e.g. `INPUT.json`). */
-    #listBareFallbacks;
-    /** Bare-file on-disk name (`INPUT.json`) to logical key (`INPUT`). */
-    #bareFileLogicalKeys;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
         this.#nativeBackend = options.nativeBackend;
-        this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, options.inputKey ?? DEFAULT_INPUT_KEY])];
-        this.#listBareFallbacks = this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })));
-        this.#bareFileLogicalKeys = new Map(this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key])));
+        this.#inputKeys = options.inputKeys;
     }
     get keyValueStoreDirectory() {
         return this.#nativeBackend.pathToKvs;
@@ -79,54 +58,27 @@ export class KeyValueStoreBackend extends CachedIdClient {
     /**
      * Remove every record from the store except the run input. Used by
      * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input. The native keep-list matches exact filenames, so every extension
-     * variant of every input key is listed.
+     * while preserving the run's input.
      */
     async purgeExceptInput() {
-        const keep = this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => `${key}${extension}`));
-        await this.#nativeBackend.purge(keep);
+        await this.#nativeBackend.purge(this.#inputKeys);
     }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
-        // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`)
-        // are enumerated alongside tracked records, under their actual on-disk name. The native reads
-        // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
         // The native `listKeys` already returns a self-describing page (items + pagination cursors)
-        // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
-        const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, this.#listBareFallbacks);
-        const presentKeys = new Set(page.items.map((record) => record.key));
-        // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
-        // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
-        // logical key itself (`INPUT`): it shadows the extension-bearing bare variants (`INPUT.json`
-        // etc.) for the same logical key, so drop those. The extensionless bare file *is* the logical
-        // key, so it is never a separate duplicate.
-        const items = page.items.filter((record) => {
-            const logicalKey = this.#bareFileLogicalKeys.get(record.key);
-            const isExtensionBearingBareFile = logicalKey !== undefined && logicalKey !== record.key;
-            return !(isExtensionBearingBareFile && presentKeys.has(logicalKey));
-        });
-        return {
-            items,
-            count: items.length,
-            limit: page.limit,
-            exclusiveStartKey: page.exclusiveStartKey,
-            isTruncated: page.isTruncated,
-            nextExclusiveStartKey: page.nextExclusiveStartKey,
-        };
+        // matching the `KeyValueStoreListKeysResult` contract.
+        return this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix);
     }
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.
      *
-     * The native `getPublicUrl` derives the URL from the key without probing bare-file extensions, so
-     * an `INPUT` that lives on disk as a hand-placed `INPUT.json` is resolved first. Nothing on disk
-     * means nothing to resolve — the requested key is used as-is, and the URL is the one the record will
-     * have once written.
+     * Existence-agnostic per the {@link KeyValueStoreBackend} contract: the URL points at the file
+     * the key is bound to if there is a record, and at the file it would be written to otherwise.
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
         parseArgument(key, keySchema);
-        const resolvedKey = (await this.resolveExistingKey(key)) ?? key;
-        return this.#nativeBackend.getPublicUrl(resolvedKey);
+        return this.#nativeBackend.getPublicUrl(key);
     }
     /**
      * Tests whether a record with the given key exists without retrieving its value.
@@ -136,14 +88,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
      */
     async recordExists(key) {
         parseArgument(key, keySchema);
-        return (await this.resolveExistingKey(key)) !== undefined;
+        return this.#nativeBackend.recordExists(key);
     }
     async getValue(key) {
         parseArgument(key, keySchema);
-        const fallbacks = this.#bareFallbacksFor(key);
-        const record = fallbacks
-            ? await this.#nativeBackend.resolveValue(key, fallbacks)
-            : await this.#nativeBackend.getValue(key);
+        const record = await this.#nativeBackend.getValue(key);
         if (record) {
             return {
                 key: record.key,
@@ -184,32 +133,4 @@ export class KeyValueStoreBackend extends CachedIdClient {
         parseArgument(key, keySchema);
         await this.#nativeBackend.deleteValue(key);
     }
-    /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Run-input
-     * keys fall back to bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists.
-     */
-    async resolveExistingKey(key) {
-        const fallbacks = this.#bareFallbacksFor(key);
-        if (fallbacks) {
-            return ((await this.#nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
-        }
-        return (await this.#nativeBackend.recordExists(key)) ? key : undefined;
-    }
-    /**
-     * Bare-file fallbacks for `key`, or `undefined` for a plain tracked-record lookup. A logical input
-     * key (`INPUT`) probes the whole extension ladder; a literal bare filename (`INPUT.json`, as listed
-     * by `listKeys`) probes only itself, so a listed key reads back under its own name.
-     */
-    #bareFallbacksFor(key) {
-        if (this.#inputKeys.includes(key)) {
-            return BARE_FILE_FALLBACKS;
-        }
-        const logicalKey = this.#bareFileLogicalKeys.get(key);
-        if (logicalKey === undefined) {
-            return undefined;
-        }
-        const { contentType } = BARE_FILE_FALLBACKS.find(({ extension }) => `${logicalKey}${extension}` === key);
-        return [{ extension: '', contentType }];
-    }
 }