@crawlee/fs-storage 4.0.0-beta.170 → 4.0.0-beta.172

package/file-system-storage.d.ts

@@ -25,6 +25,18 @@ export interface FileSystemStorageOptions {
      * @default 'single'
      */
     requestQueueAccess?: 'single' | 'shared';
+    /**
+     * The key the run input is read from — Crawlee's `inputKey` (`CRAWLEE_INPUT_KEY`).
+     *
+     * Like the conventional `INPUT`, this key may live in the default key-value store as a bare value
+     * file with no metadata sidecar (e.g. the Apify CLI writes the effective input to `__CLI_INPUT.json`
+     * and points the run at that key). It is therefore readable out-of-band (`<key>`, `<key>.json`,
+     * `<key>.txt`, `<key>.bin`) and preserved when the default store is purged, exactly like `INPUT`,
+     * which is always kept regardless of this setting.
+     *
+     * @default 'INPUT'
+     */
+    inputKey?: string;
 }
 /**
  * A file-system storage backend backed by the native `@crawlee/fs-storage-native` Rust extension.

package/file-system-storage.js

@@ -8,6 +8,7 @@ import { RequestQueueBackend } from './resource-clients/request-queue.js';
 const fileSystemStorageOptionsSchema = z.object({
     localDataDirectory: z.string(),
     requestQueueAccess: z.enum(['single', 'shared']).default('single'),
+    inputKey: z.string().min(1).default('INPUT'),
     logger: schemas.logger.optional(),
 });
 // The native package throws at load time on platforms without a published binary (e.g.
@@ -38,13 +39,15 @@ export class FileSystemStorageBackend {
     requestQueuesDirectory;
     logger;
     requestQueueAccess;
+    #inputKey;
     #keyValueStoreBackendCache = [];
     #datasetBackendCache = [];
     #requestQueueBackendCache = [];
     constructor(options) {
-        const { logger, requestQueueAccess, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
+        const { logger, requestQueueAccess, inputKey, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
         this.logger = logger;
         this.requestQueueAccess = requestQueueAccess;
+        this.#inputKey = inputKey;
         this.localDataDirectory = localDataDirectory;
         this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
         this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
@@ -103,6 +106,7 @@ export class FileSystemStorageBackend {
             cacheKey,
             nativeBackend,
             logger: this.logger,
+            inputKey: this.#inputKey,
         });
         this.#keyValueStoreBackendCache.push(newStore);
         return newStore;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.170",
+    "version": "4.0.0-beta.172",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,8 @@
     },
     "dependencies": {
         "@crawlee/fs-storage-native": ">=0.2.1-beta.0 <0.3",
-        "@crawlee/types": "4.0.0-beta.170",
-        "@crawlee/utils": "4.0.0-beta.170",
+        "@crawlee/types": "4.0.0-beta.172",
+        "@crawlee/utils": "4.0.0-beta.172",
         "zod": "^4.5.4"
     },
     "lerna": {
@@ -54,5 +54,5 @@
             }
         }
     },
-    "gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
+    "gitHead": "619809c59ea633121f3aa9c8f333efeb9c6167af"
 }

package/resource-clients/key-value-store.d.ts

@@ -12,6 +12,8 @@ export interface KeyValueStoreBackendOptions {
     cacheKey: string;
     nativeBackend: NativeFileSystemKeyValueStoreBackend;
     logger?: CrawleeLogger;
+    /** The configured run-input key, see `FileSystemStorageOptions.inputKey`. Treated like `INPUT`. */
+    inputKey?: string;
 }
 /**
  * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
@@ -34,10 +36,8 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     /**
      * Remove every record from the store except the run input. Used by
      * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input, matching the historical file-system storage behavior.
-     *
-     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
-     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     * while preserving the run's input. The native keep-list matches exact filenames, so every extension
+     * variant of every input key is listed.
      */
     purgeExceptInput(): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
@@ -62,23 +62,9 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
     deleteValue(key: string): Promise<void>;
     /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
-     * key is checked against its tracked record; the run-input keys additionally fall back to
-     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
-     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
-     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
+     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Run-input
+     * keys fall back to bare files, in which case the matched on-disk key is returned so callers like
+     * `getPublicUrl` point at the file that exists.
      */
     private resolveExistingKey;
-    /**
-     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
-     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
-     *
-     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
-     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
-     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
-     *   the tracked record first, then the bare file at that exact name (a single empty-extension
-     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
-     */
-    private bareFallbacksFor;
 }

package/resource-clients/key-value-store.js

@@ -4,9 +4,9 @@ import { z } from 'zod';
 import { isStream } from '../utils.js';
 import { CachedIdClient } from './cached-id-client.js';
 /**
- * Out-of-band ("bare") value-file fallbacks tried when the {@link ALLOWED_BARE_FILES} lookup misses the tracked
- * record, so a lookup for `INPUT` also matches a hand-placed `INPUT.json`/`.txt`/`.bin`. Passed to the
- * native `resolveValue`/`resolveExistingKey`, which do the probing and re-keying.
+ * Out-of-band ("bare") value-file fallbacks tried when a run-input lookup misses the tracked record, so a
+ * lookup for `INPUT` also matches a hand-placed `INPUT.json`/`.txt`/`.bin`. Passed to the native
+ * `resolveValue`/`resolveExistingKey`, which do the probing and re-keying.
  *
  * Each entry declares the content type to report on a match — the native client does no MIME
  * inference. An empty `contentType` is its sentinel for "keep the synthesized
@@ -18,7 +18,8 @@ const BARE_FILE_FALLBACKS = [
     { extension: '.txt', contentType: 'text/plain; charset=utf-8' },
     { extension: '.bin', contentType: '' },
 ];
-const ALLOWED_BARE_FILES = ['INPUT'];
+/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */
+const DEFAULT_INPUT_KEY = 'INPUT';
 const keySchema = z.string();
 const inputRecordShape = z.object({
     key: z.string().min(1),
@@ -33,23 +34,6 @@ const inputRecordShape = z.object({
     ]),
     contentType: z.string().min(1).optional(),
 });
-/**
- * The out-of-band ("bare") files to surface from the native `listKeys`, derived from
- * {@link ALLOWED_BARE_FILES} × {@link BARE_FILE_FALLBACKS}. Each native {@link ListBareFallback}
- * `name` is the literal on-disk filename to probe (e.g. `INPUT.json`), and the native lists a match
- * under that same `name` — which is exactly the key we return, so a listed bare file round-trips
- * through `getValue`/`recordExists` (see {@link BARE_FILE_CONTENT_TYPES}).
- */
-const LIST_BARE_FALLBACKS = ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })));
-/**
- * Lookup from a bare file's literal on-disk name (e.g. `INPUT.json`) to the content type to report
- * for it, used to read a listed bare key back directly (`getValue('INPUT.json')`). The empty-extension
- * entry (`INPUT`) is intentionally excluded: an extensionless `INPUT` lookup goes through the
- * `resolveValue` fallback probing instead, which already covers the extensionless file.
- */
-const BARE_FILE_CONTENT_TYPES = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.filter(({ extension }) => extension !== '').map(({ extension, contentType }) => [`${key}${extension}`, contentType])));
-/** Maps a bare file's on-disk name (e.g. `INPUT.json`) to its logical key (e.g. `INPUT`), for dedup. */
-const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key])));
 /**
  * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
  * extension.
@@ -62,11 +46,20 @@ export class KeyValueStoreBackend extends CachedIdClient {
     name;
     cacheKey;
     #nativeBackend;
+    /** `INPUT` plus the configured `inputKey`, deduplicated. */
+    #inputKeys;
+    /** Bare files the native `listKeys` should surface, under their on-disk name (e.g. `INPUT.json`). */
+    #listBareFallbacks;
+    /** Bare-file on-disk name (`INPUT.json`) to logical key (`INPUT`). */
+    #bareFileLogicalKeys;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
         this.#nativeBackend = options.nativeBackend;
+        this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, options.inputKey ?? DEFAULT_INPUT_KEY])];
+        this.#listBareFallbacks = this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })));
+        this.#bareFileLogicalKeys = new Map(this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key])));
     }
     get keyValueStoreDirectory() {
         return this.#nativeBackend.pathToKvs;
@@ -88,13 +81,12 @@ export class KeyValueStoreBackend extends CachedIdClient {
     /**
      * Remove every record from the store except the run input. Used by
      * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input, matching the historical file-system storage behavior.
-     *
-     * The native `purge` keep-list matches by exact key with no extension globbing, so we pass every
-     * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
+     * while preserving the run's input. The native keep-list matches exact filenames, so every extension
+     * variant of every input key is listed.
      */
     async purgeExceptInput() {
-        await this.#nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
+        const keep = this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => `${key}${extension}`));
+        await this.#nativeBackend.purge(keep);
     }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
@@ -103,7 +95,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
         // The native `listKeys` already returns a self-describing page (items + pagination cursors)
         // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
-        const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
+        const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, this.#listBareFallbacks);
         const presentKeys = new Set(page.items.map((record) => record.key));
         // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
         // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
@@ -111,7 +103,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // etc.) for the same logical key, so drop those. The extensionless bare file *is* the logical
         // key, so it is never a separate duplicate.
         const items = page.items.filter((record) => {
-            const logicalKey = BARE_FILE_LOGICAL_KEYS.get(record.key);
+            const logicalKey = this.#bareFileLogicalKeys.get(record.key);
             const isExtensionBearingBareFile = logicalKey !== undefined && logicalKey !== record.key;
             return !(isExtensionBearingBareFile && presentKeys.has(logicalKey));
         });
@@ -150,7 +142,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
     }
     async getValue(key) {
         parseArgument(key, keySchema);
-        const fallbacks = this.bareFallbacksFor(key);
+        const fallbacks = this.#bareFallbacksFor(key);
         const record = fallbacks
             ? await this.#nativeBackend.resolveValue(key, fallbacks)
             : await this.#nativeBackend.getValue(key);
@@ -195,39 +187,31 @@ export class KeyValueStoreBackend extends CachedIdClient {
         await this.#nativeBackend.deleteValue(key);
     }
     /**
-     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
-     * key is checked against its tracked record; the run-input keys additionally fall back to
-     * out-of-band bare files, in which case the matched on-disk key is returned so callers like
-     * `getPublicUrl` point at the file that exists. Two run-input shapes are handled (see
-     * {@link bareFallbacksFor}): the logical `INPUT`, which probes the conventional extensions, and a
-     * literal bare filename such as `INPUT.json` as listed by `listKeys`, which resolves itself.
+     * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Run-input
+     * keys fall back to bare files, in which case the matched on-disk key is returned so callers like
+     * `getPublicUrl` point at the file that exists.
      */
     async resolveExistingKey(key) {
-        const fallbacks = this.bareFallbacksFor(key);
+        const fallbacks = this.#bareFallbacksFor(key);
         if (fallbacks) {
             return ((await this.#nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
         }
         return (await this.#nativeBackend.recordExists(key)) ? key : undefined;
     }
     /**
-     * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or
-     * `undefined` if `key` is a plain tracked-record lookup with no bare-file probing.
-     *
-     * - The logical run-input key (`INPUT`) probes the full extension ladder (`INPUT`, `INPUT.json`,
-     *   `INPUT.txt`, `INPUT.bin`), matching how Crawlee reads run input.
-     * - A literal bare filename as surfaced by `listKeys` (`INPUT.json`/`.txt`/`.bin`) resolves itself:
-     *   the tracked record first, then the bare file at that exact name (a single empty-extension
-     *   fallback), so a listed key round-trips through `getValue`/`recordExists`.
+     * Bare-file fallbacks for `key`, or `undefined` for a plain tracked-record lookup. A logical input
+     * key (`INPUT`) probes the whole extension ladder; a literal bare filename (`INPUT.json`, as listed
+     * by `listKeys`) probes only itself, so a listed key reads back under its own name.
      */
-    // eslint-disable-next-line class-methods-use-this
-    bareFallbacksFor(key) {
-        if (ALLOWED_BARE_FILES.includes(key)) {
+    #bareFallbacksFor(key) {
+        if (this.#inputKeys.includes(key)) {
             return BARE_FILE_FALLBACKS;
         }
-        const contentType = BARE_FILE_CONTENT_TYPES.get(key);
-        if (contentType !== undefined) {
-            return [{ extension: '', contentType }];
+        const logicalKey = this.#bareFileLogicalKeys.get(key);
+        if (logicalKey === undefined) {
+            return undefined;
         }
-        return undefined;
+        const { contentType } = BARE_FILE_FALLBACKS.find(({ extension }) => `${logicalKey}${extension}` === key);
+        return [{ extension: '', contentType }];
     }
 }