@crawlee/fs-storage 4.0.0-beta.197 → 4.0.0-beta.198

package/file-system-storage.d.ts

@@ -1,5 +1,16 @@
 import type * as storage from '@crawlee/types';
 import type { CrawleeLogger } from '@crawlee/types';
+import type { AdoptionCandidate } from '@crawlee/fs-storage-native';
+/** What the key-value store hooks of {@link FileSystemStorageBackend} learn about the store at hand. */
+export interface KeyValueStoreHookOptions {
+    /** Whether the store is the run's default key-value store. */
+    isDefaultStore: boolean;
+}
+/** A file-system key-value store backend as seen by {@link FileSystemStorageBackend.purgeKeyValueStore}. */
+export interface PurgeableKeyValueStoreBackend extends storage.KeyValueStoreBackend {
+    /** Remove every record from the store except the given keys. */
+    purgeExcept(keys: string[]): Promise<void>;
+}
 export interface FileSystemStorageOptions {
     /**
      * Path to directory where the data will be saved.
@@ -25,18 +36,6 @@ export interface FileSystemStorageOptions {
      * @default 'single'
      */
     requestQueueAccess?: 'single' | 'shared';
-    /**
-     * The key the run input is read from — Crawlee's `inputKey` (`CRAWLEE_INPUT_KEY`).
-     *
-     * Like the conventional `INPUT`, this key may arrive in the default key-value store as a bare value
-     * file with no metadata sidecar (e.g. the Apify CLI writes the effective input to `__CLI_INPUT.json`
-     * and points the run at that key). Such a file is adopted into a record under this key when the
-     * store is opened, and the key is preserved when the default store is purged — exactly like
-     * `INPUT`, which is always treated as an input key regardless of this setting.
-     *
-     * @default 'INPUT'
-     */
-    inputKey?: string;
 }
 /**
  * A file-system storage backend backed by the native `@crawlee/fs-storage-native` Rust extension.
@@ -64,6 +63,26 @@ export declare class FileSystemStorageBackend implements storage.StorageBackend
     createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
     createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
     createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;
+    /**
+     * What the native `open` may turn into records: value files sitting in the store directory with no
+     * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     * Each becomes a record keyed by its filename, with the content type derived from the extension alone.
+     *
+     * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify
+     * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way.
+     *
+     * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
+     * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles)
+     * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the
+     * spared keys.
+     *
+     */
+    protected keyValueStoreAdoptionCandidates(_options: KeyValueStoreHookOptions): AdoptionCandidate[];
+    /**
+     * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the
+     * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input.
+     */
+    protected purgeKeyValueStore(store: PurgeableKeyValueStoreBackend, _options: KeyValueStoreHookOptions): Promise<void>;
     storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     /**
      * Cleans up the run-scoped storages before the run starts, sweeping the storage directories so that

package/file-system-storage.js

@@ -8,7 +8,6 @@ import { RequestQueueBackend } from './resource-clients/request-queue.js';
 const fileSystemStorageOptionsSchema = z.object({
     localDataDirectory: z.string(),
     requestQueueAccess: z.enum(['single', 'shared']).default('single'),
-    inputKey: z.string().min(1).default('INPUT'),
     logger: schemas.logger.optional(),
 });
 // The native package throws at load time on platforms without a published binary (e.g.
@@ -24,8 +23,6 @@ async function importNativeModule() {
 const DEFAULT_STORAGE_ALIAS = '__default__';
 /** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
 const DEFAULT_STORAGE_DIRECTORY = 'default';
-/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */
-const DEFAULT_INPUT_KEY = 'INPUT';
 /**
  * Content types declared for adopted files. The native client infers nothing from extensions, and
  * neither do we beyond this: a `.json` file is JSON, anything else is bytes for the
@@ -48,16 +45,13 @@ export class FileSystemStorageBackend {
     requestQueuesDirectory;
     logger;
     requestQueueAccess;
-    /** `INPUT` plus the configured `inputKey`, deduplicated. */
-    #inputKeys;
     #keyValueStoreBackendCache = [];
     #datasetBackendCache = [];
     #requestQueueBackendCache = [];
     constructor(options) {
-        const { logger, requestQueueAccess, inputKey, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
+        const { logger, requestQueueAccess, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
         this.logger = logger;
         this.requestQueueAccess = requestQueueAccess;
-        this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, inputKey])];
         this.localDataDirectory = localDataDirectory;
         this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
         this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
@@ -112,13 +106,12 @@ export class FileSystemStorageBackend {
         }
         const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory, 
         // useTestClock — always real wall-clock outside of native tests.
-        undefined, this.#adoptionCandidates(cacheKey === DEFAULT_STORAGE_DIRECTORY));
+        undefined, this.keyValueStoreAdoptionCandidates({ isDefaultStore: cacheKey === DEFAULT_STORAGE_DIRECTORY }));
         const newStore = await KeyValueStoreBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
             nativeBackend,
             logger: this.logger,
-            inputKeys: this.#inputKeys,
         });
         this.#keyValueStoreBackendCache.push(newStore);
         return newStore;
@@ -146,27 +139,19 @@ export class FileSystemStorageBackend {
     /**
      * What the native `open` may turn into records: value files sitting in the store directory with no
      * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor.
+     * Each becomes a record keyed by its filename, with the content type derived from the extension alone.
      *
-     * Each run-input key claims a bare `<key>` or `<key>.json` first — that is the layout the Apify CLI
-     * and the templates produce, and the key must end up being `INPUT` rather than `INPUT.json`. Only
-     * the default store holds the run input, so elsewhere an `INPUT.json` is just a file named
-     * `INPUT.json`, adopted by the trailing sweep like every other one.
+     * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify
+     * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way.
      *
      * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped
-     * store — purged on start unless its key is a run-input key.
+     * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles)
+     * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the
+     * spared keys.
+     *
      */
-    #adoptionCandidates(isDefaultStore) {
-        const inputCandidates = isDefaultStore
-            ? this.#inputKeys.map((key) => ({
-                key,
-                files: [
-                    { filename: key, contentType: ADOPTED_BINARY_CONTENT_TYPE },
-                    { filename: `${key}.json`, contentType: ADOPTED_JSON_CONTENT_TYPE },
-                ],
-            }))
-            : [];
+    keyValueStoreAdoptionCandidates(_options) {
         return [
-            ...inputCandidates,
             {
                 files: [
                     { filename: '*.json', contentType: ADOPTED_JSON_CONTENT_TYPE },
@@ -175,6 +160,13 @@ export class FileSystemStorageBackend {
             },
         ];
     }
+    /**
+     * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the
+     * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input.
+     */
+    async purgeKeyValueStore(store, _options) {
+        await store.purge();
+    }
     async storageExists(id, type) {
         let backends;
         let baseDir;
@@ -262,9 +254,7 @@ export class FileSystemStorageBackend {
      */
     async purge() {
         await Promise.all([
-            this.#purgeRunScopedStorages(this.keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }), 
-            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
-            async (store, isDefault) => (isDefault ? store.purgeExceptInput() : store.purge())),
+            this.#purgeRunScopedStorages(this.keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }), async (store, isDefault) => this.purgeKeyValueStore(store, { isDefaultStore: isDefault })),
             this.#purgeRunScopedStorages(this.datasetsDirectory, async (alias) => this.createDatasetBackend({ alias }), async (store) => store.purge()),
             this.#purgeRunScopedStorages(this.requestQueuesDirectory, async (alias) => this.createRequestQueueBackend({ alias }), async (store) => store.purge()),
         ]);

package/index.d.ts

@@ -1 +1,2 @@
 export * from './file-system-storage.js';
+export type { AdoptionCandidate } from '@crawlee/fs-storage-native';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.197",
+    "version": "4.0.0-beta.198",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,8 @@
     },
     "dependencies": {
         "@crawlee/fs-storage-native": ">=0.2.2 <0.3",
-        "@crawlee/types": "4.0.0-beta.197",
-        "@crawlee/utils": "4.0.0-beta.197",
+        "@crawlee/types": "4.0.0-beta.198",
+        "@crawlee/utils": "4.0.0-beta.198",
         "zod": "^4.5.4"
     },
     "lerna": {
@@ -54,5 +54,5 @@
             }
         }
     },
-    "gitHead": "6dfbf74898b378341626fbe0c5ff7655b57ac305"
+    "gitHead": "889cfffcc20b2803f0d95d7cb006ad1e7f672452"
 }

package/resource-clients/key-value-store.d.ts

@@ -12,11 +12,6 @@ export interface KeyValueStoreBackendOptions {
     cacheKey: string;
     nativeBackend: NativeFileSystemKeyValueStoreBackend;
     logger?: CrawleeLogger;
-    /**
-     * The run-input keys of this store, deduplicated — `INPUT` plus `FileSystemStorageOptions.inputKey`.
-     * All that is left of the run-input special case: the keys {@link purgeExceptInput} spares.
-     */
-    inputKeys: string[];
 }
 /**
  * A file-system key-value store backend backed by the native `@crawlee/fs-storage-native` Rust
@@ -37,11 +32,10 @@ export declare class KeyValueStoreBackend extends CachedIdClient implements stor
     drop(): Promise<void>;
     purge(): Promise<void>;
     /**
-     * Remove every record from the store except the run input. Used by
-     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input.
+     * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend}
+     * subclasses that keep some of the default store's records across the purge on start.
      */
-    purgeExceptInput(): Promise<void>;
+    purgeExcept(keys: string[]): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
     /**
      * Generates a public `file://` URL for accessing a specific record in the key-value store.

package/resource-clients/key-value-store.js

@@ -29,14 +29,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
     name;
     cacheKey;
     #nativeBackend;
-    /** See {@link KeyValueStoreBackendOptions.inputKeys}. */
-    #inputKeys;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
         this.#nativeBackend = options.nativeBackend;
-        this.#inputKeys = options.inputKeys;
     }
     get keyValueStoreDirectory() {
         return this.#nativeBackend.pathToKvs;
@@ -56,12 +53,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
         await this.#nativeBackend.purge();
     }
     /**
-     * Remove every record from the store except the run input. Used by
-     * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run
-     * while preserving the run's input.
+     * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend}
+     * subclasses that keep some of the default store's records across the purge on start.
      */
-    async purgeExceptInput() {
-        await this.#nativeBackend.purge(this.#inputKeys);
+    async purgeExcept(keys) {
+        await this.#nativeBackend.purge(keys);
     }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);