@crawlee/core 4.0.0-beta.197 → 4.0.0-beta.198

package/configuration.d.ts

@@ -25,8 +25,6 @@ export declare const crawleeConfigFields: {
     internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 1_000 */
     systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
-    /** @default 'INPUT' */
-    inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default true */
     headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default false */
@@ -105,7 +103,6 @@ export interface Configuration extends ResolvedConfigValues {
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
  * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -

package/configuration.js

@@ -55,8 +55,6 @@ export const crawleeConfigFields = {
     internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
     /** @default 1_000 */
     systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
-    /** @default 'INPUT' */
-    inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
     /** @default true */
     headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
     /** @default false */
@@ -123,7 +121,6 @@ export const crawleeConfigFields = {
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
  * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -

package/memory-storage/memory-storage.js

@@ -139,13 +139,11 @@ export class MemoryStorageBackend {
         // `#resolveStorageKey` leaves `name` unset for the default and alias-keyed storages, which is what
         // marks them as run-scoped. `'default'` is the exception — it collapses onto the default storage.
         const isRunScoped = (store) => store.name === undefined || store.name === 'default';
-        const isDefault = (store) => store.name === 'default' || store.cacheKey === 'default';
         const purgeRunScoped = async (cache, purgeStore) => {
             await Promise.all(cache.filter(isRunScoped).map(async (store) => purgeStore(store)));
         };
         await Promise.all([
-            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
-            purgeRunScoped(this.#keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
+            purgeRunScoped(this.#keyValueStoreBackendCache, async (store) => store.purge()),
             purgeRunScoped(this.#datasetBackendCache, async (store) => store.purge()),
             purgeRunScoped(this.#requestQueueBackendCache, async (store) => store.purge()),
         ]);

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -34,13 +34,6 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     getMetadata(): Promise<storage.KeyValueStoreInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;
-    /**
-     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
-     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
-     * extension variants) when purging the default store. The in-memory key has no extension, so we
-     * preserve the bare `INPUT` key only.
-     */
-    purgeExceptInput(): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
     /**
      * In-memory records are not file-backed, so there is no public file URL to return.

package/memory-storage/resource-clients/key-value-store.js

@@ -20,11 +20,6 @@ const inputRecordSchema = z.object({
     ]),
     contentType: z.string().min(1).optional(),
 });
-/**
- * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
- * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
- */
-const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
 export class KeyValueStoreBackend extends BaseClient {
     name;
     /**
@@ -57,20 +52,6 @@ export class KeyValueStoreBackend extends BaseClient {
         this.#keyValueEntries.clear();
         this.updateTimestamps(true);
     }
-    /**
-     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
-     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
-     * extension variants) when purging the default store. The in-memory key has no extension, so we
-     * preserve the bare `INPUT` key only.
-     */
-    async purgeExceptInput() {
-        for (const key of this.#keyValueEntries.keys()) {
-            if (key !== KEY_VALUE_STORE_INPUT_KEY) {
-                this.#keyValueEntries.delete(key);
-            }
-        }
-        this.updateTimestamps(true);
-    }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         const items = [];

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.197",
+    "version": "4.0.0-beta.198",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.197",
-        "@crawlee/http-client": "4.0.0-beta.197",
-        "@crawlee/types": "4.0.0-beta.197",
-        "@crawlee/utils": "4.0.0-beta.197",
+        "@crawlee/fs-storage": "4.0.0-beta.198",
+        "@crawlee/http-client": "4.0.0-beta.198",
+        "@crawlee/types": "4.0.0-beta.198",
+        "@crawlee/utils": "4.0.0-beta.198",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "6dfbf74898b378341626fbe0c5ff7655b57ac305"
+    "gitHead": "889cfffcc20b2803f0d95d7cb006ad1e7f672452"
 }

package/service_locator.js

@@ -124,7 +124,6 @@ export class ServiceLocator {
             this.#storageBackend = configuration.persistStorage
                 ? new FileSystemStorageBackend({
                     localDataDirectory: configuration.storageDir,
-                    inputKey: configuration.inputKey,
                     logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
                 })
                 : new MemoryStorageBackend({

package/storages/key_value_store.d.ts

@@ -20,8 +20,6 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -36,7 +34,7 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -492,27 +490,6 @@ export declare class KeyValueStore {
      * @ignore
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * The input is read from the default {@link KeyValueStore} under the configured input key
-     * (`CRAWLEE_INPUT_KEY`, default `INPUT`).
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
 }
 /**
  * User-function used in the  {@link KeyValueStore.forEachKey} method.

package/storages/key_value_store.js

@@ -42,8 +42,6 @@ const openOptionsSchema = z.strictObject({
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -58,7 +56,7 @@ const openOptionsSchema = z.strictObject({
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -747,30 +745,6 @@ export class KeyValueStore {
         const store = await this.open();
         return store.setValue(key, value, options);
     }
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * The input is read from the default {@link KeyValueStore} under the configured input key
-     * (`CRAWLEE_INPUT_KEY`, default `INPUT`).
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static async getInput() {
-        const store = await this.open();
-        return store.getValue(store.configuration.inputKey);
-    }
 }
 /** Normalizes a codec-serialized value into the `Buffer | ArrayBuffer` shape raw record reads promise. */
 function normalizeSerializedValue(value) {

package/storages/utils.d.ts

@@ -16,7 +16,7 @@ interface PurgeDefaultStorageOptions {
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
  * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
- * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
+ * Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
@@ -31,7 +31,7 @@ export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOption
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
  * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
- * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
+ * Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}