@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/serialization.js

@@ -1,7 +1,7 @@
 import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
 import util from 'node:util';
 import zlib from 'node:zlib';
-import StreamArray from 'stream-json/streamers/StreamArray.js';
+import StreamArray from 'stream-json/streamers/stream-array.js';
 import { z } from 'zod';
 import { parseArgument, schemas } from './validators.js';
 const pipeline = util.promisify(streamPipeline);
@@ -76,7 +76,7 @@ export async function serializeArray(data) {
 export async function deserializeArray(compressedData) {
     parseArgument(compressedData, uint8ArraySchema);
     const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
-    await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
+    await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParserAsStream(), collector);
     return chunks;
 }
 /**
@@ -90,7 +90,7 @@ export async function deserializeArray(compressedData) {
  */
 export function createDeserialize(compressedData) {
     parseArgument(compressedData, uint8ArraySchema);
-    const streamArray = StreamArray.withParser();
+    const streamArray = StreamArray.withParserAsStream();
     const destination = pluckValue(streamArray);
     streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
     return destination;

package/service_locator.d.ts

@@ -60,6 +60,7 @@ interface ServiceLocatorInterface {
     getChildLog(prefix: string): CrawleeLogger;
     /**
      * Get the storage instance manager (shared across all storage types).
+     * @internal
      */
     getStorageInstanceManager(): StorageInstanceManager;
     /**
@@ -138,7 +139,9 @@ export declare class ServiceLocator implements ServiceLocatorInterface {
     getLogger(): CrawleeLogger;
     setLogger(logger: CrawleeLogger): void;
     getChildLog(prefix: string): CrawleeLogger;
+    /** @internal */
     getStorageInstanceManager(): StorageInstanceManager;
+    /** @internal */
     reset(): void;
 }
 /**

package/service_locator.js

@@ -161,12 +161,14 @@ export class ServiceLocator {
     getChildLog(prefix) {
         return this.getLogger().child({ prefix });
     }
+    /** @internal */
     getStorageInstanceManager() {
         if (!ServiceLocator.#storageInstanceManager) {
             ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
         }
         return ServiceLocator.#storageInstanceManager;
     }
+    /** @internal */
     reset() {
         this.#configuration = undefined;
         this.#eventManager = undefined;

package/storages/dataset.d.ts

@@ -1,6 +1,5 @@
 import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import type { CrawleeLogger } from '../log.js';
 import type { JournalEntry } from './transaction.js';
 import type { DatasetStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
@@ -142,10 +141,9 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
 export declare class Dataset<Data extends Dictionary = Dictionary> {
     #private;
     readonly configuration: Configuration;
-    id: string;
-    name?: string;
-    backend: DatasetBackend<Data>;
-    log: CrawleeLogger;
+    readonly id: string;
+    readonly name?: string;
+    private readonly backend;
     /**
      * @internal
      */
@@ -171,14 +169,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
-    /**
-     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
-     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
-     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
-     */
-    private readPage;
-    /** The active transaction's buffered writes to this dataset, derived from its journal. */
-    private bufferedJournalEntries;
     /** @internal */
     commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
@@ -323,8 +313,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * @param [options] An object containing extra options for `reduce()`
      */
     reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
-    private fetchEntryPages;
-    private fetchPages;
     /**
      * Returns dataset items.
      *
@@ -391,6 +379,11 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Removes all items from the dataset but keeps the dataset itself, along with its
+     * {@link Dataset.id|`id`} and {@link Dataset.name|`name`}.
+     */
+    purge(): Promise<void>;
     /**
      * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
      *
@@ -469,6 +462,7 @@ export interface DatasetReducer<T, Data> {
      */
     (memo: T, item: Data, index: number): Awaitable<T>;
 }
+/** @internal */
 export interface DatasetOptions {
     /** Resolved metadata for the dataset, as returned by the backend's `getMetadata()`. */
     metadata: DatasetInfo;

package/storages/dataset.js

@@ -91,8 +91,8 @@ export class Dataset {
     configuration;
     id;
     name;
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- tests spy on the backend directly
     backend;
-    log;
     #statsTracker = new StorageStatsTracker({
         readCount: 0,
         writeCount: 0,
@@ -105,7 +105,6 @@ export class Dataset {
         this.id = options.metadata.id;
         this.name = options.metadata.name;
         this.backend = options.backend;
-        this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
     }
     /**
      * Backend-independent usage counters tracked for this dataset (read / write operations issued to
@@ -126,6 +125,7 @@ export class Dataset {
      *   The objects must be serializable to JSON.
      */
     async pushData(data) {
+        tryCancel();
         const transaction = activeStorageTransaction();
         parseArgument(data, schemas.anyObject);
         // Normalize to array and validate each item
@@ -152,7 +152,7 @@ export class Dataset {
      */
     async getData(options = {}) {
         try {
-            return await this.readPage(options);
+            return await this.#readPage(options);
         }
         catch (e) {
             const error = e;
@@ -167,8 +167,8 @@ export class Dataset {
      * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
      * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
      */
-    async readPage(options) {
-        const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items);
+    async #readPage(options) {
+        const buffered = this.#bufferedJournalEntries()?.flatMap((entry) => entry.items);
         // Every branch below hits the backend exactly once.
         this.#statsTracker.add('readCount');
         if (!buffered?.length) {
@@ -224,7 +224,7 @@ export class Dataset {
         };
     }
     /** The active transaction's buffered writes to this dataset, derived from its journal. */
-    bufferedJournalEntries() {
+    #bufferedJournalEntries() {
         const transaction = activeStorageTransaction();
         return transaction?.journal.filter((entry) => entry.type === 'dataset' && entry.participant === this);
     }
@@ -250,7 +250,7 @@ export class Dataset {
     async export(options = {}) {
         tryCancel();
         const items = [];
-        for await (const page of this.fetchPages(options)) {
+        for await (const page of this.#fetchPages(options)) {
             items.push(...page.items);
         }
         return items;
@@ -348,7 +348,7 @@ export class Dataset {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        const buffered = this.bufferedJournalEntries();
+        const buffered = this.#bufferedJournalEntries();
         const metadata = await this.backend.getMetadata();
         if (buffered?.length) {
             const lastWriteAt = buffered[buffered.length - 1].recordedAt;
@@ -432,16 +432,16 @@ export class Dataset {
         await this.forEach(wrappedFunc, options);
         return currentMemo;
     }
-    async *fetchEntryPages(options) {
+    async *#fetchEntryPages(options) {
         let index = options.offset ?? 0;
-        for await (const page of this.fetchPages(options)) {
+        for await (const page of this.#fetchPages(options)) {
             yield {
                 ...page,
                 items: page.items.map((item) => [index++, item]),
             };
         }
     }
-    async *fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
+    async *#fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
         let offset = options.offset ?? 0;
         const totalLimit = options.limit;
         let yielded = 0;
@@ -449,7 +449,7 @@ export class Dataset {
             const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
             if (fetchLimit <= 0)
                 break;
-            const page = await this.readPage({ ...options, offset, limit: fetchLimit });
+            const page = await this.#readPage({ ...options, offset, limit: fetchLimit });
             yield page;
             yielded += page.items.length;
             if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
@@ -483,7 +483,7 @@ export class Dataset {
     values(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchPages(options),
+            createPages: () => this.#fetchPages(options),
             extractItems: (page) => page.items,
         });
     }
@@ -513,7 +513,7 @@ export class Dataset {
     entries(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchEntryPages(options),
+            createPages: () => this.#fetchEntryPages(options),
             extractItems: (page) => page.items,
         });
     }
@@ -541,6 +541,14 @@ export class Dataset {
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
+    /**
+     * Removes all items from the dataset but keeps the dataset itself, along with its
+     * {@link Dataset.id|`id`} and {@link Dataset.name|`name`}.
+     */
+    async purge() {
+        rejectOperationInTransaction('Dataset.purge()');
+        await this.backend.purge();
+    }
     /**
      * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
      *

package/storages/index.d.ts

@@ -5,10 +5,9 @@ export * from './request_list.js';
 export type * from './request_loader.js';
 export type * from './request_manager.js';
 export * from './request_queue.js';
-export * from './storage_instance_manager.js';
-export * from './storage_stats.js';
+export type { DefaultStorageIdentifier, ExplicitStorageIdentifier, IStorage, StorageIdentifier, } from './storage_instance_manager.js';
+export { StorageInstanceManager } from './storage_instance_manager.js';
+export type { DatasetStats, KeyValueStoreStats, RequestQueueStats } from './storage_stats.js';
 export * from './utils.js';
 export * from './transaction.js';
-export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
-export * from './throttling_request_manager.js';

package/storages/index.js

@@ -3,10 +3,7 @@ export * from './key_value_store.js';
 export * from './key_value_store_codec.js';
 export * from './request_list.js';
 export * from './request_queue.js';
-export * from './storage_instance_manager.js';
-export * from './storage_stats.js';
+export { StorageInstanceManager } from './storage_instance_manager.js';
 export * from './utils.js';
 export * from './transaction.js';
-export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
-export * from './throttling_request_manager.js';

package/storages/key_value_store.d.ts

@@ -20,8 +20,6 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -29,15 +27,14 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * ```
  * {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
  * ```
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
- * unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`.
  * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
  *
  * **Example usage:**
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -65,7 +62,7 @@ export declare class KeyValueStore {
     readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
-    private readonly backend;
+    readonly backend: KeyValueStoreBackend;
     /**
      * @internal
      */
@@ -139,21 +136,6 @@ export declare class KeyValueStore {
      *   on the MIME content type of the record, or the default value if the key is missing from the store.
      */
     getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
-    /**
-     * The active transaction's last buffered write per key for this store, derived from its journal.
-     * An entry with a `null` value is a tombstone (an in-transaction deletion).
-     */
-    private bufferedJournalEntries;
-    /**
-     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
-     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
-     * round-trip); tombstoned key → `null`; otherwise the backend.
-     *
-     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
-     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
-     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
-     */
-    private readRecord;
     /**
      * Reads a record from the key-value store without parsing the value.
      *
@@ -189,9 +171,6 @@ export declare class KeyValueStore {
      */
     recordExists(key: string): Promise<boolean>;
     getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
-    private ensurePersistStateEvent;
-    private fetchKeyValuePages;
-    private fetchKeyPages;
     /**
      * Saves or deletes a record in the key-value store.
      * The function returns a promise that resolves once the record has been saved or deleted.
@@ -243,6 +222,11 @@ export declare class KeyValueStore {
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@link KeyValueStore.id|`id`} and {@link KeyValueStore.name|`name`}.
+     */
+    purge(): Promise<void>;
     /** @internal */
     clearCache(): void;
     /**
@@ -355,7 +339,9 @@ export declare class KeyValueStore {
     /**
      * Returns a file URL for the given key.
      *
-     * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
+     * The URL is derived from the key, so it is also returned for a record that does not exist (yet) —
+     * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when
+     * the storage has no file URLs at all (e.g. the in-memory storage).
      *
      * @param key The key of the record to generate the public URL for.
      */
@@ -486,27 +472,6 @@ export declare class KeyValueStore {
      * @ignore
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * The input is read from the default {@link KeyValueStore} under the configured input key
-     * (`CRAWLEE_INPUT_KEY`, default `INPUT`).
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
 }
 /**
  * User-function used in the  {@link KeyValueStore.forEachKey} method.
@@ -522,6 +487,7 @@ export interface KeyConsumer {
         size: number;
     }): Awaitable<void>;
 }
+/** @internal */
 export interface KeyValueStoreOptions {
     /** Resolved metadata for the key-value store, as returned by the backend's `getMetadata()`. */
     metadata: KeyValueStoreInfo;

package/storages/key_value_store.js

@@ -42,8 +42,6 @@ const openOptionsSchema = z.strictObject({
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -51,15 +49,14 @@ const openOptionsSchema = z.strictObject({
  * ```
  * {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
  * ```
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
- * unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`.
  * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
  *
  * **Example usage:**
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -86,7 +83,6 @@ export class KeyValueStore {
     configuration;
     id;
     name;
-    // kept as TS-private: key_value_store tests spy on the backend directly
     backend;
     #persistStateEventStarted = false;
     /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
@@ -148,7 +144,7 @@ export class KeyValueStore {
     async getValue(key, defaultValue) {
         tryCancel();
         parseArgument(key, keySchema);
-        const record = await this.readRecord(key);
+        const record = await this.#readRecord(key);
         // A missing record falls back to the default; a record that parses to a falsy value (including
         // a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
         if (!record) {
@@ -161,7 +157,7 @@ export class KeyValueStore {
      * The active transaction's last buffered write per key for this store, derived from its journal.
      * An entry with a `null` value is a tombstone (an in-transaction deletion).
      */
-    bufferedJournalEntries() {
+    #bufferedJournalEntries() {
         const transaction = activeStorageTransaction();
         if (!transaction)
             return undefined;
@@ -182,7 +178,7 @@ export class KeyValueStore {
      * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
      * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
      */
-    async readRecord(key, buffered = this.bufferedJournalEntries()) {
+    async #readRecord(key, buffered = this.#bufferedJournalEntries()) {
         const entry = buffered?.get(key);
         if (entry) {
             if (entry.value === null) {
@@ -232,7 +228,7 @@ export class KeyValueStore {
     async getRecord(key) {
         tryCancel();
         parseArgument(key, keySchema);
-        return this.readRecord(key);
+        return this.#readRecord(key);
     }
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
@@ -243,7 +239,7 @@ export class KeyValueStore {
     async recordExists(key) {
         tryCancel();
         parseArgument(key, keySchema);
-        const entry = this.bufferedJournalEntries()?.get(key);
+        const entry = this.#bufferedJournalEntries()?.get(key);
         if (entry) {
             return entry.value !== null;
         }
@@ -265,10 +261,10 @@ export class KeyValueStore {
             return this.#cache.get(key);
         }
         this.#cache.set(key, value);
-        this.ensurePersistStateEvent();
+        this.#ensurePersistStateEvent();
         return value;
     }
-    ensurePersistStateEvent() {
+    #ensurePersistStateEvent() {
         if (this.#persistStateEventStarted) {
             return;
         }
@@ -281,15 +277,15 @@ export class KeyValueStore {
         });
         this.#persistStateEventStarted = true;
     }
-    async *fetchKeyValuePages(options, mapRecord) {
+    async *#fetchKeyValuePages(options, mapRecord) {
         // Reduce the journal once for the whole iteration, not once per key inside `readRecord`.
-        const buffered = this.bufferedJournalEntries();
-        for await (const page of this.fetchKeyPages(options, buffered)) {
+        const buffered = this.#bufferedJournalEntries();
+        for await (const page of this.#fetchKeyPages(options, buffered)) {
             const results = [];
             for (const item of page) {
                 // The shared transaction-aware read, so a key that exists only in the transaction resolves
                 // here instead of being dropped (`values()` would disagree with `keys()` on length).
-                const record = await this.readRecord(item.key, buffered);
+                const record = await this.#readRecord(item.key, buffered);
                 if (record) {
                     const parsed = parseValue(record.value, record.contentType ?? null);
                     results.push(mapRecord(item.key, parsed));
@@ -298,7 +294,7 @@ export class KeyValueStore {
             yield results;
         }
     }
-    async *fetchKeyPages(options, buffered = this.bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
+    async *#fetchKeyPages(options, buffered = this.#bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
         // Buffered keys are emitted first, then the real pages with any buffered (or tombstoned) key
         // skipped - a merge-join is not an option, since `listKeys` promises no sort order.
         const shadowedKeys = new Set();
@@ -460,6 +456,16 @@ export class KeyValueStore {
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@link KeyValueStore.id|`id`} and {@link KeyValueStore.name|`name`}.
+     */
+    async purge() {
+        rejectOperationInTransaction('KeyValueStore.purge()');
+        await this.backend.purge();
+        // The auto-saved values this cache holds are no longer in the store.
+        this.#cache.clear();
+    }
     /** @internal */
     clearCache() {
         rejectOperationInTransaction('KeyValueStore.clearCache()');
@@ -491,7 +497,7 @@ export class KeyValueStore {
         parseArgument(iteratee, schemas.anyFunction);
         const parsedOptions = parseArgument(options, iteratorOptionsSchema);
         let index = 0;
-        for await (const page of this.fetchKeyPages(parsedOptions)) {
+        for await (const page of this.#fetchKeyPages(parsedOptions)) {
             for (const item of page) {
                 await iteratee(item.key, index++, { size: item.size });
             }
@@ -523,7 +529,7 @@ export class KeyValueStore {
     keys(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchKeyPages(options),
+            createPages: () => this.#fetchKeyPages(options),
             extractItems: (page) => page.map((item) => item.key),
         });
     }
@@ -553,7 +559,7 @@ export class KeyValueStore {
     values(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
+            createPages: () => this.#fetchKeyValuePages(options, (_key, value) => value),
             extractItems: (page) => page,
         });
     }
@@ -583,7 +589,7 @@ export class KeyValueStore {
     entries(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
+            createPages: () => this.#fetchKeyValuePages(options, (key, value) => [key, value]),
             extractItems: (page) => page,
         });
     }
@@ -605,7 +611,9 @@ export class KeyValueStore {
     /**
      * Returns a file URL for the given key.
      *
-     * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
+     * The URL is derived from the key, so it is also returned for a record that does not exist (yet) —
+     * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when
+     * the storage has no file URLs at all (e.g. the in-memory storage).
      *
      * @param key The key of the record to generate the public URL for.
      */
@@ -737,30 +745,6 @@ export class KeyValueStore {
         const store = await this.open();
         return store.setValue(key, value, options);
     }
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * The input is read from the default {@link KeyValueStore} under the configured input key
-     * (`CRAWLEE_INPUT_KEY`, default `INPUT`).
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static async getInput() {
-        const store = await this.open();
-        return store.getValue(store.configuration.inputKey);
-    }
 }
 /** Normalizes a codec-serialized value into the `Buffer | ArrayBuffer` shape raw record reads promise. */
 function normalizeSerializedValue(value) {

package/storages/key_value_store_codec.js

@@ -1,4 +1,4 @@
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import JSON5 from 'json5';
 import { jsonStringifyExtended } from '@apify/utilities';
 import { isBuffer, isStream } from '../byte_utils.js';
@@ -67,17 +67,12 @@ export function parseValue(body, contentTypeHeader) {
     // No content type at all → we have no basis for interpretation; hand back the raw value.
     if (contentTypeHeader === null)
         return body;
-    let contentType;
-    let charset;
-    try {
-        const result = contentTypeParser.parse(contentTypeHeader);
-        contentType = result.type;
-        charset = result.parameters.charset;
-    }
-    catch {
-        // Unparseable header → keep the original value rather than a mangled string.
+    const result = parseContentType(contentTypeHeader);
+    // Unparseable header → keep the original value rather than a mangled string.
+    if (!isTypeValid(result.type))
         return body;
-    }
+    const contentType = result.type;
+    const charset = result.parameters.charset;
     // If we can't successfully interpret it, we return the original value rather than mangling it.
     if (!areDataStringifiable(contentType, charset))
         return body;

package/storages/request_dedup_cache.d.ts

@@ -13,10 +13,8 @@
  */
 export declare class RequestDeduplicationCache {
     #private;
-    private readonly size;
     constructor(size?: number);
     get(cacheKey: string): string | null;
     add(cacheKey: string, requestId: string): void;
     clear(): void;
-    private indexOf;
 }