@crawlee/core 4.0.0-beta.190 → 4.0.0-beta.192

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.190",
+    "version": "4.0.0-beta.192",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.190",
-        "@crawlee/http-client": "4.0.0-beta.190",
-        "@crawlee/types": "4.0.0-beta.190",
-        "@crawlee/utils": "4.0.0-beta.190",
+        "@crawlee/fs-storage": "4.0.0-beta.192",
+        "@crawlee/http-client": "4.0.0-beta.192",
+        "@crawlee/types": "4.0.0-beta.192",
+        "@crawlee/utils": "4.0.0-beta.192",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "0a6af5ff4704225ddb009a7dd58bdeaad48ebb14"
+    "gitHead": "729823422c7168238d8b5fcf38dcb798cbf0af0b"
 }

package/recoverable_state.d.ts

@@ -48,9 +48,9 @@ export interface RecoverableStatePersistenceOptions {
     persistenceTimeoutMillis?: number;
 }
 /**
- * Options for configuring the RecoverableState
+ * The fields of {@link RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
  */
-export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
+export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
     /**
      * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
      *
@@ -69,14 +69,34 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>,
     /**
      * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
      * If not provided, the state is persisted as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it has to produce what
+     * {@link KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
+     * stream.
      */
     serialize?: StateConversion<TStateModel, TPersistedState>;
     /**
      * Optional conversion of a persisted value back to the state model, and the place to validate a record before
      * trusting it. If not provided, the persisted value is used as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
+     * instead of a parsed value.
      */
     deserialize?: StateConversion<TPersistedState, TStateModel>;
+    /**
+     * Content type of the persisted record. Setting it hands the record encoding over to
+     * {@link RecoverableStateBaseOptions.serialize} and {@link RecoverableStateBaseOptions.deserialize}, both of
+     * which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
+     * for `JSON.stringify`, which `serialize` can then stream out instead.
+     */
+    contentType?: string;
 }
+/**
+ * Options for configuring the RecoverableState
+ */
+export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
+    contentType?: undefined;
+} | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
  *

package/recoverable_state.js

@@ -1,3 +1,4 @@
+import { Readable } from 'node:stream';
 import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
 import { StateValidationError } from './errors.js';
 import { EventType } from './events/event_manager.js';
@@ -47,6 +48,7 @@ export class RecoverableState {
     #log;
     #serialize;
     #deserialize;
+    #contentType;
     #persistStateQuietly;
     /**
      * Initialize a new recoverable state object.
@@ -65,6 +67,10 @@ export class RecoverableState {
         this.#configuration = options.configuration;
         this.#keyValueStore = options.keyValueStore ?? null;
         this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#contentType = options.contentType;
+        if (this.#contentType !== undefined && (options.serialize === undefined || options.deserialize === undefined)) {
+            throw new Error(`A 'contentType' for the state persisted under key '${this.#persistStateKey}' requires both 'serialize' and 'deserialize' - the record is no longer JSON the default codec can handle.`);
+        }
         this.#serialize = this.#toConversion(options.serialize);
         this.#deserialize = this.#toConversion(options.deserialize);
         // A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
@@ -203,7 +209,11 @@ export class RecoverableState {
         }
         this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
         const serializedState = await this.#serialize(this.currentValue);
-        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, serializedState), 'Persisting the state');
+        await this.#withTimeout(async () => this.#contentType === undefined
+            ? keyValueStore.setValue(this.#persistStateKey, serializedState)
+            : keyValueStore.setValue(this.#persistStateKey, serializedState, {
+                contentType: this.#contentType,
+            }), 'Persisting the state');
     }
     /** Falls back to the default store when none was supplied - persistence has to write somewhere. */
     async #openKeyValueStore() {
@@ -231,7 +241,19 @@ export class RecoverableState {
         if (keyValueStore === null) {
             return;
         }
-        const storedState = await this.#withTimeout(async () => keyValueStore.getValue(this.#persistStateKey), 'Loading the persisted state');
+        // With a content type, the record is whatever `serialize` produced, so the codec's parse is skipped and
+        // `deserialize` gets the bytes as a stream - the shape a streaming parser wants.
+        // TODO: read the record as a stream instead of buffering it and wrapping (https://github.com/apify/crawlee/issues/2929).
+        const storedState = await this.#withTimeout(async () => {
+            if (this.#contentType === undefined) {
+                return keyValueStore.getValue(this.#persistStateKey);
+            }
+            const record = await keyValueStore.getRecord(this.#persistStateKey);
+            if (record === null) {
+                return null;
+            }
+            return Readable.from(Buffer.isBuffer(record.value) ? record.value : Buffer.from(record.value));
+        }, 'Loading the persisted state');
         if (storedState === null || storedState === undefined) {
             return;
         }