@crawlee/core 4.0.0-beta.179 → 4.0.0-beta.180

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.179",
+    "version": "4.0.0-beta.180",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.179",
-        "@crawlee/http-client": "4.0.0-beta.179",
-        "@crawlee/types": "4.0.0-beta.179",
-        "@crawlee/utils": "4.0.0-beta.179",
+        "@crawlee/fs-storage": "4.0.0-beta.180",
+        "@crawlee/http-client": "4.0.0-beta.180",
+        "@crawlee/types": "4.0.0-beta.180",
+        "@crawlee/utils": "4.0.0-beta.180",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "8b7816439436b46bad1007a48df9e8bef0564bd0"
+    "gitHead": "4adb7b0a96ee16824ee53d18f437961d9287f977"
 }

package/recoverable_state.d.ts

@@ -1,5 +1,6 @@
-import type { Configuration, CrawleeLogger } from '@crawlee/core';
-import { KeyValueStore } from '@crawlee/core';
+import type { Configuration } from './configuration.js';
+import type { CrawleeLogger } from './log.js';
+import { KeyValueStore } from './storages/key_value_store.js';
 import type { Awaitable } from '@crawlee/types';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 /**

package/recoverable_state.js

@@ -1,5 +1,8 @@
 import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
-import { EventType, KeyValueStore, serviceLocator, StateValidationError } from '@crawlee/core';
+import { StateValidationError } from './errors.js';
+import { EventType } from './events/event_manager.js';
+import { serviceLocator } from './service_locator.js';
+import { KeyValueStore } from './storages/key_value_store.js';
 const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
 /**
  * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects

package/storages/request_loader.d.ts

@@ -59,7 +59,7 @@ export declare function joinRequestSourceStatuses(a: RequestSourceStatus, b: Req
  * "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
  *
  * Honoring this contract matters for three reasons:
- * - **Restarts and migrations:** loaders that persist their state (see {@link IRequestLoader.persistState})
+ * - **Restarts and migrations:** loaders that persist their state (such as {@link RequestList})
  *   treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
  *   but never marked handled will be crawled again.
  * - **Termination detection:** {@link IRequestLoader.checkReadiness} only reports `finished` once nothing is
@@ -117,12 +117,6 @@ export interface IRequestLoader {
      * {@link IRequestLoader}.
      */
     markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
-    /**
-     * Persists the current state of the loader into the default {@link KeyValueStore}.
-     *
-     * Not all loaders support persistence; implementations that do not should leave this `undefined`.
-     */
-    persistState?(): Promise<void>;
     /**
      * Combines the loader with a request manager to support adding and reclaiming requests.
      *

package/storages/request_manager_tandem.d.ts

@@ -73,11 +73,6 @@ export declare class RequestManagerTandem implements IRequestManager {
      * @inheritdoc
      */
     addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
-    /**
-     * Persists the state of the underlying read-only loader, if it supports persistence.
-     * @inheritdoc
-     */
-    persistState(): Promise<void>;
     /**
      * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
      * The read-only loader is immutable and cannot be purged, so only the manager side is reset.

package/storages/request_manager_tandem.js

@@ -170,13 +170,6 @@ export class RequestManagerTandem {
     async addRequestsBatched(requests, options) {
         return (await this.getRequestManager()).addRequestsBatched(requests, options);
     }
-    /**
-     * Persists the state of the underlying read-only loader, if it supports persistence.
-     * @inheritdoc
-     */
-    async persistState() {
-        await this.#requestLoader.persistState?.();
-    }
     /**
      * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
      * The read-only loader is immutable and cannot be purged, so only the manager side is reset.

package/validators.js

@@ -7,7 +7,7 @@ export const validators = {
     // Naming it browser page for future proofing with Playwright
     browserPage: schemas.objectWithKeys(['goto', 'evaluate', '$', 'on'], 'Expected a Puppeteer Page, got something else.'),
     proxyConfiguration: schemas.objectWithKeys(['newProxyInfo'], "Expected an object implementing the IProxyConfiguration interface (missing 'newProxyInfo'), got something else."),
-    requestList: schemas.objectWithKeys(['fetchNextRequest', 'persistState'], 'Expected a RequestList, got something else.'),
+    requestList: schemas.objectWithKeys(['fetchNextRequest', 'checkReadiness'], 'Expected a RequestList, got something else.'),
     requestQueue: schemas.objectWithKeys(['fetchNextRequest', 'addRequest'], 'Expected a RequestQueue, got something else.'),
     browserPool: schemas.objectWithKeys(['newPage', 'closePage', 'extractPageState', 'injectPageState'], "Expected an object implementing the IBrowserPool interface (missing one of 'newPage', 'closePage', 'extractPageState', 'injectPageState'), got something else."),
     sessionPool: schemas.objectWithKeys(['getSession'], "Expected an object implementing the ISessionPool interface (missing 'getSession'), got something else."),