@crawlee/core 4.0.0-beta.170 → 4.0.0-beta.171

package/errors.d.ts

@@ -9,11 +9,6 @@ export declare class NonRetryableError extends Error {
  */
 export declare class CriticalError extends NonRetryableError {
 }
-/**
- * @ignore
- */
-export declare class MissingRouteError extends CriticalError {
-}
 /**
  * A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
  * error types do not have to depend on `@standard-schema/spec`.
@@ -45,34 +40,6 @@ export declare class StateValidationError extends Error {
     readonly issues: readonly SchemaIssue[];
     constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
 }
-/**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
- *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
- */
-export declare class RetryRequestError extends Error {
-    constructor(message?: string);
-}
-/**
- * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
- *
- * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
- * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
- * the domain's backoff expires, so retries are paced rather than immediate.
- */
-export declare class RequestThrottledError extends RetryRequestError {
-    constructor(message?: string);
-}
-/**
- * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
- * abandoned rather than kept waiting.
- *
- * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
- * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
- * resumes them once the domain recovers.
- */
-export declare class PersistentRateLimitError extends CriticalError {
-}
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *
@@ -81,24 +48,6 @@ export declare class PersistentRateLimitError extends CriticalError {
 export declare class SessionError extends Error {
     constructor(message?: string);
 }
-/**
- * Thrown when a requested session is not found in the referenced SessionPool.
- */
-export declare class MissingSessionError extends Error {
-    constructor(sessionId?: string);
-}
-export declare class ContextPipelineInterruptedError extends Error {
-    constructor(message?: string);
-}
-export declare class ContextPipelineInitializationError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
-}
-export declare class ContextPipelineCleanupError extends CriticalError {
-    constructor(error: unknown, options?: ErrorOptions);
-}
-export declare class RequestHandlerError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
-}
 /**
  * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
  * that ran after the request's writes had already been committed.
@@ -115,9 +64,3 @@ export declare class AfterCommitError extends NonRetryableError {
 export declare class ServiceConflictError extends Error {
     constructor(serviceName: string, newValue: unknown, existingValue: unknown);
 }
-/**
- * Thrown by crawlers when `skipNavigation` is used on a request.
- * Subclasses can catch this error to skip their own navigation-dependent logic.
- */
-export declare class NavigationSkippedError extends NonRetryableError {
-}

package/errors.js

@@ -10,11 +10,6 @@ export class NonRetryableError extends Error {
  */
 export class CriticalError extends NonRetryableError {
 }
-/**
- * @ignore
- */
-export class MissingRouteError extends CriticalError {
-}
 function formatIssues(issues) {
     return issues
         .map((issue) => {
@@ -54,38 +49,6 @@ export class StateValidationError extends Error {
         this.issues = issues;
     }
 }
-/**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
- *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
- */
-export class RetryRequestError extends Error {
-    constructor(message) {
-        super(message ?? "Request is being retried at the user's request");
-    }
-}
-/**
- * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
- *
- * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
- * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
- * the domain's backoff expires, so retries are paced rather than immediate.
- */
-export class RequestThrottledError extends RetryRequestError {
-    constructor(message) {
-        super(message ?? 'Request is being retried later because its domain is rate-limiting us');
-    }
-}
-/**
- * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
- * abandoned rather than kept waiting.
- *
- * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
- * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
- * resumes them once the domain recovers.
- */
-export class PersistentRateLimitError extends CriticalError {
-}
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *
@@ -96,34 +59,6 @@ export class SessionError extends Error {
         super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
     }
 }
-/**
- * Thrown when a requested session is not found in the referenced SessionPool.
- */
-export class MissingSessionError extends Error {
-    constructor(sessionId) {
-        super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
-    }
-}
-export class ContextPipelineInterruptedError extends Error {
-    constructor(message) {
-        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
-    }
-}
-export class ContextPipelineInitializationError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
-export class ContextPipelineCleanupError extends CriticalError {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
-export class RequestHandlerError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
 /**
  * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
  * that ran after the request's writes had already been committed.
@@ -145,9 +80,3 @@ export class ServiceConflictError extends Error {
             `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
     }
 }
-/**
- * Thrown by crawlers when `skipNavigation` is used on a request.
- * Subclasses can catch this error to skip their own navigation-dependent logic.
- */
-export class NavigationSkippedError extends NonRetryableError {
-}

package/events/index.d.ts

@@ -1,2 +1,3 @@
 export * from './event_manager.js';
 export * from './local_event_manager.js';
+export type * from './system_info.js';

package/events/system_info.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Represents the current status of the system.
+ */
+export interface SystemInfo {
+    /** If false, system is being overloaded. */
+    isSystemIdle: boolean;
+    memInfo: LoadSignalInfo;
+    eventLoopInfo: LoadSignalInfo;
+    cpuInfo: LoadSignalInfo;
+    storageBackendInfo: LoadSignalInfo;
+    memTotalBytes?: number;
+    memCurrentBytes?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    cpuCurrentUsage?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    isCpuOverloaded?: boolean;
+    /**
+     * Platform only property
+     * @internal
+     */
+    createdAt?: Date;
+    /**
+     * Status of additional load signals beyond the built-in four.
+     * Keys are `LoadSignal.name` values, values are overload info.
+     */
+    loadSignalInfo?: Record<string, LoadSignalInfo>;
+}
+export interface LoadSignalInfo {
+    isOverloaded: boolean;
+    limitRatio: number;
+    actualRatio: number;
+}

package/index.d.ts

@@ -1,22 +1,16 @@
 export * from './debug.js';
 export * from './errors.js';
-export * from './autoscaling/index.js';
 export * from './configuration.js';
 export * from './service_locator.js';
-export * from './crawlers/index.js';
-export * from './enqueue_links/index.js';
 export * from './events/index.js';
 export * from './log.js';
 export * from './owned_or_injected.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
-export * from './router.js';
 export * from './serialization.js';
-export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './memory-storage/index.js';
 export { ArgumentValidationError, validators } from './validators.js';
-export * from './cookie_utils.js';
-export * from './http.js';
 export * from './recoverable_state.js';
 export type { StorageBackend } from '@crawlee/types';
+export { EnqueueStrategy } from '@crawlee/utils';

package/index.js

@@ -1,23 +1,17 @@
 export * from './debug.js';
 export * from './errors.js';
-export * from './autoscaling/index.js';
 export * from './configuration.js';
 export * from './service_locator.js';
-export * from './crawlers/index.js';
-export * from './enqueue_links/index.js';
 export * from './events/index.js';
 export * from './log.js';
 export * from './owned_or_injected.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
-export * from './router.js';
 export * from './serialization.js';
-export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './memory-storage/index.js';
 // Not `export *`: the rest of the module re-exports `@crawlee/utils/internal` symbols, which carry no
 // semver guarantees and must not reach the public surface. Internal consumers import them directly.
 export { ArgumentValidationError, validators } from './validators.js';
-export * from './cookie_utils.js';
-export * from './http.js';
 export * from './recoverable_state.js';
+export { EnqueueStrategy } from '@crawlee/utils';

package/internal.d.ts

@@ -0,0 +1,8 @@
+export * from './iterables.js';
+export * from './storages/batched_adds.js';
+export { joinRequestSourceStatuses } from './storages/request_loader.js';
+export * from './system-info/cpu-info.js';
+export * from './system-info/memory-info.js';
+export * from './system-info/ps-tree.js';
+export * from './system-info/runtime.js';
+export * from './url.js';

package/internal.js

@@ -0,0 +1,9 @@
+// Not part of the public API: internals that other Crawlee packages need but that carry no semver guarantees.
+export * from './iterables.js';
+export * from './storages/batched_adds.js';
+export { joinRequestSourceStatuses } from './storages/request_loader.js';
+export * from './system-info/cpu-info.js';
+export * from './system-info/memory-info.js';
+export * from './system-info/ps-tree.js';
+export * from './system-info/runtime.js';
+export * from './url.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.170",
+    "version": "4.0.0-beta.171",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -8,6 +8,7 @@
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -52,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.170",
-        "@crawlee/http-client": "4.0.0-beta.170",
-        "@crawlee/types": "4.0.0-beta.170",
-        "@crawlee/utils": "4.0.0-beta.170",
+        "@crawlee/fs-storage": "4.0.0-beta.171",
+        "@crawlee/http-client": "4.0.0-beta.171",
+        "@crawlee/types": "4.0.0-beta.171",
+        "@crawlee/utils": "4.0.0-beta.171",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -63,10 +64,7 @@
         "csv-stringify": "^6.5.2",
         "json5": "^2.2.3",
         "mime-types": "^3.0.1",
-        "minimatch": "^10.0.1",
         "stream-json": "^1.9.1",
-        "tldts": "^7.0.6",
-        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0",
         "zod": "^4.5.4"
@@ -78,5 +76,5 @@
             }
         }
     },
-    "gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
+    "gitHead": "e4367913045aa04189285719c9bd6912591d34bc"
 }

package/request.d.ts

@@ -1,7 +1,9 @@
 import type { BinaryLike } from 'node:crypto';
 import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
-import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
-import type { SkippedRequestReason } from './enqueue_links/shared.js';
+import type { EnqueueStrategy } from '@crawlee/utils';
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
 export declare enum RequestState {
     UNPROCESSED = 0,
     BEFORE_NAV = 1,

package/storages/index.d.ts

@@ -9,6 +9,4 @@ export * from './storage_instance_manager.js';
 export * from './storage_stats.js';
 export * from './utils.js';
 export * from './transaction.js';
-export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
-export * from './throttling_request_manager.js';

package/storages/index.js

@@ -7,6 +7,4 @@ export * from './storage_instance_manager.js';
 export * from './storage_stats.js';
 export * from './utils.js';
 export * from './transaction.js';
-export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
-export * from './throttling_request_manager.js';

package/autoscaling/autoscaled_pool.d.ts

@@ -1,196 +0,0 @@
-import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency_system.js';
-import type { CrawleeLogger } from '../log.js';
-/**
- * The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
- * a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
- */
-export interface TaskLoopPredicates {
-    /**
-     * A function that indicates whether `runTaskFunction` should be called.
-     * This function is called every time there is free capacity for a new task and it should
-     * indicate whether it should start a new task or not by resolving to either `true` or `false`.
-     * Besides its obvious use, it is also useful for task throttling to save resources.
-     */
-    isTaskReadyFunction?: () => Promise<boolean>;
-    /**
-     * A function that is called only when there are no tasks to be processed.
-     * If it resolves to `true` then the run finishes. Being called only
-     * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
-     * keeps resolving to `true`, `isFinishedFunction()` will never be called.
-     */
-    isFinishedFunction?: () => Promise<boolean>;
-}
-export interface TaskLoopOptions extends TaskLoopPredicates {
-    /**
-     * How often the pool should check if a new task is ready, in seconds.
-     * @default 0.5
-     */
-    maybeRunIntervalSecs?: number;
-}
-/** @internal */
-export interface AutoscaledPoolOptions extends TaskLoopOptions {
-    /**
-     * The governor that decides whether there is free compute for one more task. Typically a
-     * {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
-     * multiple pools (and therefore multiple crawlers) to cap their *combined* concurrency against one budget.
-     *
-     * All concurrency/scaling/snapshotter configuration lives on the governor — the pool only owns the task loop and
-     * its cadence.
-     */
-    concurrencySystem: IConcurrencySystem;
-    /**
-     * Who this pool is, presented to the governor on every capacity query and booking so that a shared one can tell
-     * several pools apart. Worth naming meaningfully — a governor that allocates per consumer reports this `id`.
-     */
-    consumer: ConcurrencyConsumer;
-    /**
-     * A function that performs an asynchronous resource-intensive task.
-     * The function must either be labeled `async` or return a promise.
-     */
-    runTaskFunction?: () => Promise<unknown>;
-    /**
-     * Timeout in which the `runTaskFunction` needs to finish, given in seconds.
-     * @default 0
-     */
-    taskTimeoutSecs?: number;
-    log?: CrawleeLogger;
-}
-/**
- * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
- * The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
- * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
- *
- * Before running the pool, you need to implement the following three functions:
- * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
- *
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
- * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
- * resolves to `false`. The pool then queries `isFinishedFunction`. If it resolves to `true`, the run finishes after all
- * running tasks complete. If it resolves to `false`, it assumes there will be more tasks available later and keeps
- * periodically querying for tasks.
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
- *
- * The pool evaluates whether it should start a new task every time one of the tasks finishes
- * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
- *
- * **Example usage:**
- *
- * ```javascript
- * const concurrencySystem = new ConcurrencySystem({ maxConcurrency: 50 });
- * await concurrencySystem.start();
- *
- * const pool = new AutoscaledPool({
- *     concurrencySystem,
- *     consumer: { id: 'my-pool' },
- *     runTaskFunction: async () => {
- *         // Run some resource-intensive asynchronous operation here.
- *     },
- *     isTaskReadyFunction: async () => {
- *         // Tell the pool whether more tasks are ready to be processed.
- *         // Return true or false
- *     },
- *     isFinishedFunction: async () => {
- *         // Tell the pool whether it should finish
- *         // or wait for more tasks to become available.
- *         // Return true or false
- *     }
- * });
- *
- * try {
- *     await pool.run();
- * } finally {
- *     await concurrencySystem.stop();
- * }
- * ```
- *
- * @internal
- */
-export declare class AutoscaledPool {
-    #private;
-    constructor(options: AutoscaledPoolOptions);
-    /**
-     * The governor backing this pool, as supplied to the constructor — exposed as the read-only
-     * {@link IConcurrencySystem} contract.
-     *
-     * This and the two getters below are telemetry only: concurrency is configured and tuned on the concrete
-     * {@link ConcurrencySystem} its owner holds, never through the pool.
-     */
-    get system(): IConcurrencySystem;
-    /** The estimated number of parallel tasks the governor can currently support. */
-    get desiredConcurrency(): number;
-    /**
-     * The number of parallel tasks currently booked against the governor. When it is shared, this counts every
-     * borrowing pool's tasks, not just this one's.
-     */
-    get currentConcurrency(): number;
-    /**
-     * Runs the auto-scaled pool. Returns a promise that gets resolved or rejected once
-     * all the tasks are finished or one of them fails.
-     *
-     * Throws if the {@link IConcurrencySystem|concurrency system} it borrows was never started — the pool assumes
-     * a running governor and cannot start one it does not own.
-     */
-    run(): Promise<void>;
-    /**
-     * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
-     * will be spawned and all running tasks will be left in their current state.
-     *
-     * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
-     * of all the running tasks, therefore, no abortion is attempted and some of the tasks
-     * may finish, while others may not. Essentially, auto-scaled pool doesn't care about
-     * their state after the invocation of `.abort()`, but that does not mean that some
-     * parts of their asynchronous chains of commands will not execute.
-     */
-    abort(): Promise<void>;
-    /**
-     * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
-     *
-     * The function's promise will resolve once all running tasks have completed and the pool
-     * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
-     * with a timeout error after the `timeoutSecs` seconds.
-     *
-     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
-     * when `.pause()` is invoked (unlike abort, which resolves it).
-     *
-     * > *NOTE:* Pausing the pool does not suspend the (possibly shared) {@link ConcurrencySystem} — its
-     * autoscaling and resource monitoring keep running, since other pools borrowing it may still be active. To silence
-     * it during a long pause, its owner can `stop()` and `start()` it again.
-     */
-    pause(timeoutSecs?: number): Promise<void>;
-    /**
-     * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@link AutoscaledPool.pause}
-     *
-     * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
-     */
-    resume(): void;
-    /**
-     * Explicitly check the queue for new tasks. The AutoscaledPool checks the queue for new tasks periodically,
-     * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
-     */
-    notify(): Promise<void>;
-    /**
-     * Starts a new task
-     * if the number of running tasks (current concurrency) is lower than desired concurrency
-     * and the system is not currently overloaded
-     * and `isTaskReadyFunction()` returns true.
-     *
-     * It doesn't allow multiple concurrent runs of this method.
-     */
-    private maybeRunTask;
-    /**
-     * If there are no running tasks and `isFinishedFunction()` returns true then closes
-     * the pool and resolves the pool's promise returned by the run() method.
-     *
-     * It doesn't allow multiple concurrent runs of this method.
-     */
-    private maybeFinish;
-    /**
-     * Cleans up resources.
-     */
-    private destroy;
-}