@crawlee/core 4.0.0-beta.122 → 4.0.0-beta.124

package/autoscaling/concurrency_system.d.ts

@@ -53,8 +53,8 @@ export interface ConcurrencySystemOptions {
     autoscaleIntervalSecs?: number;
     /**
      * The signals that tell the system whether the machine is overloaded: per-resource tuning for the built-in four
-     * (memory, event loop, CPU, client) plus any {@link LoadSignalsOptions.custom|`custom`} implementations of
-     * your own. See {@link LoadSignalsOptions}.
+     * (memory, event loop, CPU, storage backend) plus any {@link LoadSignalsOptions.custom|`custom`}
+     * implementations of your own. See {@link LoadSignalsOptions}.
      */
     loadSignals?: LoadSignalsOptions;
     /**

package/autoscaling/index.d.ts

@@ -1,9 +1,9 @@
 export * from './autoscaled_pool.js';
 export * from './concurrency_system.js';
-export * from './client_load_signal.js';
 export * from './cpu_load_signal.js';
 export * from './event_loop_load_signal.js';
 export * from './load_signal.js';
 export * from './memory_load_signal.js';
 export * from './snapshotter.js';
+export * from './storage_backend_load_signal.js';
 export * from './system_status.js';

package/autoscaling/index.js

@@ -1,9 +1,9 @@
 export * from './autoscaled_pool.js';
 export * from './concurrency_system.js';
-export * from './client_load_signal.js';
 export * from './cpu_load_signal.js';
 export * from './event_loop_load_signal.js';
 export * from './load_signal.js';
 export * from './memory_load_signal.js';
 export * from './snapshotter.js';
+export * from './storage_backend_load_signal.js';
 export * from './system_status.js';

package/autoscaling/load_signal.d.ts

@@ -1,4 +1,4 @@
-import type { ClientInfo } from './system_status.js';
+import type { LoadSignalInfo } from './system_status.js';
 /**
  * A snapshot of a resource's overload state at a point in time.
  */
@@ -22,7 +22,7 @@ export interface LoadSignalStartContext {
  * A signal that reports whether a particular resource is overloaded. The {@link ConcurrencySystem} aggregates
  * several of them — if any one reports overload, the system is overloaded.
  *
- * The built-in signals cover memory, CPU, event loop and storage-client rate limits. Implement this interface to add
+ * The built-in signals cover memory, CPU, event loop and storage backend rate limits. Implement this interface to add
  * your own (navigation timeouts, proxy health, …) and pass them via
  * {@link LoadSignalsOptions.custom|`loadSignals.custom`}; {@link SnapshotStore} does the time-windowed
  * bookkeeping if you want it. Each built-in is also a public class, so one can be *wrapped* rather than reimplemented
@@ -32,8 +32,8 @@ export interface LoadSignal {
     /**
      * This signal's key in the reported {@link SystemInfo}, also used in logging — so it must be unique among the
      * signals of one {@link ConcurrencySystem}, which throws on a duplicate. The four built-in names (`memInfo`,
-     * `eventLoopInfo`, `cpuInfo`, `clientInfo`) land in the correspondingly named `SystemInfo` fields rather than the
-     * `loadSignalInfo` bag; taking one over means switching that built-in off.
+     * `eventLoopInfo`, `cpuInfo`, `storageBackendInfo`) land in the correspondingly named `SystemInfo` fields rather
+     * than the `loadSignalInfo` bag; taking one over means switching that built-in off.
      */
     readonly name: string;
     /**
@@ -79,7 +79,8 @@ export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
     getSample(sampleDurationMillis?: number): T[];
     /**
      * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
-     * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
+     * to compute a delta (e.g. the event loop and storage backend signals read the last entry to measure change since
+     * it).
      */
     getAll(): T[];
     /**
@@ -96,4 +97,4 @@ export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
  * evaluation logic used by `SystemStatus` for all signal types.
  * @internal
  */
-export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;
+export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): LoadSignalInfo;

package/autoscaling/load_signal.js

@@ -57,7 +57,8 @@ export class SnapshotStore {
     }
     /**
      * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
-     * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
+     * to compute a delta (e.g. the event loop and storage backend signals read the last entry to measure change since
+     * it).
      */
     getAll() {
         return this.#snapshots;

package/autoscaling/snapshotter.d.ts

@@ -1,8 +1,8 @@
-import type { ClientLoadSignalOptions } from './client_load_signal.js';
 import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
 import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
 import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
 import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
+import type { StorageBackendLoadSignalOptions } from './storage_backend_load_signal.js';
 /**
  * The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
  *
@@ -31,11 +31,11 @@ export interface LoadSignalsOptions {
      */
     cpu?: CpuLoadSignalOptions | false;
     /**
-     * Tuning for the built-in {@link ClientLoadSignal} (snapshot interval + error limit + overload ratio), or
-     * `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
+     * Tuning for the built-in {@link StorageBackendLoadSignal} (snapshot interval + error limit + overload ratio),
+     * or `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
      * signal otherwise polls it every second to no purpose.
      */
-    client?: ClientLoadSignalOptions | false;
+    storageBackend?: StorageBackendLoadSignalOptions | false;
     /**
      * Additional {@link LoadSignal} implementations — e.g. navigation timeouts or proxy health — evaluated
      * alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
@@ -53,8 +53,8 @@ export interface LoadSignalsOptions {
 export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
 /**
  * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
- * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
- * that were not switched off and driving their shared lifecycle.
+ * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
+ * the ones that were not switched off and driving their shared lifecycle.
  *
  * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
  * are simply forwarded to the corresponding constructor.

package/autoscaling/snapshotter.js

@@ -1,22 +1,22 @@
-import { ClientLoadSignal } from './client_load_signal.js';
 import { CpuLoadSignal } from './cpu_load_signal.js';
 import { EventLoopLoadSignal } from './event_loop_load_signal.js';
 import { MemoryLoadSignal } from './memory_load_signal.js';
+import { StorageBackendLoadSignal } from './storage_backend_load_signal.js';
 /**
  * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
- * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
- * that were not switched off and driving their shared lifecycle.
+ * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
+ * the ones that were not switched off and driving their shared lifecycle.
  *
  * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
  * are simply forwarded to the corresponding constructor.
  * @internal
  */
 export class Snapshotter {
-    // Absent when switched off through the corresponding option (e.g. `client: false`).
+    // Absent when switched off through the corresponding option (e.g. `storageBackend: false`).
     #memorySignal;
     #eventLoopSignal;
     #cpuSignal;
-    #clientSignal;
+    #storageBackendSignal;
     /**
      * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
      * instances. Signals switched off through the options are simply absent — the system status reports them as
@@ -27,7 +27,7 @@ export class Snapshotter {
             this.#memorySignal,
             this.#eventLoopSignal,
             this.#cpuSignal,
-            this.#clientSignal,
+            this.#storageBackendSignal,
         ];
         return builtin.filter((signal) => signal !== undefined);
     }
@@ -35,7 +35,7 @@ export class Snapshotter {
      * @param [options] All `Snapshotter` configuration options.
      */
     constructor(options = {}) {
-        const { memory = {}, eventLoop = {}, cpu = {}, client = {} } = options;
+        const { memory = {}, eventLoop = {}, cpu = {}, storageBackend = {} } = options;
         // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
         // over then too - so there is nothing to thread in here beyond the caller's tuning.
         if (memory !== false)
@@ -44,8 +44,8 @@ export class Snapshotter {
             this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);
         if (cpu !== false)
             this.#cpuSignal = new CpuLoadSignal(cpu);
-        if (client !== false)
-            this.#clientSignal = new ClientLoadSignal(client);
+        if (storageBackend !== false)
+            this.#storageBackendSignal = new StorageBackendLoadSignal(storageBackend);
     }
     /**
      * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will

package/autoscaling/{client_load_signal.d.ts → storage_backend_load_signal.d.ts}

@@ -1,18 +1,19 @@
 import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
 /**
- * A snapshot produced by the built-in client (rate-limit) signal.
+ * A snapshot produced by the built-in storage backend (rate-limit) signal.
  * @internal
  */
-export interface ClientSnapshot extends LoadSnapshot {
+export interface StorageBackendSnapshot extends LoadSnapshot {
     rateLimitErrorCount: number;
 }
 /**
- * Tuning for the built-in **client** (rate-limit) load signal, as accepted both by {@link ClientLoadSignal} and by
- * the {@link LoadSignalsOptions.client|`client`} shorthand on {@link LoadSignalsOptions}.
+ * Tuning for the built-in **storage backend** (rate-limit) load signal, as accepted both by
+ * {@link StorageBackendLoadSignal} and by the
+ * {@link LoadSignalsOptions.storageBackend|`storageBackend`} shorthand on {@link LoadSignalsOptions}.
  */
-export interface ClientLoadSignalOptions {
+export interface StorageBackendLoadSignalOptions {
     /**
-     * Defines the interval of checking the current state of the remote API client, in seconds.
+     * Defines the interval of checking the current state of the storage backend, in seconds.
      * @default 1
      */
     snapshotIntervalSecs?: number;
@@ -22,7 +23,7 @@ export interface ClientLoadSignalOptions {
      */
     maxErrors?: number;
     /**
-     * Maximum ratio of overloaded snapshots in a sample before the client counts as overloaded.
+     * Maximum ratio of overloaded snapshots in a sample before the storage backend counts as overloaded.
      * @default 0.3
      */
     overloadedRatio?: number;
@@ -33,16 +34,16 @@ export interface ClientLoadSignalOptions {
  *
  * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
  *
- * Switch it off entirely ({@link LoadSignalsOptions.client|`client: false`}) if the storage backend reports no
- * rate-limit statistics, since it otherwise polls it every second to no purpose.
+ * Switch it off entirely ({@link LoadSignalsOptions.storageBackend|`storageBackend: false`}) if the storage backend
+ * reports no rate-limit statistics, since it otherwise polls it every second to no purpose.
  *
  * @category Scaling
  */
-export declare class ClientLoadSignal implements LoadSignal {
+export declare class StorageBackendLoadSignal implements LoadSignal {
     #private;
-    readonly name = "clientInfo";
+    readonly name = "storageBackendInfo";
     readonly overloadedRatio: number;
-    constructor(options?: ClientLoadSignalOptions);
+    constructor(options?: StorageBackendLoadSignalOptions);
     start(context: LoadSignalStartContext): Promise<void>;
     stop(): Promise<void>;
     getSample(sampleDurationMillis?: number): LoadSnapshot[];

package/autoscaling/{client_load_signal.js → storage_backend_load_signal.js}

@@ -1,26 +1,26 @@
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
 import { serviceLocator } from '../service_locator.js';
 import { SnapshotStore } from './load_signal.js';
-const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
+const RATE_LIMIT_ERROR_RETRY_COUNT = 2;
 /**
  * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
  * exceeds a threshold.
  *
  * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
  *
- * Switch it off entirely ({@link LoadSignalsOptions.client|`client: false`}) if the storage backend reports no
- * rate-limit statistics, since it otherwise polls it every second to no purpose.
+ * Switch it off entirely ({@link LoadSignalsOptions.storageBackend|`storageBackend: false`}) if the storage backend
+ * reports no rate-limit statistics, since it otherwise polls it every second to no purpose.
  *
  * @category Scaling
  */
-export class ClientLoadSignal {
-    name = 'clientInfo';
+export class StorageBackendLoadSignal {
+    name = 'storageBackendInfo';
     overloadedRatio;
     #store = new SnapshotStore();
     #intervalMillis;
     #maxErrors;
     #interval;
-    #client;
+    #storageBackend;
     constructor(options = {}) {
         this.overloadedRatio = options.overloadedRatio ?? 0.3;
         this.#intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;
@@ -30,18 +30,18 @@ export class ClientLoadSignal {
     async start(context) {
         this.#store.useSampleWindow(context.maxSampleWindowMillis);
         // A new session starts from a clean slate, or its first measurement diffs the error count against the previous
-        // session's — possibly against a different backend, since the client is resolved afresh just below.
+        // session's — possibly against a different backend, since it is resolved afresh just below.
         this.#store.clear();
         // Resolved here rather than in the constructor, where asking for the backend would instantiate a default one
         // as a side effect - long before the crawler that owns the run has had a chance to register its own.
-        this.#client = serviceLocator.getStorageBackend();
+        this.#storageBackend = serviceLocator.getStorageBackend();
         this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
     }
     async stop() {
         if (this.#interval)
             betterClearInterval(this.#interval);
         this.#interval = undefined;
-        this.#client = undefined;
+        this.#storageBackend = undefined;
     }
     getSample(sampleDurationMillis) {
         return this.#store.getSample(sampleDurationMillis);
@@ -53,8 +53,8 @@ export class ClientLoadSignal {
      */
     handle(intervalCallback) {
         const now = new Date();
-        const allErrorCounts = this.#client?.stats?.rateLimitErrors ?? [];
-        const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
+        const allErrorCounts = this.#storageBackend?.stats?.rateLimitErrors ?? [];
+        const currentErrCount = allErrorCounts[RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
         const snapshot = {
             createdAt: now,
             isOverloaded: false,

package/autoscaling/system_status.d.ts

@@ -6,10 +6,10 @@ import type { Snapshotter } from './snapshotter.js';
 export interface SystemInfo {
     /** If false, system is being overloaded. */
     isSystemIdle: boolean;
-    memInfo: ClientInfo;
-    eventLoopInfo: ClientInfo;
-    cpuInfo: ClientInfo;
-    clientInfo: ClientInfo;
+    memInfo: LoadSignalInfo;
+    eventLoopInfo: LoadSignalInfo;
+    cpuInfo: LoadSignalInfo;
+    storageBackendInfo: LoadSignalInfo;
     memTotalBytes?: number;
     memCurrentBytes?: number;
     /**
@@ -31,7 +31,7 @@ export interface SystemInfo {
      * Status of additional load signals beyond the built-in four.
      * Keys are `LoadSignal.name` values, values are overload info.
      */
-    loadSignalInfo?: Record<string, ClientInfo>;
+    loadSignalInfo?: Record<string, LoadSignalInfo>;
 }
 /**
  * How far back the *current* system status looks by default — the window that gates task dispatch.
@@ -69,12 +69,12 @@ export interface SystemStatusOptions {
     /**
      * Additional load signals to include in the system status evaluation.
      * These are evaluated alongside the built-in memory, CPU, event loop,
-     * and client signals. If any signal reports overload, the system is
-     * considered overloaded. Each signal carries its own overload ratio.
+     * and storage backend signals. If any signal reports overload, the system
+     * is considered overloaded. Each signal carries its own overload ratio.
      */
     loadSignals?: LoadSignal[];
 }
-export interface ClientInfo {
+export interface LoadSignalInfo {
     isOverloaded: boolean;
     limitRatio: number;
     actualRatio: number;

package/autoscaling/system_status.js

@@ -15,7 +15,7 @@ const BUILTIN_SIGNAL_OPTION_KEYS = {
     memInfo: 'memory',
     eventLoopInfo: 'eventLoop',
     cpuInfo: 'cpu',
-    clientInfo: 'client',
+    storageBackendInfo: 'storageBackend',
 };
 const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
 /**
@@ -114,7 +114,7 @@ export class SystemStatus {
             memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
             eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
             cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
-            clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            storageBackendInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
         };
         let loadSignalInfo;
         for (const signal of this.#signals) {

package/crawlers/crawler_commons.d.ts

@@ -1,12 +1,12 @@
 import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
+import type { ReadonlyDeep } from 'type-fest';
+import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
 import type { Request, RequestOptions, Source } from '../request.js';
 import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 import type { Dataset } from '../storages/dataset.js';
 import type { KeyValueStore } from '../storages/key_value_store.js';
-import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { AddRequestsBatchedResult } from '../storages/request_queue.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
 /**
@@ -37,7 +37,7 @@ export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
  * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
  * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
  */
-export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
 /**
  * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
  * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
@@ -87,36 +87,15 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      */
     pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
+     * Add requests directly to the request queue currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
-     *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       include: [
-     *           'https://www.example.com/handbags/*',
-     *       ],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     */
-    enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
-    /**
-     * Add requests directly to the request queue.
+     * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
+     * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
      *
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
     /**
      * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
      */
@@ -131,31 +110,6 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     log: CrawleeLogger;
 }
 export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
-    /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
-     *
-     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
-     *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       include: [
-     *           'https://www.example.com/handbags/*',
-     *       ],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
-     */
-    enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
     /**
      * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *

package/enqueue_links/enqueue_links.d.ts

@@ -1,19 +1,24 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
-import type { SetRequired } from 'type-fest';
-import { Request } from '../request.js';
-import type { IRequestManager } from '../storages/request_manager.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationOptions } from '../storages/request_queue.js';
-import type { RequestTransform, SkippedRequestCallback, UrlPatternInput } from './shared.js';
-export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
+import type { Dictionary } from '@crawlee/types';
+import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
+/**
+ * Options shared by the `extractLinks()` context helper across crawler types.
+ */
+export interface ExtractLinksOptions {
+    /** A CSS selector matching links to be extracted. */
+    selector?: string;
+    /**
+     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+     * since the relative URL resolution is done inside the browser automatically.
+     */
+    baseUrl?: string;
+}
+/**
+ * Options accepted by the `enqueueUrls()` context helper exposed by `BasicCrawler`.
+ */
+export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
-    /** An array of URLs to enqueue. */
-    urls?: readonly string[];
-    /** A request manager to which the URLs will be enqueued. */
-    requestManager?: IRequestManager;
-    /** A CSS selector matching links to be enqueued. */
-    selector?: string;
     /** Sets {@link Request.userData} for newly enqueued requests. */
     userData?: Dictionary;
     /**
@@ -30,8 +35,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     skipNavigation?: boolean;
     /**
-     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
-     * since the relative URL resolution is done inside the browser automatically.
+     * A base URL that will be used to resolve relative URLs.
      */
     baseUrl?: string;
     /**
@@ -42,11 +46,11 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * Glob matching is always case-insensitive.
      * If you need case-sensitive matching, use a `RegExp`.
      *
-     * The patterns are combined with the {@link EnqueueLinksOptions.strategy|`strategy`} using AND logic - a URL
+     * The patterns are combined with the {@link EnqueueUrlsOptions.strategy|`strategy`} using AND logic - a URL
      * must match at least one `include` pattern **and** satisfy the strategy to be enqueued. To match URLs across
      * hostnames, pass an explicit {@link EnqueueStrategy.All} strategy.
      *
-     * If `undefined`, the links are enqueued based on the {@link EnqueueLinksOptions.strategy|`strategy`} alone.
+     * If `undefined`, the links are enqueued based on the {@link EnqueueUrlsOptions.strategy|`strategy`} alone.
      * Passing an empty array is not allowed.
      */
     include?: readonly UrlPatternInput[];
@@ -106,25 +110,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      *
      * @default EnqueueStrategy.SameHostname
      */
-    strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
+    strategy?: EnqueueStrategyOption;
     /**
      * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
      * You can use this option to wait for adding all of them.
      */
     waitForAllRequestsToBeAdded?: boolean;
-    /**
-     * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
-     * If provided, disallowed URLs will be ignored.
-     */
-    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
-    /**
-     * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
-     * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
-     * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
-     */
-    respectRobotsTxtFile?: boolean | {
-        userAgent?: string;
-    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
      * This is currently fired for requests skipped
@@ -134,6 +125,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
+/** The combined options accepted by a crawler context's `enqueueLinks()` helper: `extractLinks()` + `enqueueUrls()`. */
+export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
 /**
  * The different enqueueing strategies available.
  *
@@ -185,34 +178,8 @@ export declare enum EnqueueStrategy {
      */
     SameOrigin = "same-origin"
 }
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestManager,
- *   selector: 'a.product-detail',
- *   include: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
-    requestManager: {
-        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
-    };
-}): Promise<BatchAddRequestsResult>;
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -227,7 +194,12 @@ export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy
  */
 export interface ResolveBaseUrl {
     userProvidedBaseUrl?: string;
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];