@crawlee/core 4.0.0-beta.1 → 4.0.0-beta.100

package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts}

@@ -1,44 +1,33 @@
+import type { BaseHttpClient } from '@crawlee/types';
 import { type ParseSitemapOptions } from '@crawlee/utils';
-import { Configuration } from '../configuration.js';
-import type { GlobInput, RegExpInput } from '../enqueue_links/shared.js';
+import type { UrlPatternInput } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';
-import type { IRequestList } from './request_list.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { IRequestManager } from './request_manager.js';
 interface UrlConstraints {
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be included.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the `SitemapRequestList`
+     * If `include` is an empty array or `undefined`, then the `SitemapRequestLoader`
      * includes all the URLs from the sitemap.
      */
-    globs?: readonly GlobInput[];
+    include?: readonly UrlPatternInput[];
     /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be included.
+     * An array of URL patterns. Matching URLs will **not** be included.
      *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    exclude?: readonly (GlobInput | RegExp)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the `SitemapRequestList`
-     * includes all the URLs from the sitemap.
-     */
-    regexps?: readonly RegExpInput[];
+    exclude?: readonly UrlPatternInput[];
 }
-export interface SitemapRequestListOptions extends UrlConstraints {
+export interface SitemapRequestLoaderOptions extends UrlConstraints {
     /**
      * List of sitemap URLs to parse.
      */
@@ -81,27 +70,25 @@ export interface SitemapRequestListOptions extends UrlConstraints {
      */
     parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
     /**
-     * Crawlee configuration
+     * Custom HTTP client to be used for sitemap loading.
      */
-    config?: Configuration;
+    httpClient?: BaseHttpClient;
 }
 /**
  * A list of URLs to crawl parsed from a sitemap.
  *
  * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
  */
-export declare class SitemapRequestList implements IRequestList {
+export declare class SitemapRequestLoader implements IRequestLoader {
     /**
      * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
      * @internal
      */
     inProgress: Set<string>;
-    /** Set of URLs for which `reclaimRequest()` was called. */
-    private reclaimed;
     /**
      * Map of returned Request objects that have not been marked as handled yet.
      *
-     * We use this to persist custom user fields on the in-progress (or reclaimed) requests.
+     * We use this to persist custom user fields on the in-progress requests.
      */
     private requestData;
     /**
@@ -132,7 +119,7 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * Proxy URL to be used for sitemap loading.
      */
-    private proxyUrl;
+    private proxyUrl?;
     /**
      * Logger instance.
      */
@@ -157,7 +144,7 @@ export declare class SitemapRequestList implements IRequestList {
      */
     private matchesUrl;
     /**
-     * Checks whether the URL matches the `globs` / `regexps` / `exclude` provided in the `options`.
+     * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
      * @param url URL to be checked.
      * @returns `true` if the URL matches the patterns, `false` otherwise.
      */
@@ -190,15 +177,25 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * Open a sitemap and start processing it.
      *
-     * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
+     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
      *
      * Track the loading progress using the `isSitemapFullyLoaded` property.
      */
-    static open(options: SitemapRequestListOptions): Promise<SitemapRequestList>;
+    static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
+    /**
+     * @inheritDoc
+     */
+    getTotalCount(): Promise<number>;
     /**
      * @inheritDoc
      */
-    length(): number;
+    getPendingCount(): Promise<number>;
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
     /**
      * @inheritDoc
      */
@@ -210,7 +207,7 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    handledCount(): number;
+    getHandledCount(): Promise<number>;
     /**
      * @inheritDoc
      */
@@ -224,11 +221,7 @@ export declare class SitemapRequestList implements IRequestList {
      * @inheritDoc
      */
 // @ts-ignore optional peer dependency or compatibility with es2022
-    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/utils").Dictionary>, void, unknown>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(request: Request): Promise<void>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
     /**
      * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
      *
@@ -238,8 +231,7 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    markRequestHandled(request: Request): Promise<void>;
-    private ensureInProgressAndNotReclaimed;
+    markRequestAsHandled(request: Request): Promise<void>;
+    private ensureInProgress;
 }
 export {};
-//# sourceMappingURL=sitemap_request_list.d.ts.map

package/storages/{sitemap_request_list.js → sitemap_request_loader.js}

@@ -2,31 +2,28 @@ import { Transform } from 'node:stream';
 import { parseSitemap } from '@crawlee/utils';
 import { minimatch } from 'minimatch';
 import ow from 'ow';
-import defaultLog from '@apify/log';
-import { Configuration } from '../configuration.js';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links/shared.js';
+import { constructUrlPatternObjects } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
 /** @internal */
-const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LIST_STATE';
+const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
 /**
  * A list of URLs to crawl parsed from a sitemap.
  *
  * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
  */
-export class SitemapRequestList {
+export class SitemapRequestLoader {
     /**
      * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
      * @internal
      */
     inProgress = new Set();
-    /** Set of URLs for which `reclaimRequest()` was called. */
-    reclaimed = new Set();
     /**
      * Map of returned Request objects that have not been marked as handled yet.
      *
-     * We use this to persist custom user fields on the in-progress (or reclaimed) requests.
+     * We use this to persist custom user fields on the in-progress requests.
      */
     requestData = new Map();
     /**
@@ -74,7 +71,7 @@ export class SitemapRequestList {
     /**
      * Logger instance.
      */
-    log = defaultLog.child({ prefix: 'SitemapRequestList' });
+    log;
     urlExcludePatternObjects = [];
     urlPatternObjects = [];
     /** EventManager used to handle persistence */
@@ -82,6 +79,7 @@ export class SitemapRequestList {
     persistenceOptions;
     /** @internal */
     constructor(options) {
+        const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
         ow(options, ow.object.exactShape({
             sitemapUrls: ow.array.ofType(ow.string),
             proxyUrl: ow.optional.string,
@@ -90,35 +88,25 @@ export class SitemapRequestList {
             timeoutMillis: ow.optional.number,
             maxBufferSize: ow.optional.number,
             parseSitemapOptions: ow.optional.object,
-            globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-            exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
-            regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-            config: ow.optional.object,
+            include: ow.optional.array.ofType(urlPatternValidator),
+            exclude: ow.optional.array.ofType(urlPatternValidator),
             persistenceOptions: ow.optional.object,
+            httpClient: ow.optional.object,
         }));
-        const { globs, exclude, regexps, config = Configuration.getGlobalConfig() } = options;
+        const { include, exclude } = options;
+        this.log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
         if (exclude?.length) {
-            for (const excl of exclude) {
-                if (typeof excl === 'string' || 'glob' in excl) {
-                    this.urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-                }
-                else if (excl instanceof RegExp || 'regexp' in excl) {
-                    this.urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-                }
-            }
+            this.urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
         }
-        if (globs?.length) {
-            this.urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-        }
-        if (regexps?.length) {
-            this.urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
+        if (include?.length) {
+            this.urlPatternObjects.push(...constructUrlPatternObjects(include));
         }
         this.persistStateKey = options.persistStateKey;
         this.persistenceOptions = { enable: true, ...options.persistenceOptions };
         this.proxyUrl = options.proxyUrl;
         this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
         this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
-        this.events = config.getEventManager();
+        this.events = serviceLocator.getEventManager();
         this.persistState = this.persistState.bind(this);
     }
     /**
@@ -146,7 +134,7 @@ export class SitemapRequestList {
         };
     }
     /**
-     * Checks whether the URL matches the `globs` / `regexps` / `exclude` provided in the `options`.
+     * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
      * @param url URL to be checked.
      * @returns `true` if the URL matches the patterns, `false` otherwise.
      */
@@ -248,17 +236,20 @@ export class SitemapRequestList {
     /**
      * Open a sitemap and start processing it.
      *
-     * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
+     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
      *
      * Track the loading progress using the `isSitemapFullyLoaded` property.
      */
     static async open(options) {
-        const requestList = new SitemapRequestList({
-            ...options,
+        const { httpClient, ...restOptions } = options;
+        const requestList = new SitemapRequestLoader({
+            ...restOptions,
             persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
         });
         await requestList.restoreState();
-        void requestList.load({ parseSitemapOptions: options.parseSitemapOptions });
+        void requestList.load({
+            parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
+        });
         if (requestList.persistenceOptions.enable) {
             requestList.events.on("persistState" /* EventType.PERSIST_STATE */, requestList.persistState);
         }
@@ -275,8 +266,28 @@ export class SitemapRequestList {
     /**
      * @inheritDoc
      */
-    length() {
-        return this.urlQueueStream.readableLength + this.handledUrlCount - this.inProgress.size - this.reclaimed.size;
+    async getTotalCount() {
+        // Total known so far = not-yet-fetched (still buffered in the stream) + in-progress (fetched but not
+        // yet handled) + already handled.
+        return this.urlQueueStream.readableLength + this.inProgress.size + this.handledUrlCount;
+    }
+    /**
+     * @inheritDoc
+     */
+    async getPendingCount() {
+        // Pending = everything not yet handled = not-yet-fetched + in-progress.
+        return this.urlQueueStream.readableLength + this.inProgress.size;
+    }
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    async toTandem(requestManager) {
+        // Import here to avoid circular imports.
+        const { RequestManagerTandem } = await import('./request_manager_tandem.js');
+        const { RequestQueue } = await import('./request_queue.js');
+        return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
     }
     /**
      * @inheritDoc
@@ -288,12 +299,12 @@ export class SitemapRequestList {
      * @inheritDoc
      */
     async isEmpty() {
-        return this.reclaimed.size === 0 && this.urlQueueStream.readableLength === 0;
+        return this.urlQueueStream.readableLength === 0;
     }
     /**
      * @inheritDoc
      */
-    handledCount() {
+    async getHandledCount() {
         return this.handledUrlCount;
     }
     /**
@@ -328,8 +339,8 @@ export class SitemapRequestList {
                 inProgressSitemapUrl: this.sitemapParsingProgress.inProgressSitemapUrl,
                 inProgressEntries: Array.from(this.sitemapParsingProgress.inProgressEntries),
             },
-            urlQueue,
-            reclaimed: [...this.inProgress, ...this.reclaimed], // In-progress and reclaimed requests will be both retried if state is restored
+            // Re-queue in-progress requests to the front so they are retried if the state is restored.
+            urlQueue: [...this.inProgress, ...urlQueue],
             requestData: Array.from(this.requestData.entries()),
             abortLoading: this.abortLoading,
             closed: this.closed,
@@ -345,7 +356,6 @@ export class SitemapRequestList {
         if (state === null) {
             return;
         }
-        this.reclaimed = new Set(state.reclaimed);
         this.sitemapParsingProgress = {
             pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
             inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
@@ -362,17 +372,12 @@ export class SitemapRequestList {
      * @inheritDoc
      */
     async fetchNextRequest() {
-        // Try to return a reclaimed request first
-        let nextUrl = this.reclaimed.values().next().value;
-        if (nextUrl) {
-            this.reclaimed.delete(nextUrl);
+        const nextUrl = await this.readNextUrl();
+        if (!nextUrl) {
+            return null;
         }
-        else {
-            // Otherwise read next url from the stream
-            nextUrl = await this.readNextUrl();
-            if (!nextUrl) {
-                return null;
-            }
+        // A restored in-progress request already has its Request data; don't overwrite it.
+        if (!this.requestData.has(nextUrl)) {
             this.requestData.set(nextUrl, new Request({ url: nextUrl }));
         }
         this.inProgress.add(nextUrl);
@@ -389,14 +394,6 @@ export class SitemapRequestList {
             yield request;
         }
     }
-    /**
-     * @inheritDoc
-     */
-    async reclaimRequest(request) {
-        this.ensureInProgressAndNotReclaimed(request.url);
-        this.reclaimed.add(request.url);
-        this.inProgress.delete(request.url);
-    }
     /**
      * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
      *
@@ -412,19 +409,15 @@ export class SitemapRequestList {
     /**
      * @inheritDoc
      */
-    async markRequestHandled(request) {
+    async markRequestAsHandled(request) {
         this.handledUrlCount += 1;
-        this.ensureInProgressAndNotReclaimed(request.url);
+        this.ensureInProgress(request.url);
         this.inProgress.delete(request.url);
         this.requestData.delete(request.url);
     }
-    ensureInProgressAndNotReclaimed(url) {
+    ensureInProgress(url) {
         if (!this.inProgress.has(url)) {
             throw new Error(`The request is not being processed (url: ${url})`);
         }
-        if (this.reclaimed.has(url)) {
-            throw new Error(`The request was already reclaimed (url: ${url})`);
-        }
     }
 }
-//# sourceMappingURL=sitemap_request_list.js.map

package/storages/storage_instance_manager.d.ts

@@ -0,0 +1,88 @@
+import type { Constructor, DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
+export type { StorageIdentifier } from '@crawlee/types';
+/**
+ * Matches an `IStorage` – a storage "frontend" (Dataset, KeyValueStore, RequestQueue).
+ */
+export interface IStorage {
+    id: string;
+    name?: string;
+}
+type Hashable = string;
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ */
+export declare class StorageInstanceManager {
+    private readonly cache;
+    private readonly openerLocks;
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param backendOpener      A **lazy** factory that creates the sub-backend.
+     *                          Only called on a cache miss.
+     * @param backendCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, backendOpener, backendCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
+        backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
+        backendCacheKey: Hashable;
+    }): Promise<TStorage>;
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance: IStorage): void;
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache(): void;
+}
+/**
+ * A storage identifier where exactly one of `id`, `name`, or `alias` is specified.
+ * Produced by {@link resolveStorageIdentifier} from ambiguous user input.
+ */
+export type ExplicitStorageIdentifier = {
+    id: string;
+    name?: never;
+    alias?: never;
+} | {
+    id?: never;
+    name: string;
+    alias?: never;
+} | {
+    id?: never;
+    name?: never;
+    alias: string;
+};
+/**
+ * Represents the case where no identifier was provided — the caller wants the default storage.
+ */
+export interface DefaultStorageIdentifier {
+    id?: never;
+    name?: never;
+    alias?: never;
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
+ */
+export declare function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, storageBackend: StorageBackend, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;