@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111

package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts}

@@ -1,44 +1,33 @@
+import type { BaseHttpClient } from '@crawlee/types';
 import { type ParseSitemapOptions } from '@crawlee/utils';
-import { Configuration } from '../configuration.js';
-import type { GlobInput, RegExpInput } from '../enqueue_links/shared.js';
+import type { UrlPatternInput } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';
-import type { IRequestList } from './request_list.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { IRequestManager } from './request_manager.js';
 interface UrlConstraints {
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be included.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the `SitemapRequestList`
+     * If `include` is an empty array or `undefined`, then the `SitemapRequestLoader`
      * includes all the URLs from the sitemap.
      */
-    globs?: readonly GlobInput[];
+    include?: readonly UrlPatternInput[];
     /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be included.
+     * An array of URL patterns. Matching URLs will **not** be included.
      *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExp)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the `SitemapRequestList`
-     * includes all the URLs from the sitemap.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    regexps?: readonly RegExpInput[];
+    exclude?: readonly UrlPatternInput[];
 }
-export interface SitemapRequestListOptions extends UrlConstraints {
+export interface SitemapRequestLoaderOptions extends UrlConstraints {
     /**
      * List of sitemap URLs to parse.
      */
@@ -81,67 +70,22 @@ export interface SitemapRequestListOptions extends UrlConstraints {
      */
     parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
     /**
-     * Crawlee configuration
+     * Custom HTTP client to be used for sitemap loading.
      */
-    config?: Configuration;
+    httpClient?: BaseHttpClient;
 }
 /**
  * A list of URLs to crawl parsed from a sitemap.
  *
  * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
  */
-export declare class SitemapRequestList implements IRequestList {
+export declare class SitemapRequestLoader implements IRequestLoader {
+    #private;
     /**
      * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
      * @internal
      */
     inProgress: Set<string>;
-    /** Set of URLs for which `reclaimRequest()` was called. */
-    private reclaimed;
-    /**
-     * Map of returned Request objects that have not been marked as handled yet.
-     *
-     * We use this to persist custom user fields on the in-progress (or reclaimed) requests.
-     */
-    private requestData;
-    /**
-     * Object for keeping track of the sitemap parsing progress.
-     */
-    private sitemapParsingProgress;
-    /**
-     * Object stream of URLs parsed from the sitemaps.
-     * Using `highWaterMark`, this can manage the speed of the sitemap loading.
-     *
-     * Fetch the next URL to be processed using `fetchNextRequest()`.
-     */
-    private urlQueueStream;
-    /**
-     * Indicates whether the request list sitemap loading was aborted.
-     *
-     * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
-     * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
-     *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
-     */
-    private abortLoading;
-    /** Number of URLs that were marked as handled */
-    private handledUrlCount;
-    private persistStateKey?;
-    private store?;
-    private closed;
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    private proxyUrl;
-    /**
-     * Logger instance.
-     */
-    private log;
-    private urlExcludePatternObjects;
-    private urlPatternObjects;
-    /** EventManager used to handle persistence */
-    private events;
-    private persistenceOptions;
     /** @internal */
     private constructor();
     /**
@@ -157,7 +101,7 @@ export declare class SitemapRequestList implements IRequestList {
      */
     private matchesUrl;
     /**
-     * Checks whether the URL matches the `globs` / `regexps` / `exclude` provided in the `options`.
+     * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
      * @param url URL to be checked.
      * @returns `true` if the URL matches the patterns, `false` otherwise.
      */
@@ -190,15 +134,25 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * Open a sitemap and start processing it.
      *
-     * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
+     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
      *
      * Track the loading progress using the `isSitemapFullyLoaded` property.
      */
-    static open(options: SitemapRequestListOptions): Promise<SitemapRequestList>;
+    static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
     /**
      * @inheritDoc
      */
-    length(): number;
+    getTotalCount(): Promise<number>;
+    /**
+     * @inheritDoc
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
     /**
      * @inheritDoc
      */
@@ -210,7 +164,7 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    handledCount(): number;
+    getHandledCount(): Promise<number>;
     /**
      * @inheritDoc
      */
@@ -224,11 +178,7 @@ export declare class SitemapRequestList implements IRequestList {
      * @inheritDoc
      */
 // @ts-ignore optional peer dependency or compatibility with es2022
-    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/utils").Dictionary>, void, unknown>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(request: Request): Promise<void>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
     /**
      * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
      *
@@ -238,8 +188,7 @@ export declare class SitemapRequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    markRequestHandled(request: Request): Promise<void>;
-    private ensureInProgressAndNotReclaimed;
+    markRequestAsHandled(request: Request): Promise<void>;
+    private ensureInProgress;
 }
 export {};
-//# sourceMappingURL=sitemap_request_list.d.ts.map