@crawlee/core 4.0.0-beta.18 → 4.0.0-beta.180

package/storages/sitemap_request_list.d.ts

@@ -1,245 +0,0 @@
-import { type ParseSitemapOptions } from '@crawlee/utils';
-import { Configuration } from '../configuration.js';
-import type { GlobInput, RegExpInput } from '../enqueue_links/shared.js';
-import { Request } from '../request.js';
-import type { IRequestList } from './request_list.js';
-interface UrlConstraints {
-    /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the `SitemapRequestList`
-     * includes all the URLs from the sitemap.
-     */
-    globs?: readonly GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be included.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
-     *
-     * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExp)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the `SitemapRequestList`
-     * includes all the URLs from the sitemap.
-     */
-    regexps?: readonly RegExpInput[];
-}
-export interface SitemapRequestListOptions extends UrlConstraints {
-    /**
-     * List of sitemap URLs to parse.
-     */
-    sitemapUrls: string[];
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    proxyUrl?: string;
-    /**
-     * Key for persisting the state of the request list in the `KeyValueStore`.
-     */
-    persistStateKey?: string;
-    /**
-     * Persistence-related options to control how and when crawler's data gets persisted.
-     */
-    persistenceOptions?: {
-        /**
-         * Use this flag to disable or enable periodic persistence to key value store.
-         * @default true
-         */
-        enable?: boolean;
-    };
-    /**
-     * Abort signal to be used for sitemap loading.
-     */
-    signal?: AbortSignal;
-    /**
-     * Timeout for sitemap loading in milliseconds. If both `signal` and `timeoutMillis` are provided, either of them can abort the loading.
-     */
-    timeoutMillis?: number;
-    /**
-     * Maximum number of buffered URLs for the sitemap loading stream.
-     * If the buffer is full, the stream will pause until the buffer is drained.
-     *
-     * @default 200
-     */
-    maxBufferSize?: number;
-    /**
-     * Advanced options for the underlying `parseSitemap` call.
-     */
-    parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
-    /**
-     * Crawlee configuration
-     */
-    config?: Configuration;
-}
-/**
- * A list of URLs to crawl parsed from a sitemap.
- *
- * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
- */
-export declare class SitemapRequestList implements IRequestList {
-    /**
-     * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
-     * @internal
-     */
-    inProgress: Set<string>;
-    /** Set of URLs for which `reclaimRequest()` was called. */
-    private reclaimed;
-    /**
-     * Map of returned Request objects that have not been marked as handled yet.
-     *
-     * We use this to persist custom user fields on the in-progress (or reclaimed) requests.
-     */
-    private requestData;
-    /**
-     * Object for keeping track of the sitemap parsing progress.
-     */
-    private sitemapParsingProgress;
-    /**
-     * Object stream of URLs parsed from the sitemaps.
-     * Using `highWaterMark`, this can manage the speed of the sitemap loading.
-     *
-     * Fetch the next URL to be processed using `fetchNextRequest()`.
-     */
-    private urlQueueStream;
-    /**
-     * Indicates whether the request list sitemap loading was aborted.
-     *
-     * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
-     * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
-     *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
-     */
-    private abortLoading;
-    /** Number of URLs that were marked as handled */
-    private handledUrlCount;
-    private persistStateKey?;
-    private store?;
-    private closed;
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    private proxyUrl;
-    /**
-     * Logger instance.
-     */
-    private log;
-    private urlExcludePatternObjects;
-    private urlPatternObjects;
-    /** EventManager used to handle persistence */
-    private events;
-    private persistenceOptions;
-    /** @internal */
-    private constructor();
-    /**
-     * Creates a new object stream with the specified highWaterMark.
-     * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
-     * @returns A new object stream.
-     */
-    private createNewStream;
-    /**
-     * Returns a function that checks whether the provided pattern matches the closure URL.
-     * @param url URL to be checked.
-     * @returns A matcher function that checks whether the pattern matches the closure URL.
-     */
-    private matchesUrl;
-    /**
-     * Checks whether the URL matches the `globs` / `regexps` / `exclude` provided in the `options`.
-     * @param url URL to be checked.
-     * @returns `true` if the URL matches the patterns, `false` otherwise.
-     */
-    private isUrlMatchingPatterns;
-    /**
-     * Adds a URL to the queue of parsed URLs.
-     *
-     * Blocks if the stream is full until it is drained.
-     */
-    private pushNextUrl;
-    /**
-     * Reads the next URL from the queue of parsed URLs.
-     *
-     * If the stream is empty, blocks until a new URL is pushed.
-     * @returns The next URL from the queue or `null` if we have read all URLs.
-     */
-    private readNextUrl;
-    /**
-     * Indicates whether the background processing of sitemap contents has successfully finished.
-     *
-     * If this is `false`, the background processing is either still in progress or was aborted.
-     */
-    isSitemapFullyLoaded(): boolean;
-    /**
-     * Start processing the sitemaps and loading the URLs.
-     *
-     * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
-     */
-    private load;
-    /**
-     * Open a sitemap and start processing it.
-     *
-     * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
-     *
-     * Track the loading progress using the `isSitemapFullyLoaded` property.
-     */
-    static open(options: SitemapRequestListOptions): Promise<SitemapRequestList>;
-    /**
-     * @inheritDoc
-     */
-    length(): number;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    handledCount(): number;
-    /**
-     * @inheritDoc
-     */
-    persistState(): Promise<void>;
-    private restoreState;
-    /**
-     * @inheritDoc
-     */
-    fetchNextRequest(): Promise<Request | null>;
-    /**
-     * @inheritDoc
-     */
-// @ts-ignore optional peer dependency or compatibility with es2022
-    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/utils").Dictionary>, void, unknown>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(request: Request): Promise<void>;
-    /**
-     * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
-     *
-     * Calling `fetchNextRequest()` after this method will always return `null`.
-     */
-    teardown(): Promise<void>;
-    /**
-     * @inheritDoc
-     */
-    markRequestHandled(request: Request): Promise<void>;
-    private ensureInProgressAndNotReclaimed;
-}
-export {};
-//# sourceMappingURL=sitemap_request_list.d.ts.map

package/storages/sitemap_request_list.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sitemap_request_list.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOxE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAG3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMtD,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC7D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAiBD;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnD;;;OAGG;IACH,UAAU,cAAqB;IAE/B,2DAA2D;IAC3D,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAqB;IAErC;;OAEG;IACH,OAAO,CAAC,GAAG,CAAsD;IAEjE,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,OAAO;IAsDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAuClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAyBlF;;OAEG;IACH,MAAM,IAAI,MAAM;IAIhB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA+B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAkBjD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAMrD;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzD,OAAO,CAAC,+BAA+B;CAQ1C"}

package/storages/sitemap_request_list.js

@@ -1,430 +0,0 @@
-import { Transform } from 'node:stream';
-import { parseSitemap } from '@crawlee/utils';
-import { minimatch } from 'minimatch';
-import ow from 'ow';
-import defaultLog from '@apify/log';
-import { Configuration } from '../configuration.js';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links/shared.js';
-import { Request } from '../request.js';
-import { KeyValueStore } from './key_value_store.js';
-import { purgeDefaultStorages } from './utils.js';
-/** @internal */
-const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LIST_STATE';
-/**
- * A list of URLs to crawl parsed from a sitemap.
- *
- * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
- */
-export class SitemapRequestList {
-    /**
-     * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
-     * @internal
-     */
-    inProgress = new Set();
-    /** Set of URLs for which `reclaimRequest()` was called. */
-    reclaimed = new Set();
-    /**
-     * Map of returned Request objects that have not been marked as handled yet.
-     *
-     * We use this to persist custom user fields on the in-progress (or reclaimed) requests.
-     */
-    requestData = new Map();
-    /**
-     * Object for keeping track of the sitemap parsing progress.
-     */
-    sitemapParsingProgress = {
-        /**
-         * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
-         */
-        inProgressSitemapUrl: null,
-        /**
-         * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.
-         */
-        inProgressEntries: new Set(),
-        /**
-         * Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
-         */
-        pendingSitemapUrls: new Set(),
-    };
-    /**
-     * Object stream of URLs parsed from the sitemaps.
-     * Using `highWaterMark`, this can manage the speed of the sitemap loading.
-     *
-     * Fetch the next URL to be processed using `fetchNextRequest()`.
-     */
-    urlQueueStream;
-    /**
-     * Indicates whether the request list sitemap loading was aborted.
-     *
-     * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
-     * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
-     *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
-     */
-    abortLoading = false;
-    /** Number of URLs that were marked as handled */
-    handledUrlCount = 0;
-    persistStateKey;
-    store;
-    closed = false;
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    proxyUrl;
-    /**
-     * Logger instance.
-     */
-    log = defaultLog.child({ prefix: 'SitemapRequestList' });
-    urlExcludePatternObjects = [];
-    urlPatternObjects = [];
-    /** EventManager used to handle persistence */
-    events;
-    persistenceOptions;
-    /** @internal */
-    constructor(options) {
-        ow(options, ow.object.exactShape({
-            sitemapUrls: ow.array.ofType(ow.string),
-            proxyUrl: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            signal: ow.optional.any(),
-            timeoutMillis: ow.optional.number,
-            maxBufferSize: ow.optional.number,
-            parseSitemapOptions: ow.optional.object,
-            globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-            exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
-            regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-            config: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-        }));
-        const { globs, exclude, regexps, config = Configuration.getGlobalConfig() } = options;
-        if (exclude?.length) {
-            for (const excl of exclude) {
-                if (typeof excl === 'string' || 'glob' in excl) {
-                    this.urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-                }
-                else if (excl instanceof RegExp || 'regexp' in excl) {
-                    this.urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-                }
-            }
-        }
-        if (globs?.length) {
-            this.urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-        }
-        if (regexps?.length) {
-            this.urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-        }
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceOptions = { enable: true, ...options.persistenceOptions };
-        this.proxyUrl = options.proxyUrl;
-        this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
-        this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
-        this.events = config.getEventManager();
-        this.persistState = this.persistState.bind(this);
-    }
-    /**
-     * Creates a new object stream with the specified highWaterMark.
-     * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
-     * @returns A new object stream.
-     */
-    createNewStream(highWaterMark) {
-        return new Transform({
-            objectMode: true,
-            highWaterMark,
-        }).pause();
-    }
-    /**
-     * Returns a function that checks whether the provided pattern matches the closure URL.
-     * @param url URL to be checked.
-     * @returns A matcher function that checks whether the pattern matches the closure URL.
-     */
-    matchesUrl(url) {
-        return (patternObject) => {
-            const { regexp, glob } = patternObject;
-            const matchesRegex = (regexp && url.match(regexp)) || false;
-            const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
-            return Boolean(matchesRegex || matchesGlob);
-        };
-    }
-    /**
-     * Checks whether the URL matches the `globs` / `regexps` / `exclude` provided in the `options`.
-     * @param url URL to be checked.
-     * @returns `true` if the URL matches the patterns, `false` otherwise.
-     */
-    isUrlMatchingPatterns(url) {
-        return (!this.urlExcludePatternObjects.some(this.matchesUrl(url)) &&
-            (this.urlPatternObjects.length === 0 || this.urlPatternObjects.some(this.matchesUrl(url))));
-    }
-    /**
-     * Adds a URL to the queue of parsed URLs.
-     *
-     * Blocks if the stream is full until it is drained.
-     */
-    async pushNextUrl(url) {
-        return new Promise((resolve) => {
-            if (this.closed || (url && !this.isUrlMatchingPatterns(url))) {
-                resolve();
-                return;
-            }
-            if (!this.urlQueueStream.push(url)) {
-                // This doesn't work with the 'drain' event (it's not emitted for some reason).
-                this.urlQueueStream.once('readdata', () => {
-                    resolve();
-                });
-            }
-            else {
-                resolve();
-            }
-        });
-    }
-    /**
-     * Reads the next URL from the queue of parsed URLs.
-     *
-     * If the stream is empty, blocks until a new URL is pushed.
-     * @returns The next URL from the queue or `null` if we have read all URLs.
-     */
-    async readNextUrl() {
-        return new Promise((resolve) => {
-            if (this.closed) {
-                resolve(null);
-                return;
-            }
-            const result = this.urlQueueStream.read();
-            if (!result && !this.isSitemapFullyLoaded()) {
-                this.urlQueueStream.once('readable', () => {
-                    const nextUrl = this.urlQueueStream.read();
-                    resolve(nextUrl);
-                });
-            }
-            else {
-                resolve(result);
-            }
-            this.urlQueueStream.emit('readdata');
-        });
-    }
-    /**
-     * Indicates whether the background processing of sitemap contents has successfully finished.
-     *
-     * If this is `false`, the background processing is either still in progress or was aborted.
-     */
-    isSitemapFullyLoaded() {
-        return (this.sitemapParsingProgress.inProgressSitemapUrl === null &&
-            this.sitemapParsingProgress.pendingSitemapUrls.size === 0);
-    }
-    /**
-     * Start processing the sitemaps and loading the URLs.
-     *
-     * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
-     */
-    async load({ parseSitemapOptions, }) {
-        while (!this.isSitemapFullyLoaded() && !this.abortLoading) {
-            const sitemapUrl = this.sitemapParsingProgress.inProgressSitemapUrl ??
-                this.sitemapParsingProgress.pendingSitemapUrls.values().next().value;
-            try {
-                for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.proxyUrl, {
-                    ...parseSitemapOptions,
-                    maxDepth: 0,
-                    emitNestedSitemaps: true,
-                })) {
-                    if (!item.originSitemapUrl) {
-                        // This is a nested sitemap
-                        this.sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
-                        continue;
-                    }
-                    if (!this.sitemapParsingProgress.inProgressEntries.has(item.loc)) {
-                        await this.pushNextUrl(item.loc);
-                        this.sitemapParsingProgress.inProgressEntries.add(item.loc);
-                    }
-                }
-            }
-            catch (e) {
-                this.log.error('Error loading sitemap contents:', e);
-            }
-            this.sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
-            this.sitemapParsingProgress.inProgressEntries.clear();
-            this.sitemapParsingProgress.inProgressSitemapUrl = null;
-        }
-        this.urlQueueStream.end();
-    }
-    /**
-     * Open a sitemap and start processing it.
-     *
-     * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
-     *
-     * Track the loading progress using the `isSitemapFullyLoaded` property.
-     */
-    static async open(options) {
-        const requestList = new SitemapRequestList({
-            ...options,
-            persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
-        });
-        await requestList.restoreState();
-        void requestList.load({ parseSitemapOptions: options.parseSitemapOptions });
-        if (requestList.persistenceOptions.enable) {
-            requestList.events.on("persistState" /* EventType.PERSIST_STATE */, requestList.persistState);
-        }
-        options?.signal?.addEventListener('abort', () => {
-            requestList.abortLoading = true;
-        });
-        if (options.timeoutMillis) {
-            setTimeout(() => {
-                requestList.abortLoading = true;
-            }, options.timeoutMillis);
-        }
-        return requestList;
-    }
-    /**
-     * @inheritDoc
-     */
-    length() {
-        return this.urlQueueStream.readableLength + this.handledUrlCount - this.inProgress.size - this.reclaimed.size;
-    }
-    /**
-     * @inheritDoc
-     */
-    async isFinished() {
-        return ((await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.abortLoading));
-    }
-    /**
-     * @inheritDoc
-     */
-    async isEmpty() {
-        return this.reclaimed.size === 0 && this.urlQueueStream.readableLength === 0;
-    }
-    /**
-     * @inheritDoc
-     */
-    handledCount() {
-        return this.handledUrlCount;
-    }
-    /**
-     * @inheritDoc
-     */
-    async persistState() {
-        if (this.persistStateKey === undefined) {
-            return;
-        }
-        this.store ??= await KeyValueStore.open();
-        const urlQueue = [];
-        while (this.urlQueueStream.readableLength > 0) {
-            const url = this.urlQueueStream.read();
-            if (url === null) {
-                break;
-            }
-            urlQueue.push(url);
-        }
-        // Create a new stream, as we have read all the URLs from the current one.
-        // Pushing the urls back to the original stream might not be possible if it has been ended.
-        const newStream = this.createNewStream(this.urlQueueStream.readableHighWaterMark);
-        for (const url of urlQueue) {
-            newStream.push(url);
-        }
-        if (this.urlQueueStream.writableEnded) {
-            newStream.end();
-        }
-        this.urlQueueStream = newStream;
-        await this.store.setValue(this.persistStateKey, {
-            sitemapParsingProgress: {
-                pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),
-                inProgressSitemapUrl: this.sitemapParsingProgress.inProgressSitemapUrl,
-                inProgressEntries: Array.from(this.sitemapParsingProgress.inProgressEntries),
-            },
-            urlQueue,
-            reclaimed: [...this.inProgress, ...this.reclaimed], // In-progress and reclaimed requests will be both retried if state is restored
-            requestData: Array.from(this.requestData.entries()),
-            abortLoading: this.abortLoading,
-            closed: this.closed,
-        });
-    }
-    async restoreState() {
-        await purgeDefaultStorages({ onlyPurgeOnce: true });
-        if (this.persistStateKey === undefined) {
-            return;
-        }
-        this.store ??= await KeyValueStore.open();
-        const state = await this.store.getValue(this.persistStateKey);
-        if (state === null) {
-            return;
-        }
-        this.reclaimed = new Set(state.reclaimed);
-        this.sitemapParsingProgress = {
-            pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
-            inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
-            inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
-        };
-        this.requestData = new Map(state.requestData ?? []);
-        for (const url of state.urlQueue) {
-            this.urlQueueStream.push(url);
-        }
-        this.abortLoading = state.abortLoading;
-        this.closed = state.closed;
-    }
-    /**
-     * @inheritDoc
-     */
-    async fetchNextRequest() {
-        // Try to return a reclaimed request first
-        let nextUrl = this.reclaimed.values().next().value;
-        if (nextUrl) {
-            this.reclaimed.delete(nextUrl);
-        }
-        else {
-            // Otherwise read next url from the stream
-            nextUrl = await this.readNextUrl();
-            if (!nextUrl) {
-                return null;
-            }
-            this.requestData.set(nextUrl, new Request({ url: nextUrl }));
-        }
-        this.inProgress.add(nextUrl);
-        return this.requestData.get(nextUrl);
-    }
-    /**
-     * @inheritDoc
-     */
-    async *[Symbol.asyncIterator]() {
-        while (!(await this.isFinished())) {
-            const request = await this.fetchNextRequest();
-            if (!request)
-                break;
-            yield request;
-        }
-    }
-    /**
-     * @inheritDoc
-     */
-    async reclaimRequest(request) {
-        this.ensureInProgressAndNotReclaimed(request.url);
-        this.reclaimed.add(request.url);
-        this.inProgress.delete(request.url);
-    }
-    /**
-     * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
-     *
-     * Calling `fetchNextRequest()` after this method will always return `null`.
-     */
-    async teardown() {
-        this.closed = true;
-        this.abortLoading = true;
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
-        await this.persistState();
-        this.urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call
-    }
-    /**
-     * @inheritDoc
-     */
-    async markRequestHandled(request) {
-        this.handledUrlCount += 1;
-        this.ensureInProgressAndNotReclaimed(request.url);
-        this.inProgress.delete(request.url);
-        this.requestData.delete(request.url);
-    }
-    ensureInProgressAndNotReclaimed(url) {
-        if (!this.inProgress.has(url)) {
-            throw new Error(`The request is not being processed (url: ${url})`);
-        }
-        if (this.reclaimed.has(url)) {
-            throw new Error(`The request was already reclaimed (url: ${url})`);
-        }
-    }
-}
-//# sourceMappingURL=sitemap_request_list.js.map