@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/session_pool/session_pool.js

@@ -1,386 +0,0 @@
-import { AsyncQueue } from '@sapphire/async-queue';
-import ow from 'ow';
-import { serviceLocator } from '../service_locator.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
-import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
-import { createDefaultSessionFingerprint } from './fingerprint.js';
-import { Session } from './session.js';
-const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
-/**
- * Handles the rotation, creation and persistence of user-like sessions.
- * Creates a pool of {@link Session} instances, that are randomly rotated.
- * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
- * Learn more in the {@doclink guides/session-management | Session management guide}.
- *
- * Session pool is already integrated into crawlers and is always active.
- * All public methods are lazy-initialized — the pool initializes itself on first use.
- *
- * You can configure the pool with many options. See the {@link SessionPoolOptions}.
- * Session pool is by default persisted in default {@link KeyValueStore}.
- * If you want to have one pool for all runs you have to specify
- * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
- *
- * **Advanced usage:**
- *
- * ```javascript
- * const sessionPool = new SessionPool({
- *     maxPoolSize: 25,
- *     sessionOptions:{
- *          maxAgeSecs: 10,
- *          maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
- *     },
- *     persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
- *     persistStateKey: 'my-session-pool',
- * });
- *
- * // Get random session from the pool
- * const session1 = await sessionPool.getSession();
- * const session2 = await sessionPool.getSession();
- * const session3 = await sessionPool.getSession();
- *
- * // Now you can mark the session either failed or successful
- *
- * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
- * session1.markBad()
- *
- * // Marks as successful.
- * session2.markGood()
- *
- * // Retires session -> session is removed from the pool
- * session3.retire()
- *
- * ```
- *
- * **Default session allocation flow:*
- * 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
- * 2. Blocked/retired sessions stay in the pool but are never provided to the user
- * 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
- * 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
- *
- * @category Scaling
- */
-export class SessionPool {
-    static nextId = 0;
-    id;
-    log;
-    maxPoolSize;
-    createSessionFunction;
-    keyValueStore;
-    sessions = [];
-    sessionMap = new Map();
-    sessionOptions;
-    persistStateKeyValueStoreId;
-    persistStateKey;
-    listener;
-    events;
-    persistenceOptions;
-    sessionReuseStrategy;
-    initPromise;
-    queue = new AsyncQueue();
-    roundRobinIndex = 0;
-    constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            id: ow.optional.any(ow.number, ow.string),
-            maxPoolSize: ow.optional.number,
-            persistStateKeyValueStoreId: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            createSessionFunction: ow.optional.function,
-            sessionOptions: ow.optional.object,
-            log: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-            sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
-        }));
-        const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
-            enable: true,
-        }, sessionReuseStrategy = 'random', } = options;
-        this.id = id != null ? String(id) : String(SessionPool.nextId++);
-        this.sessionReuseStrategy = sessionReuseStrategy;
-        this.events = serviceLocator.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
-        // Pool Configuration
-        this.maxPoolSize = maxPoolSize;
-        this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
-        // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
-        // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
-        // having to know about it.
-        this.sessionOptions = {
-            ...sessionOptions,
-            log: this.log,
-        };
-        // Session keyValueStore
-        this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
-        this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
-    }
-    /**
-     * Gets count of usable sessions in the pool.
-     */
-    async usableSessionsCount() {
-        await this.ensureInitialized();
-        return this.sessions.filter((session) => session.isUsable()).length;
-    }
-    /**
-     * Gets count of retired sessions in the pool.
-     */
-    async retiredSessionsCount() {
-        await this.ensureInitialized();
-        return this.sessions.filter((session) => !session.isUsable()).length;
-    }
-    /**
-     * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * Called automatically on first use of any public method.
-     */
-    async ensureInitialized() {
-        if (!this.initPromise) {
-            this.initPromise = this.setupPool();
-        }
-        return this.initPromise;
-    }
-    async setupPool() {
-        if (!this.persistenceOptions.enable) {
-            return;
-        }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
-            configuration: serviceLocator.getConfiguration(),
-        });
-        if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
-        }
-        // in case of migration happened and SessionPool state should be restored from the keyValueStore.
-        await this.maybeLoadSessionPool();
-        this.listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
-    }
-    /**
-     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
-     * but this allows you to add more sessions once the max pool size is reached.
-     * This also allows you to add session with overridden session options (e.g. with specific session id).
-     * @param [options] The configuration options for the session being added to the session pool.
-     */
-    async addSession(options = {}) {
-        await this.ensureInitialized();
-        const { id } = options;
-        if (id) {
-            const sessionExists = this.sessionMap.has(id);
-            if (sessionExists) {
-                throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
-            }
-        }
-        if (!this.hasSpaceForSession()) {
-            this.removeRetiredSessions();
-        }
-        const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);
-        this.log.debug(`Adding new Session - ${newSession.id}`);
-        this.registerSession(newSession);
-    }
-    /**
-     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
-     * but this allows you to add more sessions once the max pool size is reached.
-     * This also allows you to add session with overridden session options (e.g. with specific session id).
-     * @param [options] The configuration options for the session being added to the session pool.
-     */
-    async newSession(sessionOptions) {
-        await this.ensureInitialized();
-        const newSession = await this._invokeCreateSessionFunction(sessionOptions);
-        this.registerSession(newSession);
-        return newSession;
-    }
-    /**
-     * Gets session.
-     * If there is space for new session, it creates and returns new session.
-     * If the session pool is full, it picks a session from the pool,
-     * If the picked session is usable it is returned, otherwise it creates and returns a new one.
-     * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
-     */
-    async getSession(sessionId) {
-        await this.ensureInitialized();
-        await this.queue.wait();
-        try {
-            if (sessionId) {
-                const session = this.sessionMap.get(sessionId);
-                if (session?.isUsable())
-                    return session;
-                return undefined;
-            }
-            const pickedSession = this.pickSession();
-            if (pickedSession)
-                return pickedSession;
-            if (this.hasSpaceForSession()) {
-                return await this.createSession();
-            }
-            this.removeRetiredSessions();
-            return await this.createSession();
-        }
-        finally {
-            this.queue.shift();
-        }
-    }
-    /**
-     * @param options - Override the persistence options provided in the constructor
-     */
-    async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        await this.ensureInitialized();
-        await this.keyValueStore?.setValue(this.persistStateKey, null);
-    }
-    /**
-     * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
-     */
-    async getState() {
-        await this.ensureInitialized();
-        return {
-            usableSessionsCount: await this.usableSessionsCount(),
-            retiredSessionsCount: await this.retiredSessionsCount(),
-            sessions: this.sessions.map((session) => session.getState()),
-        };
-    }
-    /**
-     * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
-     * The state is persisted automatically in regular intervals.
-     * @param options - Override the persistence options provided in the constructor
-     */
-    async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        await this.ensureInitialized();
-        this.log.debug('Persisting state', {
-            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
-            persistStateKey: this.persistStateKey,
-        });
-        await this.keyValueStore
-            ?.setValue(this.persistStateKey, await this.getState())
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
-    }
-    /**
-     * Removes listener from `persistState` event.
-     * This function should be called after you are done with using the `SessionPool` instance.
-     */
-    async teardown() {
-        if (!this.initPromise)
-            return;
-        await this.ensureInitialized();
-        if (this.listener) {
-            this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
-        }
-        await this.persistState();
-    }
-    /**
-     * Removes retired `Session` instances from `SessionPool`.
-     */
-    removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
-            if (storedSession.isUsable())
-                return true;
-            this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
-            return false;
-        });
-    }
-    /**
-     * Adds `Session` instance to `SessionPool`.
-     * @param newSession `Session` instance to be added.
-     */
-    registerSession(newSession) {
-        this.sessions.push(newSession);
-        this.sessionMap.set(newSession.id, newSession);
-    }
-    /**
-     * Gets random index.
-     */
-    getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
-    }
-    /**
-     * Creates new session without any extra behavior.
-     * @param [options]
-     * @param [options.sessionOptions] The configuration options for the session being created.
-     * @returns New session.
-     */
-    async defaultCreateSessionFunction(options = {}) {
-        ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
-        const { sessionOptions = {} } = options;
-        return new Session(sessionOptions);
-    }
-    /**
-     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
-     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
-     *
-     * A default {@link SessionFingerprint} is generated up front (host OS as
-     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
-     * and per-call options override it, and a persisted fingerprint coming
-     * through `maybeLoadSessionPool` naturally wins because it arrives in
-     * `perCallOptions`.
-     */
-    async _invokeCreateSessionFunction(perCallOptions) {
-        const sessionOptions = {
-            fingerprint: createDefaultSessionFingerprint(),
-            ...this.sessionOptions,
-            ...perCallOptions,
-        };
-        return this.createSessionFunction({ sessionOptions });
-    }
-    /**
-     * Creates new session and adds it to the pool.
-     * @returns Newly created `Session` instance.
-     */
-    async createSession() {
-        const newSession = await this._invokeCreateSessionFunction();
-        this.registerSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
-        return newSession;
-    }
-    /**
-     * Decides whether there is enough space for creating new session.
-     */
-    hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
-    }
-    /**
-     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
-     * Returns `undefined` when no session should be reused and a new one should be created instead.
-     */
-    pickSession() {
-        if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
-            return undefined;
-        if (this.sessionReuseStrategy === 'use-until-failure') {
-            return this.sessions.find((session) => session.isUsable());
-        }
-        let picked;
-        if (this.sessionReuseStrategy === 'round-robin') {
-            const index = this.roundRobinIndex % this.sessions.length;
-            this.roundRobinIndex = index + 1;
-            picked = this.sessions[index];
-        }
-        else {
-            picked = this.sessions[this.getRandomIndex()];
-        }
-        return picked.isUsable() ? picked : undefined;
-    }
-    /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     */
-    async maybeLoadSessionPool() {
-        const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
-        if (!loadedSessionPool)
-            return;
-        // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
-            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
-            persistStateKey: this.persistStateKey,
-        });
-        for (const sessionObject of loadedSessionPool.sessions) {
-            sessionObject.createdAt = new Date(sessionObject.createdAt);
-            sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);
-            if (recreatedSession.isUsable()) {
-                this.registerSession(recreatedSession);
-            }
-        }
-        this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);
-    }
-}

package/storages/access_checking.d.ts

@@ -1,12 +0,0 @@
-import type { Awaitable } from '@crawlee/types';
-/**
- * Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
- */
-export declare const checkStorageAccess: () => void | undefined;
-/**
- * Define a storage access checker function that should be used by calls to {@link checkStorageAccess} in the callbacks.
- *
- * @param checkFunction The check function that should be invoked by {@link checkStorageAccess} calls
- * @param callback The code that should be invoked with the `checkFunction` setting
- */
-export declare const withCheckedStorageAccess: <T>(checkFunction: () => void, callback: () => Awaitable<T>) => Promise<T>;

package/storages/access_checking.js

@@ -1,17 +0,0 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
-import { tryCancel } from '@apify/timeout';
-const storage = new AsyncLocalStorage();
-/**
- * Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
- */
-export const checkStorageAccess = () => {
-    tryCancel();
-    return storage.getStore()?.checkFunction();
-};
-/**
- * Define a storage access checker function that should be used by calls to {@link checkStorageAccess} in the callbacks.
- *
- * @param checkFunction The check function that should be invoked by {@link checkStorageAccess} calls
- * @param callback The code that should be invoked with the `checkFunction` setting
- */
-export const withCheckedStorageAccess = async (checkFunction, callback) => storage.run({ checkFunction }, callback);

package/storages/sitemap_request_loader.d.ts

@@ -1,249 +0,0 @@
-import type { BaseHttpClient } from '@crawlee/types';
-import { type ParseSitemapOptions } from '@crawlee/utils';
-import type { GlobInput, RegExpInput } from '../enqueue_links/shared.js';
-import { Request } from '../request.js';
-import type { IRequestLoader } from './request_loader.js';
-import type { IRequestManager } from './request_manager.js';
-interface UrlConstraints {
-    /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the `SitemapRequestLoader`
-     * includes all the URLs from the sitemap.
-     */
-    globs?: readonly GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be included.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
-     *
-     * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExp)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the `SitemapRequestLoader`
-     * includes all the URLs from the sitemap.
-     */
-    regexps?: readonly RegExpInput[];
-}
-export interface SitemapRequestLoaderOptions extends UrlConstraints {
-    /**
-     * List of sitemap URLs to parse.
-     */
-    sitemapUrls: string[];
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    proxyUrl?: string;
-    /**
-     * Key for persisting the state of the request list in the `KeyValueStore`.
-     */
-    persistStateKey?: string;
-    /**
-     * Persistence-related options to control how and when crawler's data gets persisted.
-     */
-    persistenceOptions?: {
-        /**
-         * Use this flag to disable or enable periodic persistence to key value store.
-         * @default true
-         */
-        enable?: boolean;
-    };
-    /**
-     * Abort signal to be used for sitemap loading.
-     */
-    signal?: AbortSignal;
-    /**
-     * Timeout for sitemap loading in milliseconds. If both `signal` and `timeoutMillis` are provided, either of them can abort the loading.
-     */
-    timeoutMillis?: number;
-    /**
-     * Maximum number of buffered URLs for the sitemap loading stream.
-     * If the buffer is full, the stream will pause until the buffer is drained.
-     *
-     * @default 200
-     */
-    maxBufferSize?: number;
-    /**
-     * Advanced options for the underlying `parseSitemap` call.
-     */
-    parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
-    /**
-     * Custom HTTP client to be used for sitemap loading.
-     */
-    httpClient?: BaseHttpClient;
-}
-/**
- * A list of URLs to crawl parsed from a sitemap.
- *
- * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
- */
-export declare class SitemapRequestLoader implements IRequestLoader {
-    /**
-     * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
-     * @internal
-     */
-    inProgress: Set<string>;
-    /**
-     * Map of returned Request objects that have not been marked as handled yet.
-     *
-     * We use this to persist custom user fields on the in-progress requests.
-     */
-    private requestData;
-    /**
-     * Object for keeping track of the sitemap parsing progress.
-     */
-    private sitemapParsingProgress;
-    /**
-     * Object stream of URLs parsed from the sitemaps.
-     * Using `highWaterMark`, this can manage the speed of the sitemap loading.
-     *
-     * Fetch the next URL to be processed using `fetchNextRequest()`.
-     */
-    private urlQueueStream;
-    /**
-     * Indicates whether the request list sitemap loading was aborted.
-     *
-     * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
-     * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
-     *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
-     */
-    private abortLoading;
-    /** Number of URLs that were marked as handled */
-    private handledUrlCount;
-    private persistStateKey?;
-    private store?;
-    private closed;
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    private proxyUrl?;
-    /**
-     * Logger instance.
-     */
-    private log;
-    private urlExcludePatternObjects;
-    private urlPatternObjects;
-    /** EventManager used to handle persistence */
-    private events;
-    private persistenceOptions;
-    /** @internal */
-    private constructor();
-    /**
-     * Creates a new object stream with the specified highWaterMark.
-     * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
-     * @returns A new object stream.
-     */
-    private createNewStream;
-    /**
-     * Returns a function that checks whether the provided pattern matches the closure URL.
-     * @param url URL to be checked.
-     * @returns A matcher function that checks whether the pattern matches the closure URL.
-     */
-    private matchesUrl;
-    /**
-     * Checks whether the URL matches the `globs` / `regexps` / `exclude` provided in the `options`.
-     * @param url URL to be checked.
-     * @returns `true` if the URL matches the patterns, `false` otherwise.
-     */
-    private isUrlMatchingPatterns;
-    /**
-     * Adds a URL to the queue of parsed URLs.
-     *
-     * Blocks if the stream is full until it is drained.
-     */
-    private pushNextUrl;
-    /**
-     * Reads the next URL from the queue of parsed URLs.
-     *
-     * If the stream is empty, blocks until a new URL is pushed.
-     * @returns The next URL from the queue or `null` if we have read all URLs.
-     */
-    private readNextUrl;
-    /**
-     * Indicates whether the background processing of sitemap contents has successfully finished.
-     *
-     * If this is `false`, the background processing is either still in progress or was aborted.
-     */
-    isSitemapFullyLoaded(): boolean;
-    /**
-     * Start processing the sitemaps and loading the URLs.
-     *
-     * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
-     */
-    private load;
-    /**
-     * Open a sitemap and start processing it.
-     *
-     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
-     *
-     * Track the loading progress using the `isSitemapFullyLoaded` property.
-     */
-    static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
-    /**
-     * @inheritDoc
-     */
-    getTotalCount(): Promise<number>;
-    /**
-     * @inheritDoc
-     */
-    getPendingCount(): Promise<number>;
-    /**
-     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
-     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
-     * being read from this list first.
-     */
-    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    getHandledCount(): Promise<number>;
-    /**
-     * @inheritDoc
-     */
-    persistState(): Promise<void>;
-    private restoreState;
-    /**
-     * @inheritDoc
-     */
-    fetchNextRequest(): Promise<Request | null>;
-    /**
-     * @inheritDoc
-     */
-// @ts-ignore optional peer dependency or compatibility with es2022
-    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
-    /**
-     * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
-     *
-     * Calling `fetchNextRequest()` after this method will always return `null`.
-     */
-    teardown(): Promise<void>;
-    /**
-     * @inheritDoc
-     */
-    markRequestAsHandled(request: Request): Promise<void>;
-    private ensureInProgress;
-}
-export {};