@crawlee/core 4.0.0-beta.126 → 4.0.0-beta.128

package/enqueue_links/enqueue_links.d.ts

@@ -1,4 +1,5 @@
 import type { Dictionary } from '@crawlee/types';
+import { EnqueueStrategy } from '@crawlee/utils';
 import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
 import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
 /**
@@ -127,57 +128,7 @@ export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
 }
 /** The combined options accepted by a crawler context's `enqueueLinks()` helper: `extractLinks()` + `enqueueUrls()`. */
 export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
-}
+export { EnqueueStrategy };
 /** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
 export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**

package/enqueue_links/enqueue_links.js

@@ -1,56 +1,6 @@
+import { EnqueueStrategy } from '@crawlee/utils';
 import { getDomain } from 'tldts';
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export var EnqueueStrategy;
-(function (EnqueueStrategy) {
-    /**
-     * Matches any URLs found
-     */
-    EnqueueStrategy["All"] = "all";
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameHostname"] = "same-hostname";
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameDomain"] = "same-domain";
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    EnqueueStrategy["SameOrigin"] = "same-origin";
-})(EnqueueStrategy || (EnqueueStrategy = {}));
+export { EnqueueStrategy };
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.126",
+    "version": "4.0.0-beta.128",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.126",
-        "@crawlee/http-client": "4.0.0-beta.126",
-        "@crawlee/types": "4.0.0-beta.126",
-        "@crawlee/utils": "4.0.0-beta.126",
+        "@crawlee/fs-storage": "4.0.0-beta.128",
+        "@crawlee/http-client": "4.0.0-beta.128",
+        "@crawlee/types": "4.0.0-beta.128",
+        "@crawlee/utils": "4.0.0-beta.128",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
+    "gitHead": "2c017f04d564e8fa13855bf19f31f43cb65f4f44"
 }

package/session_pool/session_pool.d.ts

@@ -181,8 +181,11 @@ export declare class SessionPool implements ISessionPool {
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    teardown(): Promise<void>;
+    teardown({ persistState }?: {
+        persistState?: boolean;
+    }): Promise<void>;
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */

package/session_pool/session_pool.js

@@ -264,15 +264,18 @@ export class SessionPool {
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    async teardown() {
+    async teardown({ persistState = true } = {}) {
         if (!this.#initPromise)
             return;
         await this.ensureInitialized();
         if (this.#listener) {
             this.#events.off(EventType.PERSIST_STATE, this.#listener);
         }
-        await this.persistState();
+        if (persistState) {
+            await this.persistState();
+        }
     }
     /**
      * Removes retired `Session` instances from `SessionPool`.

package/storages/sitemap_request_loader.d.ts

@@ -1,5 +1,5 @@
 import type { BaseHttpClient } from '@crawlee/http-client';
-import { type ParseSitemapOptions } from '@crawlee/utils';
+import { EnqueueStrategy, type ParseSitemapOptions } from '@crawlee/utils';
 import type { UrlPatternInput } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';
 import type { IRequestLoader } from './request_loader.js';
@@ -65,6 +65,13 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
      * @default 200
      */
     maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Advanced options for the underlying `parseSitemap` call.
      */

package/storages/sitemap_request_loader.js

@@ -1,5 +1,5 @@
 import { Transform } from 'node:stream';
-import { parseSitemap } from '@crawlee/utils';
+import { EnqueueStrategy, parseSitemap } from '@crawlee/utils';
 import { minimatch } from 'minimatch';
 import { z } from 'zod';
 import { constructUrlPatternObjects, urlPatternSchema } from '../enqueue_links/shared.js';
@@ -16,6 +16,7 @@ const sitemapRequestLoaderOptionsSchema = z.strictObject({
     signal: z.unknown().optional(),
     timeoutMillis: schemas.anyNumber.optional(),
     maxBufferSize: schemas.anyNumber.default(200),
+    enqueueStrategy: z.enum(EnqueueStrategy).default(EnqueueStrategy.SameHostname),
     parseSitemapOptions: z.looseObject({}).optional(),
     include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
     exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
@@ -83,6 +84,10 @@ export class SitemapRequestLoader {
      * Proxy URL to be used for sitemap loading.
      */
     #proxyUrl;
+    /**
+     * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+     */
+    #enqueueStrategy;
     /**
      * Logger instance.
      */
@@ -94,7 +99,7 @@ export class SitemapRequestLoader {
     #persistenceOptions;
     /** @internal */
     constructor(options) {
-        const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
+        const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
         this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
         if (exclude?.length) {
             this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
@@ -105,6 +110,7 @@ export class SitemapRequestLoader {
         this.#persistStateKey = persistStateKey;
         this.#persistenceOptions = { enable: true, ...persistenceOptions };
         this.#proxyUrl = proxyUrl;
+        this.#enqueueStrategy = enqueueStrategy;
         this.#urlQueueStream = this.createNewStream(maxBufferSize);
         this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
         this.#events = serviceLocator.getEventManager();
@@ -213,6 +219,7 @@ export class SitemapRequestLoader {
                     ...parseSitemapOptions,
                     maxDepth: 0,
                     emitNestedSitemaps: true,
+                    enqueueStrategy: this.#enqueueStrategy,
                 })) {
                     if (!item.originSitemapUrl) {
                         // This is a nested sitemap
@@ -326,14 +333,21 @@ export class SitemapRequestLoader {
         }
         // Create a new stream, as we have read all the URLs from the current one.
         // Pushing the urls back to the original stream might not be possible if it has been ended.
-        const newStream = this.createNewStream(this.#urlQueueStream.readableHighWaterMark);
+        const previousStream = this.#urlQueueStream;
+        const newStream = this.createNewStream(previousStream.readableHighWaterMark);
         for (const url of urlQueue) {
             newStream.push(url);
         }
-        if (this.#urlQueueStream.writableEnded) {
+        if (previousStream.writableEnded) {
             newStream.end();
         }
         this.#urlQueueStream = newStream;
+        // A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the
+        // previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so
+        // after the swap the waiter would never be notified and the background sitemap loading would hang.
+        // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
+        // been transferred to the new stream above).
+        previousStream.emit('readdata');
         await this.#store.setValue(this.#persistStateKey, {
             sitemapParsingProgress: {
                 pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
@@ -379,7 +393,7 @@ export class SitemapRequestLoader {
         }
         // A restored in-progress request already has its Request data; don't overwrite it.
         if (!this.#requestData.has(nextUrl)) {
-            this.#requestData.set(nextUrl, new Request({ url: nextUrl }));
+            this.#requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.#enqueueStrategy }));
         }
         this.inProgress.add(nextUrl);
         return this.#requestData.get(nextUrl);

package/storages/utils.js

@@ -12,11 +12,20 @@ export async function purgeDefaultStorages(configurationOrOptions, storageBacken
     const { configuration = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
     ({ storageBackend = serviceLocator.getStorageBackend() } = options);
     const casted = storageBackend;
+    const runPurge = async () => {
+        try {
+            await casted.purge?.();
+        }
+        catch (e) {
+            casted.__purged = undefined;
+            throw e;
+        }
+    };
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
     if (!onlyPurgeOnce || (configuration.purgeOnStart && !casted.__purged)) {
-        casted.__purged = true;
-        await casted.purge?.();
+        casted.__purged = runPurge();
     }
+    await casted.__purged;
 }
 /**
  * Easily create and manage state values. All state values are automatically persisted.