@crawlee/core 3.17.1-beta.57 → 3.17.1-beta.59

package/enqueue_links/enqueue_links.d.ts

@@ -1,9 +1,10 @@
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
+import { EnqueueStrategy, type RobotsTxtFile } from '@crawlee/utils';
 import type { SetRequired } from 'type-fest';
 import type { Request } from '../request';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages';
 import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared';
+export { EnqueueStrategy };
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
@@ -156,57 +157,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
-}
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.

package/enqueue_links/enqueue_links.js

@@ -4,62 +4,12 @@ exports.EnqueueStrategy = void 0;
 exports.enqueueLinks = enqueueLinks;
 exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
 const tslib_1 = require("tslib");
+const utils_1 = require("@crawlee/utils");
+Object.defineProperty(exports, "EnqueueStrategy", { enumerable: true, get: function () { return utils_1.EnqueueStrategy; } });
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const tldts_1 = require("tldts");
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const shared_1 = require("./shared");
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-var EnqueueStrategy;
-(function (EnqueueStrategy) {
-    /**
-     * Matches any URLs found
-     */
-    EnqueueStrategy["All"] = "all";
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameHostname"] = "same-hostname";
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameDomain"] = "same-domain";
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    EnqueueStrategy["SameOrigin"] = "same-origin";
-})(EnqueueStrategy || (exports.EnqueueStrategy = EnqueueStrategy = {}));
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
@@ -109,7 +59,7 @@ async function enqueueLinks(options) {
         exclude: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('glob'), ow_1.default.object.hasKeys('regexp'))),
         regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
         transformRequestFunction: ow_1.default.optional.function,
-        strategy: ow_1.default.optional.string.oneOf(Object.values(EnqueueStrategy)),
+        strategy: ow_1.default.optional.string.oneOf(Object.values(utils_1.EnqueueStrategy)),
         waitForAllRequestsToBeAdded: ow_1.default.optional.boolean,
     }));
     const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
@@ -136,19 +86,19 @@ async function enqueueLinks(options) {
         urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
     }
     if (!urlPatternObjects.length) {
-        options.strategy ?? (options.strategy = EnqueueStrategy.SameHostname);
+        options.strategy ?? (options.strategy = utils_1.EnqueueStrategy.SameHostname);
     }
     const enqueueStrategyPatterns = [];
     if (options.baseUrl) {
         const url = new URL(options.baseUrl);
         switch (options.strategy) {
-            case EnqueueStrategy.SameHostname:
+            case utils_1.EnqueueStrategy.SameHostname:
                 // We need to get the origin of the passed in domain in the event someone sets baseUrl
                 // to an url like https://example.com/deep/default/path and one of the found urls is an
                 // absolute relative path (/path/to/page)
                 enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
                 break;
-            case EnqueueStrategy.SameDomain: {
+            case utils_1.EnqueueStrategy.SameDomain: {
                 // Get the actual hostname from the base url
                 const baseUrlHostname = (0, tldts_1.getDomain)(url.hostname, { mixedInputs: false });
                 if (baseUrlHostname) {
@@ -163,12 +113,12 @@ async function enqueueLinks(options) {
                 }
                 break;
             }
-            case EnqueueStrategy.SameOrigin: {
+            case utils_1.EnqueueStrategy.SameOrigin: {
                 // The same behavior as SameHostname, but respecting the protocol of the URL
                 enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
                 break;
             }
-            case EnqueueStrategy.All:
+            case utils_1.EnqueueStrategy.All:
             default:
                 enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
                 break;
@@ -249,13 +199,13 @@ function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestU
     const originalUrlOrigin = new URL(originalRequestUrl).origin;
     const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
     // We can assume users want to go off the domain in this case
-    if (enqueueStrategy === EnqueueStrategy.All) {
+    if (enqueueStrategy === utils_1.EnqueueStrategy.All) {
         return finalUrlOrigin;
     }
     // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
     // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
     // which is the intended behavior (since we went off domain)
-    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
+    if (enqueueStrategy === utils_1.EnqueueStrategy.SameDomain) {
         const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
         const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
         if (originalHostname === finalHostname) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.57",
+    "version": "3.17.1-beta.59",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -57,11 +57,11 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
         "@apify/pseudo_url": "^2.0.30",
-        "@apify/timeout": "^0.3.0",
+        "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.57",
-        "@crawlee/types": "3.17.1-beta.57",
-        "@crawlee/utils": "3.17.1-beta.57",
+        "@crawlee/memory-storage": "3.17.1-beta.59",
+        "@crawlee/types": "3.17.1-beta.59",
+        "@crawlee/utils": "3.17.1-beta.59",
         "@sapphire/async-queue": "^1.5.1",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.2.2",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "b5412ad2cdd908c56f607b83df75ff1e2ef77584"
+    "gitHead": "8980103b700213806843d4fe1355b75519474ea4"
 }

package/storages/sitemap_request_list.d.ts

@@ -1,6 +1,7 @@
 import { type ParseSitemapOptions } from '@crawlee/utils';
 import { Configuration } from '../configuration';
 import type { GlobInput, RegExpInput } from '../enqueue_links';
+import { EnqueueStrategy } from '../enqueue_links';
 import { Request } from '../request';
 import type { IRequestList } from './request_list';
 interface UrlConstraints {
@@ -76,6 +77,13 @@ export interface SitemapRequestListOptions extends UrlConstraints {
      * @default 200
      */
     maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Advanced options for the underlying `parseSitemap` call.
      */
@@ -133,6 +141,10 @@ export declare class SitemapRequestList implements IRequestList {
      * Proxy URL to be used for sitemap loading.
      */
     private proxyUrl;
+    /**
+     * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+     */
+    private enqueueStrategy;
     /**
      * Logger instance.
      */

package/storages/sitemap_request_list.js

@@ -132,6 +132,15 @@ class SitemapRequestList {
             writable: true,
             value: void 0
         });
+        /**
+         * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+         */
+        Object.defineProperty(this, "enqueueStrategy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * Logger instance.
          */
@@ -173,6 +182,7 @@ class SitemapRequestList {
             signal: ow_1.default.optional.any(),
             timeoutMillis: ow_1.default.optional.number,
             maxBufferSize: ow_1.default.optional.number,
+            enqueueStrategy: ow_1.default.optional.string.oneOf(Object.values(enqueue_links_1.EnqueueStrategy)),
             parseSitemapOptions: ow_1.default.optional.object,
             globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
             exclude: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('glob'), ow_1.default.object.hasKeys('regexp'))),
@@ -200,6 +210,7 @@ class SitemapRequestList {
         this.persistStateKey = options.persistStateKey;
         this.persistenceOptions = { enable: true, ...options.persistenceOptions };
         this.proxyUrl = options.proxyUrl;
+        this.enqueueStrategy = options.enqueueStrategy ?? enqueue_links_1.EnqueueStrategy.SameHostname;
         this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
         this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
         this.events = config.getEventManager();
@@ -308,6 +319,7 @@ class SitemapRequestList {
                     ...parseSitemapOptions,
                     maxDepth: 0,
                     emitNestedSitemaps: true,
+                    enqueueStrategy: this.enqueueStrategy,
                 })) {
                     if (!item.originSitemapUrl) {
                         // This is a nested sitemap
@@ -457,7 +469,7 @@ class SitemapRequestList {
             if (!nextUrl) {
                 return null;
             }
-            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl }));
+            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl, enqueueStrategy: this.enqueueStrategy }));
         }
         this.inProgress.add(nextUrl);
         return this.requestData.get(nextUrl);