@crawlee/core 3.17.1-beta.9 → 3.18.0

package/autoscaling/autoscaled_pool.js

@@ -415,14 +415,17 @@ class AutoscaledPool {
         this.isStopped = true;
         await new Promise((resolve, reject) => {
             let timeout;
+            let interval;
             if (timeoutSecs) {
                 timeout = setTimeout(() => {
+                    // Clean up the polling interval to prevent it from leaking on timeout.
+                    clearInterval(interval);
                     const err = new Error("The pool's running tasks did not finish" +
                         `in ${timeoutSecs} secs after pool.pause() invocation.`);
                     reject(err);
                 }, timeoutSecs);
             }
-            const interval = setInterval(() => {
+            interval = setInterval(() => {
                 if (this._currentConcurrency <= 0) {
                     // Clean up timeout and interval to prevent process hanging.
                     if (timeout)

package/autoscaling/snapshotter.d.ts

@@ -35,13 +35,13 @@ export interface SnapshotterOptions {
     /**
      * Defines the maximum number of new rate limit errors within
      * the given interval.
-     * @default 1
+     * @default 3
      */
     maxClientErrors?: number;
     /**
      * Sets the interval in seconds for which a history of resource snapshots
      * will be kept. Increasing this to very high numbers will affect performance.
-     * @default 60
+     * @default 30
      */
     snapshotHistorySecs?: number;
     /** @internal */

package/cookie_utils.js

@@ -107,6 +107,8 @@ function mergeCookies(url, sourceCookies) {
             if (!cookieString)
                 continue;
             const cookie = tough_cookie_1.Cookie.parse(cookieString);
+            if (!cookie)
+                throw new errors_1.CookieParseError(cookieString);
             const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });

package/crawlers/statistics.d.ts

@@ -35,7 +35,7 @@ export declare class Statistics {
     /**
      * Statistic instance id.
      */
-    readonly id: number;
+    readonly id: string;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
@@ -166,13 +166,22 @@ export interface StatisticsOptions {
      * @default false
      */
     saveErrorSnapshots?: boolean;
+    /**
+     * A unique identifier for this statistics instance. This ID is used for persistence
+     * to the key value store, ensuring the same statistics can be loaded after script restarts.
+     *
+     * If not provided, an auto-incremented ID will be used for backward compatibility.
+     * This means statistics may not persist correctly across script restarts
+     * if crawler creation order changes.
+     */
+    id?: string;
 }
 /**
  * Format of the persisted stats
  */
 export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
     requestRetryHistogram: number[];
-    statsId: number;
+    statsId: string;
     requestAvgFailedDurationMillis: number;
     requestAvgFinishedDurationMillis: number;
     requestTotalDurationMillis: number;

package/crawlers/statistics.js

@@ -81,8 +81,8 @@ class Statistics {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: Statistics.id++
-        }); // assign an id while incrementing so it can be saved/restored from KV
+            value: void 0
+        });
         /**
          * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
          */
@@ -120,7 +120,7 @@ class Statistics {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: `SDK_CRAWLER_STATISTICS_${this.id}`
+            value: void 0
         });
         Object.defineProperty(this, "logIntervalMillis", {
             enumerable: true,
@@ -184,10 +184,13 @@ class Statistics {
             config: ow_1.default.optional.object,
             persistenceOptions: ow_1.default.optional.object,
             saveErrorSnapshots: ow_1.default.optional.boolean,
+            id: ow_1.default.optional.any(ow_1.default.number, ow_1.default.string),
         }));
         const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = configuration_1.Configuration.getGlobalConfig(), persistenceOptions = {
             enable: true,
-        }, saveErrorSnapshots = false, } = options;
+        }, saveErrorSnapshots = false, id, } = options;
+        this.id = id ?? String(Statistics.id++);
+        this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
         this.log = (options.log ?? log_1.log).child({ prefix: 'Statistics' });
         this.errorTracker = new error_tracker_1.ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new error_tracker_1.ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });

package/enqueue_links/enqueue_links.d.ts

@@ -1,11 +1,16 @@
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
+import { EnqueueStrategy, type RobotsTxtFile } from '@crawlee/utils';
 import type { SetRequired } from 'type-fest';
 import type { Request } from '../request';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages';
 import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared';
+export { EnqueueStrategy };
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
-    /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
+    /**
+     * Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.
+     * When called from a crawler context, the limit is further capped by what's left of the crawler's
+     * {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} budget.
+     */
     limit?: number;
     /** An array of URLs to enqueue. */
     urls?: readonly string[];
@@ -153,59 +158,11 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * 1. based on robots.txt file,
      * 2. because they don't match enqueueLinks filters,
      * 3. or because the maxRequestsPerCrawl limit has been reached
-     */
-    onSkippedRequest?: SkippedRequestCallback;
-}
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
      *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     * When calling `enqueueLinks` through a crawler context, this callback runs in addition to (after) the
+     * crawler-level `onSkippedRequest`, it does not replace it.
      */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
+    onSkippedRequest?: SkippedRequestCallback;
 }
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
@@ -244,7 +201,7 @@ export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptio
  *   request domain, or a redirected one
  * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
  */
-export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
+export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string;
 /**
  * @internal
  */

package/enqueue_links/enqueue_links.js

@@ -4,62 +4,12 @@ exports.EnqueueStrategy = void 0;
 exports.enqueueLinks = enqueueLinks;
 exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
 const tslib_1 = require("tslib");
+const utils_1 = require("@crawlee/utils");
+Object.defineProperty(exports, "EnqueueStrategy", { enumerable: true, get: function () { return utils_1.EnqueueStrategy; } });
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const tldts_1 = require("tldts");
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const shared_1 = require("./shared");
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-var EnqueueStrategy;
-(function (EnqueueStrategy) {
-    /**
-     * Matches any URLs found
-     */
-    EnqueueStrategy["All"] = "all";
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameHostname"] = "same-hostname";
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameDomain"] = "same-domain";
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    EnqueueStrategy["SameOrigin"] = "same-origin";
-})(EnqueueStrategy || (exports.EnqueueStrategy = EnqueueStrategy = {}));
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
@@ -109,7 +59,7 @@ async function enqueueLinks(options) {
         exclude: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('glob'), ow_1.default.object.hasKeys('regexp'))),
         regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
         transformRequestFunction: ow_1.default.optional.function,
-        strategy: ow_1.default.optional.string.oneOf(Object.values(EnqueueStrategy)),
+        strategy: ow_1.default.optional.string.oneOf(Object.values(utils_1.EnqueueStrategy)),
         waitForAllRequestsToBeAdded: ow_1.default.optional.boolean,
     }));
     const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
@@ -136,19 +86,19 @@ async function enqueueLinks(options) {
         urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
     }
     if (!urlPatternObjects.length) {
-        options.strategy ?? (options.strategy = EnqueueStrategy.SameHostname);
+        options.strategy ?? (options.strategy = utils_1.EnqueueStrategy.SameHostname);
     }
     const enqueueStrategyPatterns = [];
     if (options.baseUrl) {
         const url = new URL(options.baseUrl);
         switch (options.strategy) {
-            case EnqueueStrategy.SameHostname:
+            case utils_1.EnqueueStrategy.SameHostname:
                 // We need to get the origin of the passed in domain in the event someone sets baseUrl
                 // to an url like https://example.com/deep/default/path and one of the found urls is an
                 // absolute relative path (/path/to/page)
                 enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
                 break;
-            case EnqueueStrategy.SameDomain: {
+            case utils_1.EnqueueStrategy.SameDomain: {
                 // Get the actual hostname from the base url
                 const baseUrlHostname = (0, tldts_1.getDomain)(url.hostname, { mixedInputs: false });
                 if (baseUrlHostname) {
@@ -163,12 +113,12 @@ async function enqueueLinks(options) {
                 }
                 break;
             }
-            case EnqueueStrategy.SameOrigin: {
+            case utils_1.EnqueueStrategy.SameOrigin: {
                 // The same behavior as SameHostname, but respecting the protocol of the URL
                 enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
                 break;
             }
-            case EnqueueStrategy.All:
+            case utils_1.EnqueueStrategy.All:
             default:
                 enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
                 break;
@@ -249,19 +199,19 @@ function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestU
     const originalUrlOrigin = new URL(originalRequestUrl).origin;
     const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
     // We can assume users want to go off the domain in this case
-    if (enqueueStrategy === EnqueueStrategy.All) {
+    if (enqueueStrategy === utils_1.EnqueueStrategy.All) {
         return finalUrlOrigin;
     }
     // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
-    // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
-    // which is the intended behavior (since we went off domain)
-    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
+    // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning
+    // undefined here would disable the filtering entirely and enqueue every link on the redirected page
+    if (enqueueStrategy === utils_1.EnqueueStrategy.SameDomain) {
         const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
         const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
         if (originalHostname === finalHostname) {
             return finalUrlOrigin;
         }
-        return undefined;
+        return originalUrlOrigin;
     }
     // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
     // before actually finding the urls

package/enqueue_links/shared.d.ts

@@ -66,7 +66,7 @@ export declare function createRequestOptions(sources: (string | Record<string, u
 export interface RequestTransform {
     /**
      * @param original Request options to be modified.
-     * @returns The modified request options to enqueue.
+     * @returns The modified request options to enqueue, or any falsy value to skip the request.
      */
     (original: RequestOptions): RequestOptions | false | undefined | null;
 }

package/errors.d.ts

@@ -14,6 +14,26 @@ export declare class CriticalError extends NonRetryableError {
  */
 export declare class MissingRouteError extends CriticalError {
 }
+/**
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+export declare class RequestValidationError extends NonRetryableError {
+    readonly label: string | symbol;
+    readonly issues: readonly {
+        readonly message: string;
+        readonly path?: readonly (PropertyKey | {
+            key: PropertyKey;
+        })[];
+    }[];
+    constructor(label: string | symbol, issues: readonly {
+        readonly message: string;
+        readonly path?: readonly (PropertyKey | {
+            key: PropertyKey;
+        })[];
+    }[]);
+}
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

package/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SessionError = exports.RetryRequestError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
+exports.SessionError = exports.RetryRequestError = exports.RequestValidationError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
 /**
  * Errors of `NonRetryableError` type will never be retried by the crawler.
  */
@@ -20,6 +20,37 @@ exports.CriticalError = CriticalError;
 class MissingRouteError extends CriticalError {
 }
 exports.MissingRouteError = MissingRouteError;
+/**
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+class RequestValidationError extends NonRetryableError {
+    constructor(label, issues) {
+        const details = issues
+            .map((issue) => {
+            const path = (issue.path ?? [])
+                .map((segment) => (typeof segment === 'object' ? segment.key : segment))
+                .join('.');
+            return `- ${path ? `${path}: ` : ''}${issue.message}`;
+        })
+            .join('\n');
+        super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
+        Object.defineProperty(this, "label", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: label
+        });
+        Object.defineProperty(this, "issues", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: issues
+        });
+    }
+}
+exports.RequestValidationError = RequestValidationError;
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

package/http_clients/got-scraping-http-client.js

@@ -22,6 +22,14 @@ class GotScrapingHttpClient {
         });
         return {
             ...gotResult,
+            complete: gotResult.complete,
+            headers: gotResult.headers,
+            ip: gotResult.ip,
+            redirectUrls: gotResult.redirectUrls,
+            statusCode: gotResult.statusCode,
+            statusMessage: gotResult.statusMessage,
+            trailers: gotResult.trailers,
+            url: gotResult.url,
             body: gotResult.body,
             request: { url: request.url, ...gotResult.request },
         };

package/index.mjs

@@ -46,6 +46,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -73,6 +74,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -93,5 +95,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.9",
+    "version": "3.18.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -57,12 +57,13 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
         "@apify/pseudo_url": "^2.0.30",
-        "@apify/timeout": "^0.3.0",
+        "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.9",
-        "@crawlee/types": "3.17.1-beta.9",
-        "@crawlee/utils": "3.17.1-beta.9",
+        "@crawlee/memory-storage": "3.18.0",
+        "@crawlee/types": "3.18.0",
+        "@crawlee/utils": "3.18.0",
         "@sapphire/async-queue": "^1.5.1",
+        "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
@@ -83,5 +84,5 @@
             }
         }
     },
-    "gitHead": "c547ce4489d8617620a51516cc885187503f3136"
+    "gitHead": "49c115e1ce3b3fbf2bef61f16bffeadfdcf5bd19"
 }