@crawlee/core 3.16.1-beta.8 → 3.17.0

package/autoscaling/system_status.js

@@ -2,9 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SystemStatus = void 0;
 const tslib_1 = require("tslib");
-const utils_1 = require("@crawlee/utils");
 const ow_1 = tslib_1.__importDefault(require("ow"));
+const load_signal_1 = require("./load_signal");
 const snapshotter_1 = require("./snapshotter");
+/** The four built-in signal names that map to typed `SystemInfo` fields. */
+const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
 /**
  * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
  * It only exposes two functions {@link SystemStatus.getCurrentStatus}
@@ -35,31 +37,24 @@ class SystemStatus {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "maxMemoryOverloadedRatio", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxEventLoopOverloadedRatio", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxCpuOverloadedRatio", {
+        Object.defineProperty(this, "snapshotter", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "maxClientOverloadedRatio", {
+        Object.defineProperty(this, "signals", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "snapshotter", {
+        /**
+         * Per-signal ratio overrides. The built-in four get their overrides from
+         * the legacy `max*OverloadedRatio` options; custom signals use their own
+         * `overloadedRatio`.
+         */
+        Object.defineProperty(this, "ratioOverrides", {
             enumerable: true,
             configurable: true,
             writable: true,
@@ -72,15 +67,21 @@ class SystemStatus {
             maxCpuOverloadedRatio: ow_1.default.optional.number,
             maxClientOverloadedRatio: ow_1.default.optional.number,
             snapshotter: ow_1.default.optional.object,
+            loadSignals: ow_1.default.optional.array,
             config: ow_1.default.optional.object,
         }));
-        const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
+        const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], config, } = options;
         this.currentHistoryMillis = currentHistorySecs * 1000;
-        this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
-        this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
-        this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
-        this.maxClientOverloadedRatio = maxClientOverloadedRatio;
         this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
+        // Built-in signals from the snapshotter + any custom signals
+        this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
+        // Allow legacy options to override the built-in signal ratios
+        this.ratioOverrides = {
+            memInfo: maxMemoryOverloadedRatio,
+            eventLoopInfo: maxEventLoopOverloadedRatio,
+            cpuInfo: maxCpuOverloadedRatio,
+            clientInfo: maxClientOverloadedRatio,
+        };
     }
     /**
      * Returns an {@link SystemInfo} object with the following structure:
@@ -124,81 +125,33 @@ class SystemStatus {
      * Returns a system status object.
      */
     _isSystemIdle(sampleDurationMillis) {
-        const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
-        const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
-        const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
-        const clientInfo = this._isClientOverloaded(sampleDurationMillis);
-        return {
-            isSystemIdle: !memInfo.isOverloaded &&
-                !eventLoopInfo.isOverloaded &&
-                !cpuInfo.isOverloaded &&
-                !clientInfo.isOverloaded,
-            memInfo,
-            eventLoopInfo,
-            cpuInfo,
-            clientInfo,
+        const result = {
+            isSystemIdle: true,
+            memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
         };
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the memory has been overloaded in the last sampleDurationMillis.
-     */
-    _isMemoryOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the event loop has been overloaded in the last sampleDurationMillis.
-     */
-    _isEventLoopOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the CPU has been overloaded in the last sampleDurationMillis.
-     */
-    _isCpuOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
-    }
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the client has been overloaded in the last sampleDurationMillis.
-     */
-    _isClientOverloaded(sampleDurationMillis) {
-        const sample = this.snapshotter.getClientSample(sampleDurationMillis);
-        return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
-    }
-    /**
-     * Returns an object with sample information and an isOverloaded property
-     * set to true if at least the ratio of snapshots in the sample are overloaded.
-     */
-    _isSampleOverloaded(sample, ratio) {
-        if (sample.length === 0) {
-            return {
-                isOverloaded: false,
-                limitRatio: ratio,
-                actualRatio: 0,
-            };
+        let loadSignalInfo;
+        for (const signal of this.signals) {
+            const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
+            const sample = signal.getSample(sampleDurationMillis);
+            const info = (0, load_signal_1.evaluateLoadSignalSample)(sample, ratio);
+            if (info.isOverloaded) {
+                result.isSystemIdle = false;
+            }
+            if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
+                result[signal.name] = info;
+            }
+            else {
+                loadSignalInfo ?? (loadSignalInfo = {});
+                loadSignalInfo[signal.name] = info;
+            }
         }
-        const weights = [];
-        const values = [];
-        for (let i = 1; i < sample.length; i++) {
-            const previous = sample[i - 1];
-            const current = sample[i];
-            const weight = +current.createdAt - +previous.createdAt;
-            weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
-            values.push(+current.isOverloaded);
+        if (loadSignalInfo) {
+            result.loadSignalInfo = loadSignalInfo;
         }
-        const wAvg = sample.length === 1 ? +sample[0].isOverloaded : (0, utils_1.weightedAvg)(values, weights);
-        return {
-            isOverloaded: wAvg > ratio,
-            limitRatio: ratio,
-            actualRatio: Math.round(wAvg * 1000) / 1000,
-        };
+        return result;
     }
 }
 exports.SystemStatus = SystemStatus;
-//# sourceMappingURL=system_status.js.map

package/configuration.d.ts

@@ -294,4 +294,3 @@ export declare class Configuration {
     static resetGlobalState(): void;
     protected buildOptions(options: ConfigurationOptions): void;
 }
-//# sourceMappingURL=configuration.d.ts.map

package/configuration.js

@@ -350,4 +350,3 @@ Object.defineProperty(Configuration, "storage", {
     writable: true,
     value: new node_async_hooks_1.AsyncLocalStorage()
 });
-//# sourceMappingURL=configuration.js.map

package/cookie_utils.d.ts

@@ -40,4 +40,3 @@ export declare function cookieStringToToughCookie(cookieString: string): CookieO
  * @internal
  */
 export declare function mergeCookies(url: string, sourceCookies: string[]): string;
-//# sourceMappingURL=cookie_utils.d.ts.map

package/cookie_utils.js

@@ -118,4 +118,3 @@ function mergeCookies(url, sourceCookies) {
     }
     return jar.getCookieStringSync(url);
 }
-//# sourceMappingURL=cookie_utils.js.map

package/crawlers/crawler_commons.d.ts

@@ -192,4 +192,3 @@ export declare class RequestHandlerResult {
     private getKeyValueStoreChangedValue;
     private setKeyValueStoreChangedValue;
 }
-//# sourceMappingURL=crawler_commons.d.ts.map

package/crawlers/crawler_commons.js

@@ -169,4 +169,3 @@ class RequestHandlerResult {
     }
 }
 exports.RequestHandlerResult = RequestHandlerResult;
-//# sourceMappingURL=crawler_commons.js.map

package/crawlers/crawler_extension.d.ts

@@ -9,4 +9,3 @@ export declare abstract class CrawlerExtension {
     log: Log;
     getCrawlerOptions(): Record<string, unknown>;
 }
-//# sourceMappingURL=crawler_extension.d.ts.map

package/crawlers/crawler_extension.js

@@ -27,4 +27,3 @@ class CrawlerExtension {
     }
 }
 exports.CrawlerExtension = CrawlerExtension;
-//# sourceMappingURL=crawler_extension.js.map

package/crawlers/crawler_utils.d.ts

@@ -7,4 +7,3 @@ export declare function handleRequestTimeout({ session, errorMessage }: {
     session?: Session;
     errorMessage: string;
 }): void;
-//# sourceMappingURL=crawler_utils.d.ts.map

package/crawlers/crawler_utils.js

@@ -12,4 +12,3 @@ function handleRequestTimeout({ session, errorMessage }) {
     const timeoutSecs = Number(timeoutMillis) / 1000;
     throw new timeout_1.TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
 }
-//# sourceMappingURL=crawler_utils.js.map

package/crawlers/error_snapshotter.d.ts

@@ -56,4 +56,3 @@ export declare class ErrorSnapshotter {
     generateFilename(error: ErrnoException): string;
 }
 export {};
-//# sourceMappingURL=error_snapshotter.d.ts.map

package/crawlers/error_snapshotter.js

@@ -145,4 +145,3 @@ Object.defineProperty(ErrorSnapshotter, "SNAPSHOT_PREFIX", {
     writable: true,
     value: 'ERROR_SNAPSHOT'
 });
-//# sourceMappingURL=error_snapshotter.js.map

package/crawlers/error_tracker.d.ts

@@ -51,4 +51,3 @@ export declare class ErrorTracker {
     captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext): Promise<void>;
     reset(): void;
 }
-//# sourceMappingURL=error_tracker.d.ts.map

package/crawlers/error_tracker.js

@@ -328,4 +328,3 @@ class ErrorTracker {
 }
 exports.ErrorTracker = ErrorTracker;
 _ErrorTracker_options = new WeakMap();
-//# sourceMappingURL=error_tracker.js.map

package/crawlers/index.d.ts

@@ -4,4 +4,3 @@ export * from './crawler_utils';
 export * from './statistics';
 export * from './error_tracker';
 export * from './error_snapshotter';
-//# sourceMappingURL=index.d.ts.map

package/crawlers/index.js

@@ -7,4 +7,3 @@ tslib_1.__exportStar(require("./crawler_utils"), exports);
 tslib_1.__exportStar(require("./statistics"), exports);
 tslib_1.__exportStar(require("./error_tracker"), exports);
 tslib_1.__exportStar(require("./error_snapshotter"), exports);
-//# sourceMappingURL=index.js.map

package/crawlers/statistics.d.ts

@@ -201,4 +201,3 @@ export interface StatisticState {
     retryErrors: Record<string, unknown>;
     requestsWithStatusCode: Record<string, number>;
 }
-//# sourceMappingURL=statistics.d.ts.map

package/crawlers/statistics.js

@@ -451,4 +451,3 @@ Object.defineProperty(Statistics, "id", {
     writable: true,
     value: 0
 });
-//# sourceMappingURL=statistics.js.map

package/enqueue_links/enqueue_links.d.ts

@@ -139,6 +139,14 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * If provided, disallowed URLs will be ignored.
      */
     robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
+    /**
+     * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
+     * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
+     * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
+     */
+    respectRobotsTxtFile?: boolean | {
+        userAgent?: string;
+    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
      * This is currently fired for requests skipped
@@ -246,4 +254,3 @@ export interface ResolveBaseUrl {
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
-//# sourceMappingURL=enqueue_links.d.ts.map

package/enqueue_links/enqueue_links.js

@@ -95,6 +95,7 @@ async function enqueueLinks(options) {
         urls: ow_1.default.array.ofType(ow_1.default.string),
         requestQueue: ow_1.default.object.hasKeys('addRequestsBatched'),
         robotsTxtFile: ow_1.default.optional.object.hasKeys('isAllowed'),
+        respectRobotsTxtFile: ow_1.default.optional.any(ow_1.default.boolean, ow_1.default.object.exactShape({ userAgent: ow_1.default.optional.string })),
         onSkippedRequest: ow_1.default.optional.function,
         forefront: ow_1.default.optional.boolean,
         skipNavigation: ow_1.default.optional.boolean,
@@ -184,10 +185,11 @@ async function enqueueLinks(options) {
         }
     }
     let requestOptions = (0, shared_1.createRequestOptions)(urls, options);
-    if (robotsTxtFile) {
+    if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
+        const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
         const skippedRequests = [];
         requestOptions = requestOptions.filter((request) => {
-            if (robotsTxtFile.isAllowed(request.url)) {
+            if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
                 return true;
             }
             skippedRequests.push(request);
@@ -221,15 +223,14 @@ async function enqueueLinks(options) {
         await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
         return filtered;
     }
-    let requests = await createFilteredRequests();
-    if (typeof limit === 'number' && limit < requests.length) {
-        await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
-        requests = requests.slice(0, limit);
-    }
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
+    const { addedRequests, requestsOverLimit } = await requestQueue.addRequestsBatched(await createFilteredRequests(), {
         forefront,
         waitForAllRequestsToBeAdded,
+        maxNewRequests: limit,
     });
+    if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
+        await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
+    }
     return { processedRequests: addedRequests, unprocessedRequests: [] };
 }
 /**
@@ -272,4 +273,3 @@ function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestU
 function ignoreHttpSchema(pattern) {
     return pattern.replace(/^(https?):\/\//, 'http{s,}://');
 }
-//# sourceMappingURL=enqueue_links.js.map

package/enqueue_links/index.d.ts

@@ -1,3 +1,2 @@
 export * from './enqueue_links';
 export * from './shared';
-//# sourceMappingURL=index.d.ts.map

package/enqueue_links/index.js

@@ -3,4 +3,3 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
 tslib_1.__exportStar(require("./enqueue_links"), exports);
 tslib_1.__exportStar(require("./shared"), exports);
-//# sourceMappingURL=index.js.map

package/enqueue_links/shared.d.ts

@@ -70,4 +70,3 @@ export interface RequestTransform {
      */
     (original: RequestOptions): RequestOptions | false | undefined | null;
 }
-//# sourceMappingURL=shared.d.ts.map

package/enqueue_links/shared.js

@@ -221,4 +221,3 @@ function createPatternObjectMatcher(urlPatternObject) {
     }
     return { ...urlPatternObject, match };
 }
-//# sourceMappingURL=shared.js.map

package/errors.d.ts

@@ -30,4 +30,3 @@ export declare class RetryRequestError extends Error {
 export declare class SessionError extends RetryRequestError {
     constructor(message?: string);
 }
-//# sourceMappingURL=errors.d.ts.map

package/errors.js

@@ -42,4 +42,3 @@ class SessionError extends RetryRequestError {
     }
 }
 exports.SessionError = SessionError;
-//# sourceMappingURL=errors.js.map

package/events/event_manager.d.ts

@@ -49,4 +49,3 @@ export declare abstract class EventManager {
     waitForAllListenersToComplete(): Promise<boolean>;
 }
 export {};
-//# sourceMappingURL=event_manager.d.ts.map

package/events/event_manager.js

@@ -115,4 +115,3 @@ class EventManager {
     }
 }
 exports.EventManager = EventManager;
-//# sourceMappingURL=event_manager.js.map

package/events/index.d.ts

@@ -1,3 +1,2 @@
 export * from './event_manager';
 export * from './local_event_manager';
-//# sourceMappingURL=index.d.ts.map

package/events/index.js

@@ -3,4 +3,3 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
 tslib_1.__exportStar(require("./event_manager"), exports);
 tslib_1.__exportStar(require("./local_event_manager"), exports);
-//# sourceMappingURL=index.js.map

package/events/local_event_manager.d.ts

@@ -25,5 +25,5 @@ export declare class LocalEventManager extends EventManager {
     private createSystemInfo;
     private createCpuInfo;
     private createMemoryInfo;
+    private getMemoryInfo;
 }
-//# sourceMappingURL=local_event_manager.d.ts.map

package/events/local_event_manager.js

@@ -96,14 +96,9 @@ class LocalEventManager extends event_manager_1.EventManager {
     }
     async createMemoryInfo() {
         try {
-            if (this.config.get('systemInfoV2')) {
-                const memInfo = await (0, utils_1.getMemoryInfoV2)(await this.isContainerizedWrapper());
-                return {
-                    memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
-                };
-            }
-            const memInfo = await (0, utils_1.getMemoryInfo)();
+            const memInfo = await this.getMemoryInfo();
             return {
+                memTotalBytes: memInfo.totalBytes,
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
             };
         }
@@ -112,6 +107,11 @@ class LocalEventManager extends event_manager_1.EventManager {
             return {};
         }
     }
+    async getMemoryInfo() {
+        if (this.config.get('systemInfoV2')) {
+            return (0, utils_1.getMemoryInfoV2)(await this.isContainerizedWrapper());
+        }
+        return (0, utils_1.getMemoryInfo)();
+    }
 }
 exports.LocalEventManager = LocalEventManager;
-//# sourceMappingURL=local_event_manager.js.map

package/http_clients/base-http-client.d.ts

@@ -131,4 +131,3 @@ export interface BaseHttpClient {
  */
 export declare function processHttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'>({ searchParams, form, json, username, password, ...request }: HttpRequestOptions<TResponseType>): HttpRequest<TResponseType>;
 export {};
-//# sourceMappingURL=base-http-client.d.ts.map

package/http_clients/base-http-client.js

@@ -33,4 +33,3 @@ function processHttpRequestOptions({ searchParams, form, json, username, passwor
     }
     return { ...request, body, url, headers };
 }
-//# sourceMappingURL=base-http-client.js.map

package/http_clients/form-data-like.d.ts

@@ -64,4 +64,3 @@ export interface FormDataLike {
     readonly [Symbol.toStringTag]?: string;
 }
 export {};
-//# sourceMappingURL=form-data-like.d.ts.map

package/http_clients/form-data-like.js

@@ -3,4 +3,3 @@
  * This is copied from https://github.com/octet-stream/form-data-encoder
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=form-data-like.js.map

package/http_clients/got-scraping-http-client.d.ts

@@ -12,4 +12,3 @@ export declare class GotScrapingHttpClient implements BaseHttpClient {
      */
     stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
 }
-//# sourceMappingURL=got-scraping-http-client.d.ts.map

package/http_clients/got-scraping-http-client.js

@@ -74,4 +74,3 @@ class GotScrapingHttpClient {
     }
 }
 exports.GotScrapingHttpClient = GotScrapingHttpClient;
-//# sourceMappingURL=got-scraping-http-client.js.map

package/http_clients/index.d.ts

@@ -1,3 +1,2 @@
 export * from './base-http-client';
 export * from './got-scraping-http-client';
-//# sourceMappingURL=index.d.ts.map

package/http_clients/index.js

@@ -3,4 +3,3 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
 tslib_1.__exportStar(require("./base-http-client"), exports);
 tslib_1.__exportStar(require("./got-scraping-http-client"), exports);
-//# sourceMappingURL=index.js.map

package/index.d.ts

@@ -17,4 +17,3 @@ export * from './cookie_utils';
 export * from './recoverable_state';
 export { PseudoUrl } from '@apify/pseudo_url';
 export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -21,4 +21,3 @@ tslib_1.__exportStar(require("./cookie_utils"), exports);
 tslib_1.__exportStar(require("./recoverable_state"), exports);
 var pseudo_url_1 = require("@apify/pseudo_url");
 Object.defineProperty(exports, "PseudoUrl", { enumerable: true, get: function () { return pseudo_url_1.PseudoUrl; } });
-//# sourceMappingURL=index.js.map

package/index.mjs

@@ -26,6 +26,7 @@ export const LoggerJson = mod.LoggerJson;
 export const LoggerText = mod.LoggerText;
 export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
 export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
+export const MemoryLoadSignal = mod.MemoryLoadSignal;
 export const MissingRouteError = mod.MissingRouteError;
 export const NonRetryableError = mod.NonRetryableError;
 export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
@@ -53,6 +54,7 @@ export const Session = mod.Session;
 export const SessionError = mod.SessionError;
 export const SessionPool = mod.SessionPool;
 export const SitemapRequestList = mod.SitemapRequestList;
+export const SnapshotStore = mod.SnapshotStore;
 export const Snapshotter = mod.Snapshotter;
 export const Statistics = mod.Statistics;
 export const StorageManager = mod.StorageManager;
@@ -65,11 +67,15 @@ export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
 export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
 export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
 export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
+export const createClientLoadSignal = mod.createClientLoadSignal;
+export const createCpuLoadSignal = mod.createCpuLoadSignal;
 export const createDeserialize = mod.createDeserialize;
+export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
+export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
 export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
 export const getCookiesFromResponse = mod.getCookiesFromResponse;
 export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;

package/log.d.ts

@@ -2,4 +2,3 @@ import type { LoggerOptions } from '@apify/log';
 import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
 export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
 export type { LoggerOptions };
-//# sourceMappingURL=log.d.ts.map

package/log.js

@@ -9,4 +9,3 @@ Object.defineProperty(exports, "Logger", { enumerable: true, get: function () {
 Object.defineProperty(exports, "LoggerJson", { enumerable: true, get: function () { return log_1.LoggerJson; } });
 Object.defineProperty(exports, "LoggerText", { enumerable: true, get: function () { return log_1.LoggerText; } });
 Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function () { return log_1.LogLevel; } });
-//# sourceMappingURL=log.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.16.1-beta.8",
+    "version": "3.17.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,14 +59,14 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.16.1-beta.8",
-        "@crawlee/types": "3.16.1-beta.8",
-        "@crawlee/utils": "3.16.1-beta.8",
+        "@crawlee/memory-storage": "3.17.0",
+        "@crawlee/types": "3.17.0",
+        "@crawlee/utils": "3.17.0",
         "@sapphire/async-queue": "^1.5.1",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
-        "got-scraping": "^4.0.0",
+        "got-scraping": "^4.2.1",
         "json5": "^2.2.3",
         "minimatch": "^9.0.0",
         "ow": "^0.28.1",
@@ -83,5 +83,5 @@
             }
         }
     },
-    "gitHead": "dfb5c92b9494cf3536fa8ff7ee1fb1929b33e451"
+    "gitHead": "8f2663aedfd2280ca35b9185b4e287f65a2ce969"
 }

package/proxy_configuration.d.ts

@@ -244,4 +244,3 @@ export declare class ProxyConfiguration {
     protected _throwNoOptionsProvided(): never;
 }
 export {};
-//# sourceMappingURL=proxy_configuration.d.ts.map

package/proxy_configuration.js

@@ -336,4 +336,3 @@ class ProxyConfiguration {
     }
 }
 exports.ProxyConfiguration = ProxyConfiguration;
-//# sourceMappingURL=proxy_configuration.js.map

package/recoverable_state.d.ts

@@ -120,4 +120,3 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>> {
      */
     private loadSavedState;
 }
-//# sourceMappingURL=recoverable_state.d.ts.map

package/recoverable_state.js

@@ -197,4 +197,3 @@ class RecoverableState {
     }
 }
 exports.RecoverableState = RecoverableState;
-//# sourceMappingURL=recoverable_state.js.map