@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.22

package/configuration.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Configuration = void 0;
+const consts_1 = require("@apify/consts");
+const storage_local_1 = require("@apify/storage-local");
+const async_hooks_1 = require("async_hooks");
+const events_1 = require("events");
+const typedefs_1 = require("./typedefs");
+const events_2 = require("./events");
+/**
+ * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
+ *
+ * 1. When using `Apify` class, we can get the instance configuration via `sdk.config`
+ *     ```js
+ *     const { Apify } = require('apify');
+ *
+ *     const sdk = new Apify({ token: '123' });
+ *     console.log(sdk.config.get('token')); // '123'
+ *     ```
+ * 2. To get the global configuration (singleton instance). It will respect the environment variables.
+ *     ```js
+ *     // returns the token from APIFY_TOKEN env var
+ *     console.log(Configuration.getGlobalConfig().get('token'));
+ *     ```
+ *
+ * ## Supported Configuration Options
+ *
+ * Key | Environment Variable | Default Value
+ * ---|---|---
+ * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
+ * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
+ * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
+ * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
+ *
+ * ## Advanced Configuration Options
+ *
+ * Key | Environment Variable | Default Value
+ * ---|---|---
+ * `actorId` | `APIFY_ACTOR_ID` | -
+ * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
+ * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
+ * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
+ * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
+ * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
+ * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
+ * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
+ * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
+ * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
+ *
+ * ## Not Supported environment variables
+ *
+ * - `MEMORY_MBYTES`
+ * - `HEADLESS`
+ * - `XVFB`
+ * - `CHROME_EXECUTABLE_PATH`
+ */
+class Configuration {
+    /**
+     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
+     */
+    constructor(options = {}) {
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "services", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        this.options = new Map((0, typedefs_1.entries)(options));
+        // Increase the global limit for event emitter memory leak warnings.
+        events_1.EventEmitter.defaultMaxListeners = 50;
+    }
+    /**
+     * Returns configured value. First checks the environment variables, then provided configuration,
+     * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
+     * in the above section.
+     */
+    get(key, defaultValue) {
+        // prefer env vars
+        const envKey = Configuration.ENV_MAP_REVERSED[key] ?? key;
+        const envValue = process.env[envKey];
+        if (envValue != null) {
+            return this._castEnvValue(key, envValue);
+        }
+        // check instance level options
+        if (this.options.has(key)) {
+            return this.options.get(key);
+        }
+        // fallback to defaults
+        return (defaultValue ?? Configuration.DEFAULTS[key] ?? envValue);
+    }
+    _castEnvValue(key, value) {
+        if (Configuration.INTEGER_VARS.includes(key)) {
+            return +value;
+        }
+        if (Configuration.BOOLEAN_VARS.includes(key)) {
+            // 0, false and empty string are considered falsy values
+            return !['0', 'false', ''].includes(String(value).toLowerCase());
+        }
+        return value;
+    }
+    /**
+     * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
+     * To reset a value, we can omit the `value` argument or pass `undefined` there.
+     */
+    set(key, value) {
+        this.options.set(key, value);
+    }
+    /**
+     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
+     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
+     * return the same client instance.
+     *
+     * Caching works based on the API URL and token, so calling this method with different options will return
+     * multiple instances, one for each variant of the options.
+     * @internal
+     */
+    getStorageClient() {
+        if (this.options.has('storageClient')) {
+            return this.options.get('storageClient');
+        }
+        const options = this.options.get('storageClientOptions');
+        return this.createStorageLocal(options);
+    }
+    getEventManager() {
+        if (this.options.has('eventManager')) {
+            return this.options.get('eventManager');
+        }
+        if (this.services.has('eventManager')) {
+            return this.services.get('eventManager');
+        }
+        const eventManager = new events_2.LocalEventManager(this);
+        this.services.set('eventManager', eventManager);
+        return eventManager;
+    }
+    /**
+     * Creates an instance of ApifyStorageLocal using options as defined in the environment variables or in this `Configuration` instance.
+     * @internal
+     */
+    createStorageLocal(options = {}) {
+        const cacheKey = `StorageLocal~${JSON.stringify(options)}`;
+        if (this.services.has(cacheKey)) {
+            return this.services.get(cacheKey);
+        }
+        const storage = new storage_local_1.ApifyStorageLocal(options);
+        this.services.set(cacheKey, storage);
+        process.on('exit', () => {
+            // TODO this is not public API, need to update storage local with some teardown
+            storage.dbConnections.closeAllConnections();
+        });
+        return storage;
+    }
+    useStorageClient(client) {
+        this.options.set('storageClient', client);
+    }
+    useEventManager(events) {
+        this.options.set('eventManager', events);
+    }
+    /**
+     * Returns the global configuration instance. It will respect the environment variables.
+     * When used inside SDK instance (`sdk.main()`), the return value is equal to `sdk.config`.
+     *
+     */
+    static getGlobalConfig() {
+        if (Configuration.storage.getStore()) {
+            return Configuration.storage.getStore();
+        }
+        Configuration.globalConfig ?? (Configuration.globalConfig = new Configuration());
+        return Configuration.globalConfig;
+    }
+    /**
+     * Gets default {@link StorageClient} instance.
+     */
+    static getStorageClient() {
+        return this.getGlobalConfig().getStorageClient();
+    }
+    /**
+     * Gets default {@link EventManager} instance.
+     */
+    static getEventManager() {
+        return this.getGlobalConfig().getEventManager();
+    }
+    /**
+     * Resets global configuration instance. The default instance holds configuration based on env vars,
+     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
+     */
+    static resetGlobalState() {
+        delete this.globalConfig;
+    }
+}
+exports.Configuration = Configuration;
+/**
+ * Maps environment variables to config keys (e.g. `APIFY_PROXY_PORT` to `proxyPort`)
+ */
+Object.defineProperty(Configuration, "ENV_MAP", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        // TODO prefix once we have a package name
+        AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
+        APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId',
+        APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
+        APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
+        APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis',
+        APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
+        APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis',
+        APIFY_INPUT_KEY: 'inputKey',
+        APIFY_ACTOR_ID: 'actorId',
+        APIFY_ACTOR_RUN_ID: 'actorRunId',
+        APIFY_ACTOR_TASK_ID: 'actorTaskId',
+        APIFY_CONTAINER_PORT: 'containerPort',
+        APIFY_CONTAINER_URL: 'containerUrl',
+        APIFY_PROXY_HOSTNAME: 'proxyHostname',
+        APIFY_PROXY_PASSWORD: 'proxyPassword',
+        APIFY_PROXY_PORT: 'proxyPort',
+        // not supported, use env vars directly:
+        // APIFY_MEMORY_MBYTES: 'memoryMbytes',
+        // APIFY_HEADLESS: 'headless',
+        // APIFY_XVFB: 'xvfb',
+        // APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
+    }
+});
+/**
+ * Maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`).
+ */
+Object.defineProperty(Configuration, "ENV_MAP_REVERSED", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: (0, typedefs_1.entries)(Configuration.ENV_MAP).reduce((obj, [key, value]) => {
+        obj[value] = key;
+        return obj;
+    }, {})
+});
+Object.defineProperty(Configuration, "BOOLEAN_VARS", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: []
+});
+Object.defineProperty(Configuration, "INTEGER_VARS", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: ['proxyPort', 'memoryMbytes', 'containerPort']
+});
+Object.defineProperty(Configuration, "DEFAULTS", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        defaultKeyValueStoreId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
+        defaultDatasetId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_DATASET_ID],
+        defaultRequestQueueId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
+        maxUsedCpuRatio: 0.95,
+        availableMemoryRatio: 0.25,
+        storageClientOptions: {},
+        inputKey: 'INPUT',
+        proxyHostname: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_HOSTNAME],
+        proxyPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_PORT],
+        containerPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_PORT],
+        containerUrl: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_URL],
+        metamorphAfterSleepMillis: 300000,
+        persistStateIntervalMillis: 60000,
+        systemInfoIntervalMillis: 60000,
+    }
+});
+/**
+ * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
+ * @internal
+ */
+Object.defineProperty(Configuration, "storage", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: new async_hooks_1.AsyncLocalStorage()
+});
+//# sourceMappingURL=configuration.js.map

package/configuration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;AAAA,0CAAyD;AACzD,wDAAmF;AACnF,6CAAgD;AAChD,mCAAsC;AACtC,yCAAiD;AAEjD,qCAA2D;AA2B3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAa,aAAa;IAsEtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAP9C;;;;;WAAmG;QACnG;;;;mBAAmB,IAAI,GAAG,EAAmB;WAAC;QAO1C,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;QAEzC,oEAAoE;QACpE,qBAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,kBAAkB;QAClB,MAAM,MAAM,GAAG,aAAa,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErC,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAEO,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACnF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAQ,CAAC;IACnD,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,UAAoC,EAAE;QACrD,MAAM,QAAQ,GAAG,gBAAgB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE3D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAsB,CAAC;SAC3D;QAED,MAAM,OAAO,GAAG,IAAI,iCAAiB,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,+EAA+E;YAC/E,OAAO,CAAC,aAAa,CAAC,mBAAmB,EAAE,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;;AA7NL,sCA8NC;AA7NG;;GAEG;AACH;;;;WAAyB;QACrB,0CAA0C;QAC1C,sBAAsB,EAAE,sBAAsB;QAE9C,wBAAwB,EAAE,kBAAkB;QAC5C,gCAAgC,EAAE,wBAAwB;QAC1D,8BAA8B,EAAE,uBAAuB;QACvD,kCAAkC,EAAE,2BAA2B;QAC/D,mCAAmC,EAAE,4BAA4B;QACjE,kCAAkC,EAAE,4BAA4B;QAChE,eAAe,EAAE,UAAU;QAC3B,cAAc,EAAE,SAAS;QACzB,kBAAkB,EAAE,YAAY;QAChC,mBAAmB,EAAE,aAAa;QAClC,oBAAoB,EAAE,eAAe;QACrC,mBAAmB,EAAE,cAAc;QACnC,oBAAoB,EAAE,eAAe;QACrC,oBAAoB,EAAE,eAAe;QACrC,gBAAgB,EAAE,WAAW;QAE7B,wCAAwC;QACxC,uCAAuC;QACvC,8BAA8B;QAC9B,sBAAsB;QACtB,wDAAwD;KAC3D;GAAC;AAEF;;GAEG;AACH;;;;WAAkC,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC1F,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACjB,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAA4B,CAAC;GAAC;AAEjC;;;;WAAwC,EAAE;GAAC;AAE3C;;;;WAA8B,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,CAAC;GAAC;AAE7E;;;;WAA0B;QACtB,sBAAsB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,0BAA0B,CAAC;QAC3E,gBAAgB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,kBAAkB,CAAC;QAC7D,qBAAqB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,wBAAwB,CAAC;QACxE,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,QAAQ,EAAE,OAAO;QACjB,aAAa,EAAE,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACtD,SAAS,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,UAAU,CAAC;QAC/C,aAAa,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACvD,YAAY,EAAE,uBAAc,CAAC,iBAAQ,CAAC,aAAa,CAAC;QACpD,yBAAyB,EAAE,MAAO;QAClC,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,+BAAiB,EAAiB;GAAC"}

package/constants.d.ts

@@ -0,0 +1,3 @@
+/** @internal */
+export declare const STATUS_CODES_BLOCKED: number[];
+//# sourceMappingURL=constants.d.ts.map

package/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,eAAO,MAAM,oBAAoB,UAAkB,CAAC"}

package/constants.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STATUS_CODES_BLOCKED = void 0;
+/** @internal */
+exports.STATUS_CODES_BLOCKED = [401, 403, 429];
+//# sourceMappingURL=constants.js.map

package/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":";;;AAAA,gBAAgB;AACH,QAAA,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC"}

package/crawlers/crawler_commons.d.ts

@@ -0,0 +1,25 @@
+import { Log } from '../log';
+import { ProxyInfo } from '../proxy_configuration';
+import { Request } from '../request';
+import { Session } from '../session_pool/session';
+export interface CrawlingContext extends Record<PropertyKey, unknown> {
+    id: string;
+    /**
+     * The original {@link Request} object.
+     */
+    request: Request;
+    session?: Session;
+    /**
+     * An object with information about currently used proxy by the crawler
+     * and configured by the {@link ProxyConfiguration} class.
+     */
+    proxyInfo?: ProxyInfo;
+    log: Log;
+}
+export interface CrawlerHandleFailedRequestInput extends CrawlingContext {
+    /**
+     * The Error thrown by `handleRequestFunction`.
+     */
+    error: Error;
+}
+//# sourceMappingURL=crawler_commons.d.ts.map

package/crawlers/crawler_commons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,+BAAgC,SAAQ,eAAe;IACpE;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;CAChB"}

package/crawlers/crawler_commons.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=crawler_commons.js.map

package/crawlers/crawler_commons.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":""}

package/crawlers/crawler_extension.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
+ * @category Crawlers
+ * @ignore
+ */
+export declare abstract class CrawlerExtension {
+    name: string;
+    log: import("@apify/log/log").Log;
+    getCrawlerOptions(): Record<string, unknown>;
+}
+//# sourceMappingURL=crawler_extension.d.ts.map

package/crawlers/crawler_extension.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_extension.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,8BAAsB,gBAAgB;IAClC,IAAI,SAAyB;IAC7B,GAAG,+BAA2C;IAE9C,iBAAiB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAG/C"}

package/crawlers/crawler_extension.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlerExtension = void 0;
+const log_1 = require("../log");
+/**
+ * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
+ * @category Crawlers
+ * @ignore
+ */
+class CrawlerExtension {
+    constructor() {
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: log_1.log.child({ prefix: this.name })
+        });
+    }
+    getCrawlerOptions() {
+        throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`);
+    }
+}
+exports.CrawlerExtension = CrawlerExtension;
+//# sourceMappingURL=crawler_extension.js.map

package/crawlers/crawler_extension.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_extension.js","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":";;;AAAA,gCAA2C;AAE3C;;;;GAIG;AACH,MAAsB,gBAAgB;IAAtC;QACI;;;;mBAAO,IAAI,CAAC,WAAW,CAAC,IAAI;WAAC;QAC7B;;;;mBAAM,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;WAAC;IAKlD,CAAC;IAHG,iBAAiB;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,kDAAkD,CAAC,CAAC;IACpF,CAAC;CACJ;AAPD,4CAOC"}

package/crawlers/crawler_utils.d.ts

@@ -0,0 +1,20 @@
+import { Session } from '../session_pool/session';
+/**
+ * Handles timeout request
+ * @internal
+ */
+export declare function handleRequestTimeout({ session, errorMessage }: {
+    session?: Session;
+    errorMessage: string;
+}): void;
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ * @internal
+ */
+export declare function mergeCookies(url: string, sourceCookies: string[]): string;
+/**
+ * @internal
+ */
+export declare function diffCookies(url: string, cookieString1?: string, cookieString2?: string): string;
+//# sourceMappingURL=crawler_utils.d.ts.map

package/crawlers/crawler_utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,SAAK,EAAE,aAAa,SAAK,GAAG,MAAM,CAmBvF"}

package/crawlers/crawler_utils.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diffCookies = exports.mergeCookies = exports.handleRequestTimeout = void 0;
+const tough_cookie_1 = require("tough-cookie");
+const log_1 = require("../log");
+/**
+ * Handles timeout request
+ * @internal
+ */
+function handleRequestTimeout({ session, errorMessage }) {
+    session?.markBad();
+    const timeoutMillis = errorMessage.match(/(\d+)\s?ms/)?.[1]; // first capturing group
+    const timeoutSecs = Number(timeoutMillis) / 1000;
+    throw new Error(`Navigation timed out after ${timeoutSecs} seconds.`);
+}
+exports.handleRequestTimeout = handleRequestTimeout;
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ * @internal
+ */
+function mergeCookies(url, sourceCookies) {
+    const jar = new tough_cookie_1.CookieJar();
+    // ignore empty cookies
+    for (const sourceCookieString of sourceCookies.filter((c) => c)) {
+        const cookies = sourceCookieString.split(/ *; */).filter((c) => c); // ignore extra spaces
+        for (const cookieString of cookies) {
+            const cookie = tough_cookie_1.Cookie.parse(cookieString);
+            const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
+                return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
+            });
+            if (similarKeyCookie) {
+                log_1.log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+            }
+            jar.setCookieSync(cookie, url);
+        }
+    }
+    return jar.getCookieStringSync(url);
+}
+exports.mergeCookies = mergeCookies;
+/**
+ * @internal
+ */
+function diffCookies(url, cookieString1 = '', cookieString2 = '') {
+    if (cookieString1 === cookieString2 || !cookieString2) {
+        return '';
+    }
+    if (!cookieString1) {
+        return cookieString2;
+    }
+    const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
+    const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
+    const added = cookies2.filter((newCookie) => {
+        return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
+    });
+    const jar = new tough_cookie_1.CookieJar();
+    added.forEach((cookie) => jar.setCookieSync(cookie, url));
+    return jar.getCookieStringSync(url);
+}
+exports.diffCookies = diffCookies;
+//# sourceMappingURL=crawler_utils.js.map

package/crawlers/crawler_utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,+CAAiD;AACjD,gCAA6B;AAG7B;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AAC1E,CAAC;AALD,oDAKC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE;QAC7D,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB;QAE1F,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE;YAChC,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE;gBAClB,SAAG,CAAC,UAAU,CAAC,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC;aAC1H;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC;KACJ;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAtBD,oCAsBC;AAED;;GAEG;AACH,SAAgB,WAAW,CAAC,GAAW,EAAE,aAAa,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE;IAC3E,IAAI,aAAa,KAAK,aAAa,IAAI,CAAC,aAAa,EAAE;QACnD,OAAO,EAAE,CAAC;KACb;IAED,IAAI,CAAC,aAAa,EAAE;QAChB,OAAO,aAAa,CAAC;KACxB;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;IACrH,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;IAErH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAC5B,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IAE1D,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAnBD,kCAmBC"}

package/crawlers/index.d.ts

@@ -0,0 +1,5 @@
+export * from './crawler_commons';
+export * from './crawler_extension';
+export * from './crawler_utils';
+export * from './statistics';
+//# sourceMappingURL=index.d.ts.map

package/crawlers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC"}

package/crawlers/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./crawler_commons"), exports);
+tslib_1.__exportStar(require("./crawler_extension"), exports);
+tslib_1.__exportStar(require("./crawler_utils"), exports);
+tslib_1.__exportStar(require("./statistics"), exports);
+//# sourceMappingURL=index.js.map

package/crawlers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,8DAAoC;AACpC,0DAAgC;AAChC,uDAA6B"}

package/crawlers/statistics.d.ts

@@ -0,0 +1,147 @@
+import { Configuration } from '../configuration';
+/**
+ * @ignore
+ */
+declare class Job {
+    private lastRunAt;
+    private runs;
+    private durationMillis?;
+    run(): number;
+    finish(): number;
+    retryCount(): number;
+}
+/**
+ * The statistics class provides an interface to collecting and logging run
+ * statistics for requests.
+ *
+ * All statistic information is saved on key value store
+ * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * migrations and abort/resurrect
+ *
+ * @category Crawlers
+ */
+export declare class Statistics {
+    private static id;
+    /**
+     * Statistic instance id.
+     */
+    readonly id: number;
+    /**
+     * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
+     */
+    state: StatisticState;
+    /**
+     * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
+     */
+    readonly requestRetryHistogram: number[];
+    private keyValueStore;
+    private persistStateKey;
+    private logIntervalMillis;
+    private logMessage;
+    private listener;
+    private requestsInProgress;
+    private readonly log;
+    private instanceStart;
+    private logInterval;
+    private events;
+    /**
+     * @internal
+     */
+    constructor(options?: StatisticsOptions);
+    /**
+     * Set the current statistic instance to pristine values
+     */
+    reset(): void;
+    /**
+     * Starts a job
+     * @ignore
+     */
+    startJob(id: number | string): void;
+    /**
+     * Mark job as finished and sets the state
+     * @ignore
+     */
+    finishJob(id: number | string): void;
+    /**
+     * Mark job as failed and sets the state
+     * @ignore
+     */
+    failJob(id: number | string): void;
+    /**
+     * Calculate the current statistics
+     */
+    calculate(): {
+        requestAvgFailedDurationMillis: number;
+        requestAvgFinishedDurationMillis: number;
+        requestsFinishedPerMinute: number;
+        requestsFailedPerMinute: number;
+        requestTotalDurationMillis: number;
+        requestsTotal: number;
+        crawlerRuntimeMillis: number;
+    };
+    /**
+     * Initializes the key value store for persisting the statistics,
+     * displaying the current state in predefined intervals
+     */
+    startCapturing(): Promise<void>;
+    /**
+     * Stops logging and remove event listeners, then persist
+     */
+    stopCapturing(): Promise<void>;
+    protected _saveRetryCountForJob(job: Job): void;
+    /**
+     * Persist internal state to the key value store
+     */
+    persistState(): Promise<void>;
+    /**
+     * Loads the current statistic from the key value store if any
+     */
+    protected _maybeLoadStatistics(): Promise<void>;
+    protected _teardown(): void;
+    /**
+     * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
+     * or through `keyValueStore.setValue('KEY', statsInstance)`
+     */
+    toJSON(): StatisticPersistedState;
+}
+interface StatisticsOptions {
+    logIntervalSecs?: number;
+    logMessage?: string;
+    config?: Configuration;
+}
+/**
+ * Format of the persisted stats
+ */
+export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
+    requestRetryHistogram: number[];
+    statsId: number;
+    requestAvgFailedDurationMillis: number;
+    requestAvgFinishedDurationMillis: number;
+    requestsFinishedPerMinute: number;
+    requestsFailedPerMinute: number;
+    requestTotalDurationMillis: number;
+    requestsTotal: number;
+    crawlerRuntimeMillis: number;
+    crawlerLastStartTimestamp: number;
+    statsPersistedAt: string;
+}
+/**
+ * Contains the statistics state
+ */
+export interface StatisticState {
+    requestsFinished: number;
+    requestsFailed: number;
+    requestsRetries: number;
+    requestsFailedPerMinute: number;
+    requestsFinishedPerMinute: number;
+    requestMinDurationMillis: number;
+    requestMaxDurationMillis: number;
+    requestTotalFailedDurationMillis: number;
+    requestTotalFinishedDurationMillis: number;
+    crawlerStartedAt: Date | string | null;
+    crawlerFinishedAt: Date | string | null;
+    crawlerRuntimeMillis: number;
+    statsPersistedAt: Date | string | null;
+}
+export {};
+//# sourceMappingURL=statistics.d.ts.map