apify 3.1.1 → 3.1.2-beta.26

package/src/cli.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+
+// eslint-disable-next-line
+const importLocal = require('import-local');
+
+if (!importLocal(__filename)) {
+    // eslint-disable-next-line
+    require('apify-cli');
+}

package/src/configuration.ts

@@ -0,0 +1,202 @@
+import { ENV_VARS, LOCAL_ENV_VARS } from '@apify/consts';
+import { Configuration as CoreConfiguration } from '@crawlee/core';
+import type { ConfigurationOptions as CoreConfigurationOptions } from '@crawlee/core';
+
+export interface ConfigurationOptions extends CoreConfigurationOptions {
+    metamorphAfterSleepMillis?: number;
+    actorEventsWsUrl?: string;
+    token?: string;
+    actorId?: string;
+    actorRunId?: string;
+    actorTaskId?: string;
+    apiBaseUrl?: string;
+    containerPort?: number;
+    containerUrl?: string;
+    proxyHostname?: string;
+    proxyPassword?: string;
+    proxyPort?: number;
+    proxyStatusUrl?: string;
+    isAtHome?: boolean;
+    userId?: string;
+    inputSecretsPrivateKeyPassphrase?: string;
+    inputSecretsPrivateKeyFile?: string;
+}
+
+/**
+ * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
+ *
+ * 1. When using `Actor` class, we can get the instance configuration via `sdk.config`
+ *
+ *    ```javascript
+ *    import { Actor } from 'apify';
+ *    import { BasicCrawler } from 'crawlee';
+ *
+ *    const sdk = new Actor({ token: '123' });
+ *    console.log(sdk.config.get('token')); // '123'
+ *
+ *    const crawler = new BasicCrawler({
+ *        // ... crawler options
+ *    }, sdk.config);
+ *    ```
+ *
+ * 2. To get the global configuration (singleton instance). It will respect the environment variables.
+ *
+ *    ```javascript
+ *    import { BasicCrawler, Configuration } from 'crawlee';
+ *
+ *    // Get the global configuration
+ *    const config = Configuration.getGlobalConfig();
+ *    // Set the 'persistStateIntervalMillis' option
+ *    // of global configuration to 30 seconds
+ *    config.set('persistStateIntervalMillis', 30_000);
+ *
+ *    // No need to pass the configuration to the crawler,
+ *    // as it's using the global configuration by default
+ *    const crawler = new BasicCrawler();
+ *    ```
+ *
+ * ## Supported Configuration Options
+ *
+ * Key | Environment Variable | Default Value
+ * ---|---|---
+ * `memoryMbytes` | `APIFY_MEMORY_MBYTES` | -
+ * `headless` | `APIFY_HEADLESS` | -
+ * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
+ * `token` | `APIFY_TOKEN` | -
+ * `isAtHome` | `APIFY_IS_AT_HOME` | -
+ * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
+ * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
+ * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
+ *
+ * ## Advanced Configuration Options
+ *
+ * Key | Environment Variable | Default Value
+ * ---|---|---
+ * `actorEventsWsUrl` | `APIFY_ACTOR_EVENTS_WS_URL` | -
+ * `actorId` | `APIFY_ACTOR_ID` | -
+ * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
+ * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
+ * `apiBaseUrl` | `APIFY_API_BASE_URL` | `'https://api.apify.com'`
+ * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
+ * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
+ * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
+ * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
+ * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
+ * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
+ * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
+ * `proxyStatusUrl` | `APIFY_PROXY_STATUS_URL` | `'http://proxy.apify.com'`
+ * `userId` | `APIFY_USER_ID` | -
+ * `xvfb` | `APIFY_XVFB` | -
+ * `chromeExecutablePath` | `APIFY_CHROME_EXECUTABLE_PATH` | -
+ * `defaultBrowserPath` | `APIFY_DEFAULT_BROWSER_PATH` | -
+ */
+export class Configuration extends CoreConfiguration {
+    /** @inheritDoc */
+    static override globalConfig?: Configuration;
+
+    // maps environment variables to config keys (e.g. `APIFY_MEMORY_MBYTES` to `memoryMbytes`)
+    protected static override ENV_MAP = {
+        // regular crawlee env vars are also supported
+        ...super.ENV_MAP,
+
+        // support crawlee env vars prefixed with `APIFY_` too
+        APIFY_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
+        APIFY_PURGE_ON_START: 'purgeOnStart',
+        APIFY_MEMORY_MBYTES: 'memoryMbytes',
+        APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId',
+        APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
+        APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
+        APIFY_INPUT_KEY: 'inputKey',
+        APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
+        APIFY_HEADLESS: 'headless',
+        APIFY_XVFB: 'xvfb',
+        APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
+        APIFY_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
+        APIFY_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
+
+        // as well as apify specific ones
+        APIFY_TOKEN: 'token',
+        APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis',
+        APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis', // for BC, seems to be unused
+        APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl',
+        APIFY_ACTOR_ID: 'actorId',
+        APIFY_API_BASE_URL: 'apiBaseUrl',
+        APIFY_IS_AT_HOME: 'isAtHome',
+        APIFY_ACTOR_RUN_ID: 'actorRunId',
+        APIFY_ACTOR_TASK_ID: 'actorTaskId',
+        APIFY_CONTAINER_PORT: 'containerPort',
+        APIFY_CONTAINER_URL: 'containerUrl',
+        APIFY_USER_ID: 'userId',
+        APIFY_PROXY_HOSTNAME: 'proxyHostname',
+        APIFY_PROXY_PASSWORD: 'proxyPassword',
+        APIFY_PROXY_STATUS_URL: 'proxyStatusUrl',
+        APIFY_PROXY_PORT: 'proxyPort',
+        APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE: 'inputSecretsPrivateKeyFile',
+        APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE: 'inputSecretsPrivateKeyPassphrase',
+    };
+
+    protected static override INTEGER_VARS = [...super.INTEGER_VARS, 'proxyPort', 'containerPort', 'metamorphAfterSleepMillis'];
+
+    protected static override BOOLEAN_VARS = [...super.BOOLEAN_VARS, 'isAtHome'];
+
+    protected static override DEFAULTS = {
+        ...super.DEFAULTS,
+        defaultKeyValueStoreId: LOCAL_ENV_VARS[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
+        defaultDatasetId: LOCAL_ENV_VARS[ENV_VARS.DEFAULT_DATASET_ID],
+        defaultRequestQueueId: LOCAL_ENV_VARS[ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
+        inputKey: 'INPUT',
+        apiBaseUrl: 'https://api.apify.com',
+        proxyStatusUrl: 'http://proxy.apify.com',
+        proxyHostname: LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME],
+        proxyPort: +LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT],
+        containerPort: +LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT],
+        containerUrl: LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL],
+        metamorphAfterSleepMillis: 300e3,
+        persistStateIntervalMillis: 60e3, // This value is mentioned in jsdoc in `events.js`, if you update it here, update it there too.
+    };
+
+    /**
+     * @inheritDoc
+     */
+    override get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U {
+        return super.get(key as keyof CoreConfigurationOptions, defaultValue);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    override set(key: keyof ConfigurationOptions, value?: any) {
+        super.set(key as keyof CoreConfigurationOptions, value);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    static override getGlobalConfig(): Configuration {
+        if (Configuration.storage.getStore()) {
+            return Configuration.storage.getStore() as Configuration;
+        }
+
+        Configuration.globalConfig ??= new Configuration();
+        return Configuration.globalConfig as Configuration;
+    }
+
+    /**
+     * Resets global configuration instance. The default instance holds configuration based on env vars,
+     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
+     */
+    static override resetGlobalState(): void {
+        delete this.globalConfig;
+    }
+}
+
+// monkey patch the core class so it respects the new options too
+CoreConfiguration.getGlobalConfig = Configuration.getGlobalConfig;
+// @ts-expect-error protected property
+CoreConfiguration.ENV_MAP = Configuration.ENV_MAP;
+// @ts-expect-error protected property
+CoreConfiguration.INTEGER_VARS = Configuration.INTEGER_VARS;
+// @ts-expect-error protected property
+CoreConfiguration.BOOLEAN_VARS = Configuration.BOOLEAN_VARS;
+// @ts-expect-error protected property
+CoreConfiguration.DEFAULTS = Configuration.DEFAULTS;

package/src/index.ts

@@ -0,0 +1,11 @@
+export * from './actor';
+export * from './configuration';
+export * from './proxy_configuration';
+export * from './platform_event_manager';
+export * from './key_value_store';
+export {
+    Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent,
+    RequestQueue, QueueOperationInfo, RequestQueueOperationOptions, RequestQueueOptions, QueueOperationInfoOptions,
+    KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText,
+} from '@crawlee/core';
+export { ApifyClient, ApifyClientOptions } from 'apify-client';

package/src/key_value_store.ts

@@ -0,0 +1,25 @@
+import type { StorageManagerOptions } from '@crawlee/core';
+import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
+
+/**
+ * @inheritDoc
+ */
+export class KeyValueStore extends CoreKeyValueStore {
+    /**
+     * Returns a URL for the given key that may be used to publicly
+     * access the value in the remote key-value store.
+     */
+    getPublicUrl(key: string): string {
+        return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`;
+    }
+
+    /**
+     * @inheritDoc
+     */
+    static override async open(storeIdOrName?: string | null, options: StorageManagerOptions = {}): Promise<KeyValueStore> {
+        return super.open(storeIdOrName, options) as unknown as KeyValueStore;
+    }
+}
+
+// @ts-expect-error extension of the core class to make this only a type-issue
+CoreKeyValueStore.prototype.getPublicUrl = KeyValueStore.prototype.getPublicUrl;

package/src/platform_event_manager.ts

@@ -0,0 +1,118 @@
+import { ACTOR_EVENT_NAMES, ENV_VARS } from '@apify/consts';
+import WebSocket from 'ws';
+import { EventType, EventManager } from '@crawlee/core';
+import { betterClearInterval } from '@apify/utilities';
+import { Configuration } from './configuration';
+
+/**
+ * Gets an instance of a Node.js'
+ * [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)
+ * class that emits various events from the SDK or the Apify platform.
+ * The event emitter is initialized by calling the {@apilink Actor.main} function.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * Actor.on('cpuInfo', (data) => {
+ *   if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');
+ * });
+ * ```
+ *
+ * The following events are emitted:
+ *
+ * - `cpuInfo`: `{ "isCpuOverloaded": Boolean }`
+ *   The event is emitted approximately every second
+ *   and it indicates whether the actor is using the maximum of available CPU resources.
+ *   If that's the case, the actor should not add more workload.
+ *   For example, this event is used by the {@apilink AutoscaledPool} class.
+ * - `migrating`: `void`
+ *   Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.
+ *   You can use it to persist the state of the actor and abort the run, to speed up migration.
+ *   For example, this is used by the {@apilink RequestList} class.
+ * - `aborting`: `void`
+ *   When a user aborts an actor run on the Apify platform, they can choose to abort gracefully to allow
+ *   the actor some time before getting killed. This graceful abort emits the `aborting` event which the SDK
+ *   uses to gracefully stop running crawls and you can use it to do your own cleanup as well.
+ * - `persistState`: `{ "isMigrating": Boolean }`
+ *   Emitted in regular intervals (by default 60 seconds) to notify all components of Apify SDK that it is time to persist
+ *   their state, in order to avoid repeating all work when the actor restarts.
+ *   This event is automatically emitted together with the `migrating` event,
+ *   in which case the `isMigrating` flag is set to `true`. Otherwise the flag is `false`.
+ *   Note that the `persistState` event is provided merely for user convenience,
+ *   you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
+ */
+export class PlatformEventManager extends EventManager {
+    /** Websocket connection to actor events. */
+    private eventsWs?: WebSocket;
+
+    constructor(override readonly config = Configuration.getGlobalConfig()) {
+        super();
+    }
+
+    /**
+     * Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
+     * This is an internal function that is automatically called by `Actor.main()`.
+     */
+    override async init() {
+        if (this.initialized) {
+            return;
+        }
+
+        await super.init();
+        const eventsWsUrl = this.config.get('actorEventsWsUrl');
+
+        // Locally there is no web socket to connect, so just print a log message.
+        if (!eventsWsUrl) {
+            this.log.debug(`Environment variable ${ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, no events from Apify platform will be emitted.`);
+            return;
+        }
+
+        this.createWebSocketConnection(eventsWsUrl);
+    }
+
+    private createWebSocketConnection(eventsWsUrl: string) {
+        this.eventsWs = new WebSocket(eventsWsUrl);
+        this.eventsWs.on('message', (message) => {
+            if (!message) return;
+
+            try {
+                const {
+                    name,
+                    data,
+                } = JSON.parse(String(message));
+                this.events.emit(name, data);
+
+                if (name === ACTOR_EVENT_NAMES.MIGRATING) {
+                    betterClearInterval(this.intervals.persistState!); // Don't send any other persist state event.
+                    this.events.emit(EventType.PERSIST_STATE, { isMigrating: true });
+                }
+            } catch (err) {
+                this.log.exception(err as Error, 'Cannot parse actor event');
+            }
+        });
+        this.eventsWs.on('error', (err) => {
+            // Don't print this error as this happens in the case of very short Actor.main().
+            if (err.message === 'WebSocket was closed before the connection was established') return;
+
+            this.log.exception(err, 'web socket connection failed');
+        });
+        this.eventsWs.on('close', () => {
+            this.log.debug('web socket has been closed');
+            this.eventsWs = undefined;
+        });
+    }
+
+    /**
+     * Closes websocket providing events from Actor infrastructure and also stops sending internal events
+     * of Apify package such as `persistState`.
+     * This is automatically called at the end of `Actor.main()`.
+     */
+    override async close() {
+        if (!this.initialized) {
+            return;
+        }
+
+        await super.close();
+        this.eventsWs?.close();
+    }
+}