apify 3.7.3-beta.9 → 4.0.0-beta.13

package/dist/input-schemas.js

@@ -1,11 +1,7 @@
-"use strict";
 // TODO: https://github.com/apify/apify-shared-js/issues/547
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDefaultsFromInputSchema = exports.readInputSchema = exports.noActorInputSchemaDefinedMarker = void 0;
-const tslib_1 = require("tslib");
-const node_fs_1 = require("node:fs");
-const node_path_1 = require("node:path");
-const node_process_1 = tslib_1.__importDefault(require("node:process"));
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import process from 'node:process';
 // These paths are used *if* there is no `input` field in the actor.json configuration file!
 const DEFAULT_INPUT_SCHEMA_PATHS = [
     ['.actor', 'INPUT_SCHEMA.json'],
@@ -16,8 +12,8 @@ const DEFAULT_INPUT_SCHEMA_PATHS = [
 const ACTOR_SPECIFICATION_FOLDER = '.actor';
 const LOCAL_CONFIG_NAME = 'actor.json';
 const readJSONIfExists = (path) => {
-    if ((0, node_fs_1.existsSync)(path)) {
-        const content = (0, node_fs_1.readFileSync)(path, 'utf8');
+    if (existsSync(path)) {
+        const content = readFileSync(path, 'utf8');
         return JSON.parse(content);
     }
     return null;
@@ -25,21 +21,21 @@ const readJSONIfExists = (path) => {
 /**
  * @ignore
  */
-exports.noActorInputSchemaDefinedMarker = Symbol.for('apify.noActorInputSchemaDefined');
-const readInputSchema = () => {
-    const localConfig = readJSONIfExists((0, node_path_1.join)(node_process_1.default.cwd(), ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_NAME));
+export const noActorInputSchemaDefinedMarker = Symbol.for('apify.noActorInputSchemaDefined');
+export const readInputSchema = () => {
+    const localConfig = readJSONIfExists(join(process.cwd(), ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_NAME));
     // Input schema nested in the actor config
     if (typeof localConfig?.input === 'object') {
         return localConfig.input;
     }
     // Input schema path from the actor config
     if (typeof localConfig?.input === 'string') {
-        const fullPath = (0, node_path_1.join)(node_process_1.default.cwd(), ACTOR_SPECIFICATION_FOLDER, localConfig.input);
+        const fullPath = join(process.cwd(), ACTOR_SPECIFICATION_FOLDER, localConfig.input);
         return readJSONIfExists(fullPath);
     }
     // Try to find it from possible default paths
     for (const path of DEFAULT_INPUT_SCHEMA_PATHS) {
-        const fullPath = (0, node_path_1.join)(node_process_1.default.cwd(), ...path);
+        const fullPath = join(process.cwd(), ...path);
         const result = readJSONIfExists(fullPath);
         if (result) {
             return result;
@@ -47,12 +43,11 @@ const readInputSchema = () => {
     }
     // If we are in an Actor context, BUT we do not have an input schema defined, we want to skip the warning
     if (!localConfig?.input) {
-        return exports.noActorInputSchemaDefinedMarker;
+        return noActorInputSchemaDefinedMarker;
     }
     return null;
 };
-exports.readInputSchema = readInputSchema;
-const getDefaultsFromInputSchema = (inputSchema) => {
+export const getDefaultsFromInputSchema = (inputSchema) => {
     const defaults = {};
     for (const [key, fieldSchema] of Object.entries(inputSchema.properties)) {
         if (fieldSchema.default !== undefined) {
@@ -61,4 +56,3 @@ const getDefaultsFromInputSchema = (inputSchema) => {
     }
     return defaults;
 };
-exports.getDefaultsFromInputSchema = getDefaultsFromInputSchema;

package/dist/key_value_store.d.ts

@@ -1,4 +1,4 @@
-import type { StorageManagerOptions } from '@crawlee/core';
+import type { StorageOpenOptions } from '@crawlee/core';
 import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
 /**
  * @inheritDoc
@@ -7,10 +7,15 @@ export declare class KeyValueStore extends CoreKeyValueStore {
     /**
      * Returns a URL for the given key that may be used to publicly
      * access the value in the remote key-value store.
+     *
+     * On the Apify platform the URL is signed with the store's
+     * `urlSigningSecretKey` so that anyone with the URL can read the record
+     * without authentication. Locally we delegate to crawlee's default
+     * implementation (which produces a `file://` URL or returns `undefined`).
      */
-    getPublicUrl(key: string): string;
+    getPublicUrl(key: string): Promise<string | undefined>;
     /**
      * @inheritDoc
      */
-    static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
+    static open(storeIdOrName?: string | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
 }

package/dist/key_value_store.js

@@ -1,30 +1,34 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.KeyValueStore = void 0;
-const core_1 = require("@crawlee/core");
-const apify_client_1 = require("apify-client");
-const utilities_1 = require("@apify/utilities");
-// @ts-ignore newer crawlee versions already declare this method in core
-const { getPublicUrl } = core_1.KeyValueStore.prototype;
+import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
+import { KeyValueStoreClient as RemoteKeyValueStoreClient } from 'apify-client';
+import { createHmacSignature } from '@apify/utilities';
 /**
  * @inheritDoc
  */
-class KeyValueStore extends core_1.KeyValueStore {
+export class KeyValueStore extends CoreKeyValueStore {
     /**
      * Returns a URL for the given key that may be used to publicly
      * access the value in the remote key-value store.
+     *
+     * On the Apify platform the URL is signed with the store's
+     * `urlSigningSecretKey` so that anyone with the URL can read the record
+     * without authentication. Locally we delegate to crawlee's default
+     * implementation (which produces a `file://` URL or returns `undefined`).
      */
-    getPublicUrl(key) {
+    async getPublicUrl(key) {
         const config = this.config;
-        const isLocalStore = !(
-        // eslint-disable-next-line dot-notation
-        (this['client'] instanceof apify_client_1.KeyValueStoreClient));
-        if (isLocalStore && getPublicUrl) {
-            return getPublicUrl.call(this, key);
+        // Detect a remote (Apify) store by its client type rather than by
+        // `isAtHome`, so that a `forceCloud` store opened locally still gets a
+        // signed Apify URL (matching the platform behaviour). `client` is
+        // `private` on `CoreKeyValueStore`, so bypass the visibility check.
+        const { client } = this;
+        const isLocalStore = !(client instanceof RemoteKeyValueStoreClient);
+        if (isLocalStore) {
+            return super.getPublicUrl(key);
         }
-        const publicUrl = new URL(`${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`);
-        if (this.storageObject?.urlSigningSecretKey) {
-            publicUrl.searchParams.append('signature', (0, utilities_1.createHmacSignature)(this.storageObject.urlSigningSecretKey, key));
+        const publicUrl = new URL(`${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`);
+        const metadata = (await client.getMetadata());
+        if (metadata?.urlSigningSecretKey) {
+            publicUrl.searchParams.append('signature', createHmacSignature(metadata.urlSigningSecretKey, key));
         }
         return publicUrl.toString();
     }
@@ -35,6 +39,3 @@ class KeyValueStore extends core_1.KeyValueStore {
         return super.open(storeIdOrName, options);
     }
 }
-exports.KeyValueStore = KeyValueStore;
-// @ts-ignore newer crawlee versions already declare this method in core
-core_1.KeyValueStore.prototype.getPublicUrl = KeyValueStore.prototype.getPublicUrl;

package/dist/platform_event_manager.d.ts

@@ -26,15 +26,10 @@ import { Configuration } from './configuration.js';
  *   You can use it to persist the state of the Actor and gracefully stop your in-progress tasks,
  *   so that they are not interrupted by the migration.
  *   For example, this is used by the {@link RequestList} class.
- *   If you pass `gracefulShutdown: true` to {@link Actor.init}, the SDK will automatically call
- *   {@link Actor.reboot} when this event is received, which speeds up the migration and lets the
- *   run continue on a new worker.
  * - `aborting`: `void`
  *   When a user aborts an Actor run on the Apify platform, they can choose to abort gracefully to allow
  *   the Actor some time before getting killed. This graceful abort emits the `aborting` event which the SDK
  *   uses to gracefully stop running crawls and you can use it to do your own cleanup as well.
- *   If you pass `gracefulShutdown: true` to {@link Actor.init}, the SDK will automatically call
- *   {@link Actor.exit} when this event is received.
  * - `persistState`: `{ "isMigrating": Boolean }`
  *   Emitted in regular intervals (by default 60 seconds) to notify all components of Apify SDK that it is time to persist
  *   their state, in order to avoid repeating all work when the Actor restarts.

package/dist/platform_event_manager.js

@@ -1,11 +1,8 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PlatformEventManager = void 0;
-const core_1 = require("@crawlee/core");
-const ws_1 = require("ws");
-const consts_1 = require("@apify/consts");
-const utilities_1 = require("@apify/utilities");
-const configuration_js_1 = require("./configuration.js");
+import { EventManager } from '@crawlee/core';
+import { WebSocket } from 'ws';
+import { ACTOR_ENV_VARS, ACTOR_EVENT_NAMES } from '@apify/consts';
+import { betterClearInterval } from '@apify/utilities';
+import { Configuration } from './configuration.js';
 /**
  * Gets an instance of a Node.js'
  * [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)
@@ -32,15 +29,10 @@ const configuration_js_1 = require("./configuration.js");
  *   You can use it to persist the state of the Actor and gracefully stop your in-progress tasks,
  *   so that they are not interrupted by the migration.
  *   For example, this is used by the {@link RequestList} class.
- *   If you pass `gracefulShutdown: true` to {@link Actor.init}, the SDK will automatically call
- *   {@link Actor.reboot} when this event is received, which speeds up the migration and lets the
- *   run continue on a new worker.
  * - `aborting`: `void`
  *   When a user aborts an Actor run on the Apify platform, they can choose to abort gracefully to allow
  *   the Actor some time before getting killed. This graceful abort emits the `aborting` event which the SDK
  *   uses to gracefully stop running crawls and you can use it to do your own cleanup as well.
- *   If you pass `gracefulShutdown: true` to {@link Actor.init}, the SDK will automatically call
- *   {@link Actor.exit} when this event is received.
  * - `persistState`: `{ "isMigrating": Boolean }`
  *   Emitted in regular intervals (by default 60 seconds) to notify all components of Apify SDK that it is time to persist
  *   their state, in order to avoid repeating all work when the Actor restarts.
@@ -49,22 +41,15 @@ const configuration_js_1 = require("./configuration.js");
  *   Note that the `persistState` event is provided merely for user convenience,
  *   you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
  */
-class PlatformEventManager extends core_1.EventManager {
-    constructor(config = configuration_js_1.Configuration.getGlobalConfig()) {
-        super();
-        Object.defineProperty(this, "config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: config
-        });
-        /** Websocket connection to Actor events. */
-        Object.defineProperty(this, "eventsWs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
+export class PlatformEventManager extends EventManager {
+    config;
+    /** Websocket connection to Actor events. */
+    eventsWs;
+    constructor(config = Configuration.getGlobalConfig()) {
+        super({
+            persistStateIntervalMillis: config.persistStateIntervalMillis,
         });
+        this.config = config;
     }
     /**
      * Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
@@ -75,24 +60,24 @@ class PlatformEventManager extends core_1.EventManager {
             return;
         }
         await super.init();
-        const eventsWsUrl = this.config.get('actorEventsWsUrl');
+        const eventsWsUrl = this.config.actorEventsWsUrl;
         // Locally there is no web socket to connect, so just print a log message.
         if (!eventsWsUrl) {
-            this.log.debug(`Environment variable ${consts_1.ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL} is not set, no events from Apify platform will be emitted.`);
+            this.log.debug(`Environment variable ${ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL} is not set, no events from Apify platform will be emitted.`);
             return;
         }
         this.createWebSocketConnection(eventsWsUrl);
     }
     createWebSocketConnection(eventsWsUrl) {
-        this.eventsWs = new ws_1.WebSocket(eventsWsUrl);
+        this.eventsWs = new WebSocket(eventsWsUrl);
         this.eventsWs.on('message', (message) => {
             if (!message)
                 return;
             try {
                 const { name, data } = JSON.parse(String(message));
                 this.events.emit(name, data);
-                if (name === consts_1.ACTOR_EVENT_NAMES.MIGRATING) {
-                    (0, utilities_1.betterClearInterval)(this.intervals.persistState); // Don't send any other persist state event.
+                if (name === ACTOR_EVENT_NAMES.MIGRATING) {
+                    betterClearInterval(this.intervals.persistState); // Don't send any other persist state event.
                     this.events.emit("persistState" /* EventType.PERSIST_STATE */, {
                         isMigrating: true,
                     });
@@ -126,4 +111,3 @@ class PlatformEventManager extends core_1.EventManager {
         this.eventsWs?.close();
     }
 }
-exports.PlatformEventManager = PlatformEventManager;

package/dist/proxy_configuration.d.ts

@@ -1,6 +1,8 @@
-import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions, ProxyInfo as CoreProxyInfo } from '@crawlee/core';
+import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions } from '@crawlee/core';
 import { ProxyConfiguration as CoreProxyConfiguration } from '@crawlee/core';
+import type { ProxyInfo as CoreProxyInfo } from '@crawlee/types';
 import { Configuration } from './configuration.js';
+type NewUrlOptions = Parameters<CoreProxyConfiguration['newProxyInfo']>[0];
 export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions {
     /**
      * User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
@@ -24,32 +26,27 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
      * on the Apify cloud, or when using the [Apify CLI](https://github.com/apify/apify-cli).
      */
     countryCode?: string;
+    /**
+     * If set, all proxied requests will use IP addresses geolocated to the specified subdivision (e.g. US state).
+     * Requires `countryCode` to be set. The value must follow the ISO 3166-2 subdivision code format,
+     * e.g. `'CA'` for California when `countryCode` is `'US'`.
+     */
+    subdivisionCode?: string;
     /**
      * Same option as `groups` which can be used to
-     * configure the proxy by UI input schema. You should use the `groups` option in your crawler code.
+     * configurate the proxy by UI input schema. You should use the `groups` option in your crawler code.
      */
     apifyProxyGroups?: string[];
     /**
      * Same option as `countryCode` which can be used to
-     * configure the proxy by UI input schema. You should use the `countryCode` option in your crawler code.
+     * configurate the proxy by UI input schema. You should use the `countryCode` option in your crawler code.
      */
     apifyProxyCountry?: string;
-    /**
-     * If set, all proxied requests will use IP addresses geolocated to the specified subdivision (e.g. US state).
-     * Requires `countryCode` to be set. The value must follow the ISO 3166-2 subdivision code format,
-     * e.g. `'CA'` for California when `countryCode` is `'US'`.
-     */
-    subdivisionCode?: string;
     /**
      * Same option as `subdivisionCode` which can be used to
-     * configure the proxy by UI input schema. You should use the `subdivisionCode` option in your crawler code.
+     * configurate the proxy by UI input schema. You should use the `subdivisionCode` option in your crawler code.
      */
     apifyProxySubdivision?: string;
-    /**
-     * Multiple different ProxyConfigurationOptions stratified into tiers. Crawlee crawlers will switch between those tiers
-     * based on the blocked request statistics.
-     */
-    tieredProxyConfig?: Omit<ProxyConfigurationOptions, keyof CoreProxyConfigurationOptions | 'tieredProxyConfig'>[];
     /**
      * As part of the init process, we verify the configuration by checking the proxy status endpoint.
      * This can make the init slower, to opt-out of this, use `checkAccess: false` (defaults to `true`).
@@ -80,9 +77,6 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
  *   requestHandler({ proxyInfo }) {
  *       // Getting used proxy URL
  *       const proxyUrl = proxyInfo.url;
- *
- *       // Getting ID of used Session
- *       const sessionIdentifier = proxyInfo.sessionId;
  *   }
  * })
  *
@@ -93,7 +87,7 @@ export interface ProxyInfo extends CoreProxyInfo {
      * An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).
      * If not provided, the proxy will select the groups automatically.
      */
-    groups: string[];
+    groups?: string[];
     /**
      * If set and relevant proxies are available in your Apify account, all proxied requests will
      * use IP addresses that are geolocated to the specified country. For example `GB` for IPs
@@ -170,51 +164,27 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
      *
      * You should use the {@link createProxyConfiguration} function to create a pre-initialized
      * `ProxyConfiguration` instance instead of calling this manually.
-     *
-     * As part of the init process, we verify the configuration by checking the proxy status endpoint.
-     * This can make the init slower, to opt-out of this, use `checkAccess: false`.
      */
     initialize(options?: {
         checkAccess?: boolean;
     }): Promise<boolean>;
     /**
-     * This function creates a new {@link ProxyInfo} info object.
-     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
-     * the currently used proxy via the requestHandler parameter `proxyInfo`.
-     * Use it if you want to work with a rich representation of a proxy URL.
-     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
-     *  {@link ProxyInfo} is always returned as a type string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
-     * @return Represents information about used proxy and its configuration.
-     */
-    newProxyInfo(sessionId?: string | number, options?: Parameters<CoreProxyConfiguration['newProxyInfo']>[1]): Promise<ProxyInfo | undefined>;
-    /**
-     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
-     * @return A string with a proxy URL, including authentication credentials and port number.
-     *  For example, `http://bob:password123@proxy.example.com:8000`
+     * Returns a new {@link ProxyInfo} object with a fresh proxy URL. Each call mints an
+     * independent URL; for Apify Proxy a random session id is embedded so consecutive
+     * calls resolve to different IPs.
+     */
+    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
+    /**
+     * Returns a new proxy URL. For Apify Proxy, each call generates a URL with a fresh
+     * random session id, so consecutive calls return independent URLs. For custom
+     * `proxyUrls`, the URLs are rotated round-robin.
      */
-    newUrl(sessionId?: string | number, options?: Parameters<CoreProxyConfiguration['newUrl']>[1]): Promise<string | undefined>;
-    protected _generateTieredProxyUrls(tieredProxyConfig: NonNullable<ProxyConfigurationOptions['tieredProxyConfig']>, globalOptions: ProxyConfigurationOptions): string[][];
+    newUrl(options?: NewUrlOptions): Promise<string | undefined>;
     /**
      * Returns proxy username.
      */
-    protected _getUsername(sessionId?: string): string;
-    protected composeDefaultUrl(sessionId?: string): string;
+    protected _getUsername(sessionId: string): string;
+    protected composeDefaultUrl(sessionId: string): string;
     /**
      * Fetch & set the proxy password from Apify API if an Apify token is provided.
      */
@@ -239,3 +209,4 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
      */
     protected _throwCannotCombineCustomWithApify(): void;
 }
+export {};