apify 3.7.3-beta.9 → 4.0.0-beta.13

package/dist/proxy_configuration.js

@@ -1,21 +1,19 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ProxyConfiguration = void 0;
-const tslib_1 = require("tslib");
-const core_1 = require("@crawlee/core");
-const utils_1 = require("@crawlee/utils");
-const ow_1 = tslib_1.__importDefault(require("ow"));
-const consts_1 = require("@apify/consts");
-const utilities_1 = require("@apify/utilities");
-const actor_js_1 = require("./actor.js");
-const configuration_js_1 = require("./configuration.js");
-// https://docs.apify.com/proxy/datacenter-proxy#username-parameters
-const MAX_SESSION_ID_LENGTH = 50;
-const CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS = 4000;
+import { ProxyConfiguration as CoreProxyConfiguration } from '@crawlee/core';
+import { gotScraping } from 'got-scraping';
+import ow from 'ow';
+import { APIFY_ENV_VARS, APIFY_PROXY_VALUE_REGEX } from '@apify/consts';
+import { cryptoRandomObjectId } from '@apify/utilities';
+import { Actor } from './actor.js';
+import { Configuration } from './configuration.js';
+const CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS = 4_000;
 const CHECK_ACCESS_MAX_ATTEMPTS = 2;
 const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;
 // ISO 3166-2 subdivision codes are 1–3 uppercase alphanumeric characters, e.g. 'CA' (California), 'NSW' (New South Wales), '9' (Wien, AT-9)
 const SUBDIVISION_CODE_REGEX = /^[A-Z0-9]{1,3}$/;
+// Apify Proxy session identifier embedded in the proxy username — opaque to
+// users; a fresh one is minted for every URL the SDK hands out so that the
+// returned proxy URLs are independent.
+const SESSION_ID_LENGTH = 12;
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
  * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -49,86 +47,43 @@ const SUBDIVISION_CODE_REGEX = /^[A-Z0-9]{1,3}$/;
  * ```
  * @category Scaling
  */
-class ProxyConfiguration extends core_1.ProxyConfiguration {
+export class ProxyConfiguration extends CoreProxyConfiguration {
+    config;
+    groups;
+    countryCode;
+    subdivisionCode;
+    password;
+    hostname;
+    port;
+    usesApifyProxy;
     /**
      * @internal
      */
-    constructor(options = {}, config = configuration_js_1.Configuration.getGlobalConfig()) {
+    constructor(options = {}, config = Configuration.getGlobalConfig()) {
         const { proxyUrls, newUrlFunction, ...rest } = options;
         super({
             proxyUrls,
             newUrlFunction,
             ['validateRequired']: false,
         });
-        Object.defineProperty(this, "config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: config
-        });
-        Object.defineProperty(this, "groups", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "countryCode", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "subdivisionCode", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "password", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "hostname", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "port", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "usesApifyProxy", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        (0, ow_1.default)(rest, ow_1.default.object.partialShape({
-            groups: ow_1.default.optional.array.ofType(ow_1.default.string.matches(consts_1.APIFY_PROXY_VALUE_REGEX)),
-            apifyProxyGroups: ow_1.default.optional.array.ofType(ow_1.default.string.matches(consts_1.APIFY_PROXY_VALUE_REGEX)),
-            countryCode: ow_1.default.optional.string.matches(COUNTRY_CODE_REGEX),
-            apifyProxyCountry: ow_1.default.optional.string.matches(COUNTRY_CODE_REGEX),
-            subdivisionCode: ow_1.default.optional.string.matches(SUBDIVISION_CODE_REGEX),
-            apifyProxySubdivision: ow_1.default.optional.string.matches(SUBDIVISION_CODE_REGEX),
-            password: ow_1.default.optional.string,
-            tieredProxyUrls: ow_1.default.optional.array.ofType(ow_1.default.array.ofType(ow_1.default.string)),
-            tieredProxyConfig: ow_1.default.optional.array.ofType(ow_1.default.object),
+        this.config = config;
+        ow(rest, ow.object.exactShape({
+            groups: ow.optional.array.ofType(ow.string.matches(APIFY_PROXY_VALUE_REGEX)),
+            apifyProxyGroups: ow.optional.array.ofType(ow.string.matches(APIFY_PROXY_VALUE_REGEX)),
+            countryCode: ow.optional.string.matches(COUNTRY_CODE_REGEX),
+            apifyProxyCountry: ow.optional.string.matches(COUNTRY_CODE_REGEX),
+            subdivisionCode: ow.optional.string.matches(SUBDIVISION_CODE_REGEX),
+            apifyProxySubdivision: ow.optional.string.matches(SUBDIVISION_CODE_REGEX),
+            password: ow.optional.string,
         }));
-        const { groups = [], apifyProxyGroups = [], countryCode, apifyProxyCountry, subdivisionCode, apifyProxySubdivision, password = config.get('proxyPassword'), tieredProxyConfig, tieredProxyUrls, } = options;
-        this.tieredProxyUrls ?? (this.tieredProxyUrls = tieredProxyUrls);
-        if (tieredProxyConfig) {
-            this.tieredProxyUrls = this._generateTieredProxyUrls(tieredProxyConfig, options);
-        }
+        const { groups = [], apifyProxyGroups = [], countryCode, apifyProxyCountry, subdivisionCode, apifyProxySubdivision, password = config.proxyPassword, } = options;
         const groupsToUse = groups.length ? groups : apifyProxyGroups;
         const countryCodeToUse = countryCode || apifyProxyCountry;
         const subdivisionCodeToUse = subdivisionCode || apifyProxySubdivision;
-        const hostname = config.get('proxyHostname');
-        const port = config.get('proxyPort');
+        const hostname = config.proxyHostname;
+        const port = config.proxyPort;
+        // The Apify Proxy subdivision is expressed as part of the country
+        // username parameter (`country-US_CA`), so a country is required.
         if (subdivisionCodeToUse && !countryCodeToUse) {
             throw new Error('ProxyConfiguration: "subdivisionCode" requires "countryCode" to be set.');
         }
@@ -157,9 +112,6 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
      *
      * You should use the {@link createProxyConfiguration} function to create a pre-initialized
      * `ProxyConfiguration` instance instead of calling this manually.
-     *
-     * As part of the init process, we verify the configuration by checking the proxy status endpoint.
-     * This can make the init slower, to opt-out of this, use `checkAccess: false`.
      */
     async initialize(options) {
         if (this.usesApifyProxy) {
@@ -167,16 +119,16 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
                 await this._setPasswordIfToken();
             }
             if (!this.password) {
-                if (actor_js_1.Actor.isAtHome()) {
-                    throw new Error(`Apify Proxy password must be provided using options.password or the "${consts_1.APIFY_ENV_VARS.PROXY_PASSWORD}" environment variable. ` +
-                        `You can also provide your Apify token via the "${consts_1.APIFY_ENV_VARS.TOKEN}" environment variable, ` +
-                        `so that the SDK can fetch the proxy password from Apify API, when ${consts_1.APIFY_ENV_VARS.PROXY_PASSWORD} is not defined`);
+                if (Actor.isAtHome()) {
+                    throw new Error(`Apify Proxy password must be provided using options.password or the "${APIFY_ENV_VARS.PROXY_PASSWORD}" environment variable. ` +
+                        `You can also provide your Apify token via the "${APIFY_ENV_VARS.TOKEN}" environment variable, ` +
+                        `so that the SDK can fetch the proxy password from Apify API, when ${APIFY_ENV_VARS.PROXY_PASSWORD} is not defined`);
                 }
                 else {
                     this.log.warning(`No proxy password or token detected, running without proxy. To use Apify Proxy locally, ` +
-                        `provide options.password or "${consts_1.APIFY_ENV_VARS.PROXY_PASSWORD}" environment variable. ` +
-                        `You can also provide your Apify token via the "${consts_1.APIFY_ENV_VARS.TOKEN}" environment variable, ` +
-                        `so that the SDK can fetch the proxy password from Apify API, when ${consts_1.APIFY_ENV_VARS.PROXY_PASSWORD} is not defined`);
+                        `provide options.password or "${APIFY_ENV_VARS.PROXY_PASSWORD}" environment variable. ` +
+                        `You can also provide your Apify token via the "${APIFY_ENV_VARS.TOKEN}" environment variable, ` +
+                        `so that the SDK can fetch the proxy password from Apify API, when ${APIFY_ENV_VARS.PROXY_PASSWORD} is not defined`);
                 }
             }
             if (options?.checkAccess !== false) {
@@ -186,104 +138,59 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
         return true;
     }
     /**
-     * This function creates a new {@link ProxyInfo} info object.
-     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
-     * the currently used proxy via the requestHandler parameter `proxyInfo`.
-     * Use it if you want to work with a rich representation of a proxy URL.
-     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
-     *  {@link ProxyInfo} is always returned as a type string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
-     * @return Represents information about used proxy and its configuration.
+     * Returns a new {@link ProxyInfo} object with a fresh proxy URL. Each call mints an
+     * independent URL; for Apify Proxy a random session id is embedded so consecutive
+     * calls resolve to different IPs.
      */
-    async newProxyInfo(sessionId, options) {
-        if (typeof sessionId === 'number')
-            sessionId = `${sessionId}`;
-        (0, ow_1.default)(sessionId, ow_1.default.optional.string.maxLength(MAX_SESSION_ID_LENGTH).matches(consts_1.APIFY_PROXY_VALUE_REGEX));
-        const proxyInfo = await super.newProxyInfo(sessionId, options);
-        if (!proxyInfo)
-            return proxyInfo;
-        const { groups, countryCode, subdivisionCode, password, port, hostname } = (this.usesApifyProxy ? this : new URL(proxyInfo.url));
-        return {
-            ...proxyInfo,
-            sessionId,
-            groups,
-            countryCode,
-            subdivisionCode,
-            // this.password is not encoded, but the password from the URL will be, we need to normalize
-            password: this.usesApifyProxy ? (password ?? '') : decodeURIComponent(password),
-            hostname,
-            port: port,
+    async newProxyInfo(options) {
+        const url = await this.newUrl(options);
+        if (!url)
+            return undefined;
+        const parsed = new URL(url);
+        const result = {
+            url,
+            username: decodeURIComponent(parsed.username),
+            password: decodeURIComponent(parsed.password),
+            hostname: parsed.hostname,
+            port: parsed.port,
         };
+        if (this.usesApifyProxy) {
+            result.groups = this.groups;
+            if (this.countryCode !== undefined)
+                result.countryCode = this.countryCode;
+            if (this.subdivisionCode !== undefined)
+                result.subdivisionCode = this.subdivisionCode;
+        }
+        return result;
     }
     /**
-     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
-     * @return A string with a proxy URL, including authentication credentials and port number.
-     *  For example, `http://bob:password123@proxy.example.com:8000`
+     * Returns a new proxy URL. For Apify Proxy, each call generates a URL with a fresh
+     * random session id, so consecutive calls return independent URLs. For custom
+     * `proxyUrls`, the URLs are rotated round-robin.
      */
-    async newUrl(sessionId, options) {
-        if (typeof sessionId === 'number')
-            sessionId = `${sessionId}`;
-        (0, ow_1.default)(sessionId, ow_1.default.optional.string.maxLength(MAX_SESSION_ID_LENGTH).matches(consts_1.APIFY_PROXY_VALUE_REGEX));
-        if (this.newUrlFunction) {
-            return ((await this._callNewUrlFunction(sessionId, {
-                request: options?.request,
-            })) ?? undefined);
+    async newUrl(options) {
+        if (this.newUrlFunction || this.proxyUrls) {
+            return super.newUrl(options);
         }
-        if (this.proxyUrls) {
-            return this._handleCustomUrl(sessionId) ?? undefined;
-        }
-        if (this.tieredProxyUrls) {
-            return this._handleTieredUrl(sessionId ?? (0, utilities_1.cryptoRandomObjectId)(6), options).proxyUrl ?? undefined;
-        }
-        return this.composeDefaultUrl(sessionId);
-    }
-    _generateTieredProxyUrls(tieredProxyConfig, globalOptions) {
-        return tieredProxyConfig.map((config) => [
-            new ProxyConfiguration({
-                ...globalOptions,
-                ...config,
-                tieredProxyConfig: undefined,
-            }).composeDefaultUrl(),
-        ]);
+        return this.composeDefaultUrl(cryptoRandomObjectId(SESSION_ID_LENGTH));
     }
     /**
      * Returns proxy username.
      */
     _getUsername(sessionId) {
-        let username;
         const { groups, countryCode, subdivisionCode } = this;
         const parts = [];
         if (groups && groups.length) {
             parts.push(`groups-${groups.join('+')}`);
         }
-        if (sessionId) {
-            parts.push(`session-${sessionId}`);
-        }
+        parts.push(`session-${sessionId}`);
         if (subdivisionCode) {
             parts.push(`country-${countryCode}_${subdivisionCode}`);
         }
         else if (countryCode) {
             parts.push(`country-${countryCode}`);
         }
-        username = parts.join(',');
-        if (parts.length === 0)
-            username = 'auto';
-        return username;
+        return parts.join(',');
     }
     composeDefaultUrl(sessionId) {
         const username = this._getUsername(sessionId);
@@ -298,15 +205,15 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
      */
     // TODO: Make this private
     async _setPasswordIfToken() {
-        const token = this.config.get('token');
+        const { token } = this.config;
         if (!token)
             return;
         try {
-            const user = await actor_js_1.Actor.apifyClient.user().get();
+            const user = await Actor.apifyClient.user().get();
             this.password = user.proxy?.password;
         }
         catch (error) {
-            if (actor_js_1.Actor.isAtHome()) {
+            if (Actor.isAtHome()) {
                 throw error;
             }
             else {
@@ -336,7 +243,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
         // Throw only on the platform, locally we just print a warning and run requests without the proxy.
         // This is because the user might not have set up things correctly yet.
         // It still fails on the platform, where we don't want to allow this behavior.
-        if (actor_js_1.Actor.isAtHome()) {
+        if (Actor.isAtHome()) {
             throw new Error(connectionError);
         }
         this.log.warning(connectionError);
@@ -346,7 +253,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
      * Apify Proxy can be down for a second or a minute, but this should not crash processes.
      */
     async _fetchStatus() {
-        const proxyStatusUrl = this.config.get('proxyStatusUrl', 'http://proxy.apify.com');
+        const { proxyStatusUrl } = this.config;
         const requestOpts = {
             url: `${proxyStatusUrl}/?format=json`,
             proxyUrl: await this.newUrl(),
@@ -355,7 +262,7 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
         };
         for (let attempt = 1; attempt <= CHECK_ACCESS_MAX_ATTEMPTS; attempt++) {
             try {
-                const response = await (0, utils_1.gotScraping)(requestOpts);
+                const response = await gotScraping(requestOpts);
                 return response.body;
             }
             catch {
@@ -375,4 +282,3 @@ class ProxyConfiguration extends core_1.ProxyConfiguration {
             '"options.subdivisionCode" or "options.apifyProxySubdivision".');
     }
 }
-exports.ProxyConfiguration = ProxyConfiguration;

package/dist/storage.d.ts

@@ -1,5 +1,5 @@
-import type { IStorage } from '@crawlee/core';
-import type { Constructor, StorageClient } from '@crawlee/types';
+import type { Constructor, IStorage, StorageOpenOptions } from '@crawlee/core';
+import type { StorageClient } from '@crawlee/types';
 import type { Configuration } from './configuration.js';
 export interface OpenStorageOptions {
     /**
@@ -51,6 +51,8 @@ export interface OpenStorageContext {
     purgedStorageAliases: Set<string>;
 }
 /**
- * Opens a storage by its identifier, handling alias resolution and local purging.
+ * Opens a storage by its identifier, handling Apify alias resolution and local purging.
  */
-export declare function openStorage<T extends IStorage>(storageClass: Constructor<T>, identifier: StorageIdentifier | null | undefined, context: OpenStorageContext): Promise<T>;
+export declare function openStorage<T extends IStorage>(storageClass: Constructor<T> & {
+    open(id?: string | null, options?: StorageOpenOptions): Promise<T>;
+}, identifier: StorageIdentifier | null | undefined, context: OpenStorageContext): Promise<T>;

package/dist/storage.js

@@ -1,8 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.openStorage = openStorage;
-const core_1 = require("@crawlee/core");
-const apify_client_1 = require("apify-client");
+import { ApifyStorageClient } from './apify_storage_client.js';
 const STORAGE_TYPE_KEYS = {
     Dataset: 'datasets',
     KeyValueStore: 'keyValueStores',
@@ -11,7 +7,7 @@ const STORAGE_TYPE_KEYS = {
 const parsedStoragesJson = new Map();
 /**
  * Resolves a {@link StorageIdentifier} to a plain string ID or name
- * that can be passed to Crawlee's `StorageManager.openStorage()`.
+ * that can be passed to crawlee v4's `<Storage>.open()`.
  */
 function resolveStorageIdentifier(storageType, identifier, config) {
     if (identifier === null || identifier === undefined) {
@@ -27,8 +23,8 @@ function resolveStorageIdentifier(storageType, identifier, config) {
         return identifier.name;
     }
     // { alias: string }
-    const storagesJson = config.get('actorStoragesJson');
-    if (config.get('isAtHome') && storagesJson) {
+    const storagesJson = config.actorStoragesJson;
+    if (config.isAtHome && storagesJson) {
         let storages;
         try {
             if (!parsedStoragesJson.has(storagesJson)) {
@@ -50,30 +46,34 @@ function resolveStorageIdentifier(storageType, identifier, config) {
     // When using local storage, just use the alias as a name.
     // When using platform storage, we can't just make up a name — the alias must be
     // in ACTOR_STORAGES_JSON.
-    if (config.get('isAtHome')) {
+    if (config.isAtHome) {
         throw new Error(`Storage alias "${identifier.alias}" cannot be resolved because ACTOR_STORAGES_JSON is not set. ` +
             `Aliases are only available for storages declared in the Actor's schema.`);
     }
     return identifier.alias;
 }
 /**
- * Opens a storage by its identifier, handling alias resolution and local purging.
+ * Opens a storage by its identifier, handling Apify alias resolution and local purging.
  */
-async function openStorage(storageClass, identifier, context) {
+export async function openStorage(storageClass, identifier, context) {
     const isAlias = identifier !== null && identifier !== undefined && typeof identifier === 'object' && 'alias' in identifier;
-    if (isAlias && !context.config.get('isAtHome') && context.client instanceof apify_client_1.ApifyClient) {
+    if (isAlias && !context.config.isAtHome && context.client instanceof ApifyStorageClient) {
         throw new Error('The `alias` option is not allowed for Apify-based storages running outside of Apify');
     }
     const resolvedIdOrName = resolveStorageIdentifier(storageClass.name, identifier, context.config);
     // When running locally, purge aliased storages on first open
-    // (similar to how Crawlee purges default storages on start)
+    // (similar to how crawlee purges default storages on start).
     if (isAlias &&
-        !context.config.get('isAtHome') &&
-        context.config.get('purgeOnStart') &&
+        !context.config.isAtHome &&
+        context.config.purgeOnStart &&
         !context.purgedStorageAliases.has(identifier.alias)) {
         context.purgedStorageAliases.add(identifier.alias);
-        const existingStorage = await core_1.StorageManager.openStorage(storageClass, resolvedIdOrName, context.client, context.config);
+        const existingStorage = await storageClass.open(resolvedIdOrName ?? null, {
+            storageClient: context.client,
+        });
         await existingStorage.drop();
     }
-    return core_1.StorageManager.openStorage(storageClass, resolvedIdOrName, context.client, context.config);
+    return storageClass.open(resolvedIdOrName ?? null, {
+        storageClient: context.client,
+    });
 }

package/dist/utils.d.ts

@@ -13,3 +13,8 @@ export declare function getSystemInfo(): {
  * @internal
  */
 export declare function checkCrawleeVersion(): void;
+/**
+ * Prints a warning if this version of Apify SDK is outdated.
+ * @ignore
+ */
+export declare function printOutdatedSdkWarning(): void;

package/dist/utils.js

@@ -1,33 +1,36 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getSystemInfo = getSystemInfo;
-exports.checkCrawleeVersion = checkCrawleeVersion;
-const node_os_1 = require("node:os");
-const node_path_1 = require("node:path");
+import { existsSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { type } from 'node:os';
+import { normalize } from 'node:path';
 // @ts-ignore if we enable resolveJsonModule, we end up with `src` folder in `dist`
-const package_json_1 = require("@crawlee/core/package.json");
+import crawleePkgJson from '@crawlee/core/package.json' with { type: 'json' };
 // @ts-ignore if we enable resolveJsonModule, we end up with `src` folder in `dist`
-const package_json_2 = require("apify-client/package.json");
-const fs_extra_1 = require("fs-extra");
+import apifyClientPkgJson from 'apify-client/package.json' with { type: 'json' };
+// eslint-disable-next-line import/extensions
+import { readJSONSync } from 'fs-extra/esm';
+import semver from 'semver';
+import { APIFY_ENV_VARS } from '@apify/consts';
+import log from '@apify/log';
 // @ts-ignore if we enable resolveJsonModule, we end up with `src` folder in `dist`
-const package_json_3 = require("../package.json");
+import apifyPkgJson from '../package.json' with { type: 'json' };
+const require = createRequire(import.meta.url);
 /**
  * Gets info about system, node version and apify package version.
  * @internal
  */
-function getSystemInfo() {
+export function getSystemInfo() {
     return {
-        apifyVersion: package_json_3.version,
-        apifyClientVersion: package_json_2.version,
-        crawleeVersion: package_json_1.version,
-        osType: (0, node_os_1.type)(),
+        apifyVersion: apifyPkgJson.version,
+        apifyClientVersion: apifyClientPkgJson.version,
+        crawleeVersion: crawleePkgJson.version,
+        osType: type(),
         nodeVersion: process.version,
     };
 }
 /**
  * @internal
  */
-function checkCrawleeVersion() {
+export function checkCrawleeVersion() {
     const resolve = (name) => {
         try {
             return require.resolve(name);
@@ -38,28 +41,41 @@ function checkCrawleeVersion() {
     };
     const paths = [
         // when users install `crawlee` package, we need to check its core dependency
-        (0, node_path_1.normalize)(`${process.cwd()}/node_modules/crawlee/node_modules/@crawlee/core/package.json`),
+        normalize(`${process.cwd()}/node_modules/crawlee/node_modules/@crawlee/core/package.json`),
         // when users install `@crawlee/cheerio` or other crawler package, we need to check the dependency under basic crawler package
-        (0, node_path_1.normalize)(`${process.cwd()}/node_modules/@crawlee/basic/node_modules/@crawlee/core/package.json`),
+        normalize(`${process.cwd()}/node_modules/@crawlee/basic/node_modules/@crawlee/core/package.json`),
         // also check paths via `require.resolve` to support pnpm
         resolve('crawlee/package.json'),
         resolve('@crawlee/basic/package.json'),
     ];
     for (const path of paths) {
         // ignore unresolved paths or paths that are not in the project directory
-        if (!(0, fs_extra_1.pathExistsSync)(path) || !path.startsWith(process.cwd())) {
+        if (!existsSync(path) || !path.startsWith(process.cwd())) {
             continue;
         }
         let version;
         try {
-            version = (0, fs_extra_1.readJSONSync)(path).version;
+            version = readJSONSync(path).version;
         }
         catch {
             //
         }
-        if (version != null && version !== package_json_1.version) {
-            const details = `User installed version (${version}) found in ${path}.\nSDK uses ${package_json_1.version} from ${require.resolve('@crawlee/core')}`;
-            throw new Error(`Detected incompatible Crawlee version used by the SDK. User installed ${version} but the SDK uses ${package_json_1.version}.\n\n${details}`);
+        if (version != null && version !== crawleePkgJson.version) {
+            const details = `User installed version (${version}) found in ${path}.\nSDK uses ${crawleePkgJson.version} from ${require.resolve('@crawlee/core')}`;
+            throw new Error(`Detected incompatible Crawlee version used by the SDK. User installed ${version} but the SDK uses ${crawleePkgJson.version}.\n\n${details}`);
         }
     }
 }
+/**
+ * Prints a warning if this version of Apify SDK is outdated.
+ * @ignore
+ */
+export function printOutdatedSdkWarning() {
+    if (process.env[APIFY_ENV_VARS.DISABLE_OUTDATED_WARNING])
+        return;
+    const latestApifyVersion = process.env[APIFY_ENV_VARS.SDK_LATEST_VERSION];
+    if (!latestApifyVersion || !semver.lt(apifyPkgJson.version, latestApifyVersion))
+        return;
+    log.warning(`You are using an outdated version (${apifyPkgJson.version}) of Apify SDK. We recommend you to update to the latest version (${latestApifyVersion}).
+         Read more about Apify SDK versioning at: https://help.apify.com/en/articles/3184510-updates-and-versioning-of-apify-sdk`);
+}

package/package.json

@@ -1,18 +1,17 @@
 {
   "name": "apify",
-  "version": "3.7.3-beta.9",
+  "version": "4.0.0-beta.13",
   "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=22.0.0"
   },
+  "type": "module",
   "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
-      "import": "./dist/index.mjs",
-      "require": "./dist/index.js",
-      "types": "./dist/index.d.ts"
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
     },
     "./package.json": "./package.json"
   },
@@ -59,20 +58,23 @@
     "@apify/log": "^2.4.3",
     "@apify/timeout": "^0.3.0",
     "@apify/utilities": "^2.13.0",
-    "@crawlee/core": "^3.14.1",
-    "@crawlee/types": "^3.14.1",
-    "@crawlee/utils": "^3.14.1",
+    "@crawlee/core": "^4.0.0-beta.61",
+    "@crawlee/types": "^4.0.0-beta.61",
+    "@crawlee/utils": "^4.0.0-beta.61",
     "apify-client": "^2.17.0",
     "fs-extra": "^11.2.0",
-    "ow": "^0.28.2",
+    "got-scraping": "^4.2.1",
+    "ow": "^2.0.0",
     "semver": "^7.5.4",
     "tslib": "^2.6.2",
-    "ws": "^8.18.0"
+    "ws": "^8.18.0",
+    "zod": "^3.24.0 || ^4.0.0"
   },
   "devDependencies": {
     "@apify/oxlint-config": "^0.2.5",
     "@apify/tsconfig": "^0.1.2",
     "@commitlint/config-conventional": "^21.0.0",
+    "@crawlee/memory-storage": "^4.0.0-beta.61",
     "@playwright/browser-chromium": "^1.60.0",
     "@types/content-type": "^1.1.8",
     "@types/fs-extra": "^11.0.4",
@@ -81,13 +83,12 @@
     "@types/tough-cookie": "^4.0.5",
     "@types/ws": "^8.5.12",
     "commitlint": "^21.0.0",
-    "crawlee": "^3.13.5",
-    "gen-esm-wrapper": "^1.1.3",
+    "crawlee": "^4.0.0-beta.61",
     "globby": "^16.0.0",
     "husky": "^9.1.7",
     "lint-staged": "^17.0.0",
     "oxfmt": "0.52.0",
-    "oxlint": "1.68.0",
+    "oxlint": "1.67.0",
     "oxlint-tsgolint": "0.22.0",
     "playwright": "^1.60.0",
     "puppeteer": "^25.0.0",
@@ -106,7 +107,7 @@
   },
   "scripts": {
     "clean": "rimraf ./dist ./tsconfig.build.tsbuildinfo",
-    "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
+    "compile": "tsc -p tsconfig.build.json",
     "fixApifyExport": "node ./scripts/temp_fix_apify_exports.mjs",
     "build": "pnpm clean && pnpm compile && pnpm fixApifyExport",
     "ci:build": "pnpm build",