crawlee-one 1.0.0

package/dist/cjs/lib/actor.d.ts

@@ -0,0 +1,189 @@
+/// <reference types="lodash" />
+import { Actor } from 'apify';
+import { BasicCrawler, CrawlingContext, RouterHandler, ProxyConfiguration, BasicCrawlerOptions } from 'crawlee';
+import * as Sentry from '@sentry/node';
+import type { CrawlerMeta, CrawlerType } from '../types';
+import type { MaybePromise } from '../utils/types';
+import { RouteHandler, CrawlerRouterWrapper, RouteMatcher } from './router';
+import { MetamorphActorInput } from './config';
+import { itemCacheKey, pushData } from './dataset/pushData';
+type MaybeAsyncFn<R, Args extends any[]> = R | ((...args: Args) => MaybePromise<R>);
+export interface ActorDefinition<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>> {
+    /**
+     * Actor input which you can get e.g. via `Actor.getInput()`
+     *
+     * Input is automatically retrieved if undefined.
+     */
+    input?: MaybeAsyncFn<Input, [ActorDefinition<Ctx, Labels, Input>]>;
+    /** Validation for the actor input. Should throw error if validation fails. */
+    validateInput?: (input: Input | null) => MaybePromise<void>;
+    /**
+     * Router instance that redirects the request to handlers.
+     * @example
+     * import { createCheerioRouter } from 'crawlee';
+     *
+     * ({
+     *    ...
+     *   router: createCheerioRouter(),
+     * })
+     */
+    router: MaybeAsyncFn<RouterHandler<Ctx>, [ActorDefinitionWithInput<Ctx, Labels, Input>]>;
+    /**
+     * Criteria that un-labelled requests are matched against.
+     *
+     * E.g. If `match` function returns truthy value,
+     * the request is passed to the `action` function for processing.
+     *
+     * @example
+     * ({
+     *   ...
+     *   routes: [{
+     *     // If match returns true, the request is forwarded to handler
+     *     // with label JOB_DETAIL.
+     *     name: 'Job detail',
+     *     handlerLabel: routeLabels.JOB_DETAIL,
+     *     match: (url) => isUrlOfJobOffer(url),
+     *   }, {
+     *     // Define custom action function:
+     *     // If match returns true, we replace this request with new one
+     *     // pointing to new domain.
+     *     name: 'Main page',
+     *     handlerLabel: null,
+     *     match: (url) => url.match(/example\.com\/?(?:[?#~]|$)/i),
+     *     action: async (url, ctx, _, handlers) => {
+     *       ctx.log.info(`Redirecting to https://www.new-domain.com`);
+     *       await ctx.crawler.addRequests(['https://www.new-domain.com'], { forefront: true });
+     *     },
+     *   }],
+     * })
+     */
+    routes: MaybeAsyncFn<RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input>, Labels>[], [
+        ActorDefinitionWithInput<Ctx, Labels, Input>
+    ]>;
+    /** Handlers for the labelled requests. The object keys are the labels. */
+    routeHandlers: MaybeAsyncFn<Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input>>>, [ActorDefinitionWithInput<Ctx, Labels, Input>]>;
+    /**
+     * Provides the option to modify or extend all router handlers by wrapping
+     * them in these functions.
+     *
+     * Wrappers are applied from right to left. That means that wrappers `[A, B, C]`
+     * will be applied like so `A( B( C( handler ) ) )`.
+     */
+    routerWrappers?: MaybeAsyncFn<CrawlerRouterWrapper<Ctx, ActorRouterContext<Ctx, Labels, Input>>[], [ActorDefinitionWithInput<Ctx, Labels, Input>]>;
+    proxy?: MaybeAsyncFn<ProxyConfiguration, [ActorDefinitionWithInput<Ctx, Labels, Input>]>;
+    createCrawler: (actorCtx: Omit<ActorContext<Ctx, Labels, Input>, 'crawler' | 'runCrawler' | 'metamorph' | 'pushData'>) => MaybePromise<Ctx['crawler']>;
+}
+/** ActorDefinition object where the input is already resolved */
+export type ActorDefinitionWithInput<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>> = Omit<ActorDefinition<Ctx, Labels, Input>, 'input'> & {
+    input: Input | null;
+    state: Record<string, unknown>;
+};
+/** Context available while creating an Apify/Crawlee crawler */
+export interface ActorContext<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>> {
+    crawler: Ctx['crawler'];
+    /**
+     * This function wraps `crawler.run(requests, runOtions)` with additional
+     * features:
+     * - Automatically metamorph into another actor after the run finishes
+     */
+    runCrawler: RunCrawler<Ctx>;
+    /** Trigger actor metamorph, using actor's inputs as defaults. */
+    metamorph: Metamorph;
+    pushData: typeof pushData;
+    proxy?: ProxyConfiguration;
+    router: RouterHandler<Ctx>;
+    routes: RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input>, Labels>[];
+    routeHandlers: Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input>>>;
+    /** Original config from which this actor context was created */
+    config: ActorDefinition<Ctx, Labels, Input>;
+    /** Read-only inputs passed to the actor */
+    input: Input | null;
+    /** Mutable state that is shared across setup and teardown hooks */
+    state: Record<string, unknown>;
+}
+type OrigRunCrawler<T extends CrawlingContext<any, any>> = BasicCrawler<T>['run'];
+/** Extended type of `crawler.run()` function */
+export type RunCrawler<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>> = (requests?: Parameters<OrigRunCrawler<Ctx>>[0], options?: Parameters<OrigRunCrawler<Ctx>>[1]) => ReturnType<OrigRunCrawler<Ctx>>;
+/** Trigger actor metamorph, using actor's inputs as defaults. */
+export type Metamorph = (overrides?: MetamorphActorInput) => Promise<void>;
+/** Context passed to user-defined functions passed from input */
+export type ActorHookContext = Pick<ActorContext, 'input' | 'state'> & {
+    Actor: typeof Actor;
+    itemCacheKey: typeof itemCacheKey;
+};
+/** Context passed to route handlers */
+export type ActorRouterContext<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>> = {
+    actor: ActorContext<Ctx, Labels, Input>;
+};
+/**
+ * Create opinionated Apify crawler that uses router for handling requests.
+ *
+ * This is a quality-of-life function that does the following for you:
+ *
+ * 1) TypeScript - Ensure all components use the same Crawler / CrawlerContext.
+ *
+ * 2) Get Actor input from `Actor.getInput` if not given.
+ *
+ * 3) (Optional) Validate Actor input
+ *
+ * 4) Set up router such that requests that reach default route are
+ * redirected to labelled routes based on the "routes" items.
+ *
+ * 5) Register all route handlers for you.
+ *
+ * 6) (Optional) Wrap all route handlers in a wrapper. Use this e.g.
+ * if you want to add a field to the context object, or handle errors
+ * from a single place.
+ */
+export declare const createApifyActor: <Ctx extends CrawlingContext<unknown, import("crawlee").Dictionary> = CrawlingContext<BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>>, import("crawlee").Dictionary>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>>(config: ActorDefinition<Ctx, Labels, Input>) => Promise<ActorContext<Ctx, Labels, Input>>;
+/**
+ * Create default configuration for an Apify actor
+ * and run the actor within the `Actor.main()` context.
+ */
+export declare const createAndRunApifyActor: <TCrawlerType extends CrawlerType, Ctx extends CrawlerMeta<TCrawlerType, any>["context"] = CrawlingContext<BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>>, import("crawlee").Dictionary>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>>({ actorType, actorName, actorConfig, crawlerConfigDefaults, crawlerConfigOverrides, sentryOptions, onActorReady, }: {
+    /** String idetifying the actor class, e.g. `'cheerio'` */
+    actorType: TCrawlerType;
+    actorName: string;
+    /** Config passed to the {@link createApifyActor} */
+    actorConfig: Omit<ActorDefinition<Ctx, Labels, Input>, "router" | "createCrawler"> & Partial<Pick<ActorDefinition<Ctx, Labels, Input>, "router" | "createCrawler">>;
+    /**
+     * If using default `createCrawler` implementation, these are crawler options
+     * that may be overriden by user input.
+     */
+    crawlerConfigDefaults?: CrawlerMeta<TCrawlerType, any>["options"] | undefined;
+    /**
+     * If using default `createCrawler` implementation, these are crawler options
+     * that will override user input.
+     *
+     * This is useful for testing env.
+     */
+    crawlerConfigOverrides?: CrawlerMeta<TCrawlerType, any>["options"] | undefined;
+    /**
+     * Sentry configuration. If using default `createCrawler` implementation,
+     * failed requests are optionally reported to Sentry.
+     *
+     * To disable Sentry, set `"enabled": false`.
+     */
+    sentryOptions?: Sentry.NodeOptions | undefined;
+    /**
+     * Callback with the created actor. The callback is called within
+     * the `Actor.main()` context.
+     */
+    onActorReady?: ((actor: ActorContext<Ctx, Labels, Input>) => MaybePromise<void>) | undefined;
+}) => Promise<void>;
+/** Given the actor input, create common crawler options. */
+export declare const createHttpCrawlerOptions: <TOpts extends BasicCrawlerOptions<any> = BasicCrawlerOptions<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>>, Input extends Record<string, any> = Record<string, any>>({ input, defaults, overrides, }: {
+    /** Actor input */
+    input: Input | null;
+    /**
+     * Default config options set by us. These may be overriden
+     * by values from actor input (set by user).
+     */
+    defaults?: TOpts | undefined;
+    /**
+     * These config options will overwrite both the default and user
+     * options. This is useful for hard-setting values e.g. in tests.
+     */
+    overrides?: TOpts | undefined;
+}) => Partial<TOpts> & import("lodash").Dictionary<TOpts["requestHandler"] | TOpts["handleRequestFunction"] | TOpts["requestList"] | TOpts["requestQueue"] | TOpts["requestHandlerTimeoutSecs"] | TOpts["handleRequestTimeoutSecs"] | TOpts["errorHandler"] | TOpts["failedRequestHandler"] | TOpts["handleFailedRequestFunction"] | TOpts["maxRequestRetries"] | TOpts["maxRequestsPerCrawl"] | TOpts["autoscaledPoolOptions"] | TOpts["minConcurrency"] | TOpts["maxConcurrency"] | TOpts["maxRequestsPerMinute"] | TOpts["keepAlive"] | TOpts["useSessionPool"] | TOpts["sessionPoolOptions"] | TOpts["loggingInterval"] | TOpts["log"]>;
+export {};

package/dist/cjs/lib/actor.js

@@ -0,0 +1,225 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createHttpCrawlerOptions = exports.createAndRunApifyActor = exports.createApifyActor = void 0;
+const apify_1 = require("apify");
+const crawlee_1 = require("crawlee");
+const lodash_1 = require("lodash");
+const router_1 = require("./router");
+const config_1 = require("./config");
+const errorHandler_1 = require("./error/errorHandler");
+const sentry_1 = require("./error/sentry");
+const log_1 = require("./log");
+const pushData_1 = require("./dataset/pushData");
+const isRouter = (r) => {
+    return !!(r.addHandler && r.addDefaultHandler);
+};
+const isFunc = (f) => {
+    return typeof f === 'function';
+};
+/** Run a function that was defined as a string via Actor input */
+const evalInputHook = (actor, fnStr, args = []) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!fnStr)
+        return;
+    const hookCtx = {
+        Actor: apify_1.Actor,
+        input: actor.input,
+        state: actor.state,
+        itemCacheKey: pushData_1.itemCacheKey,
+    };
+    const hookFn = eval(fnStr);
+    yield hookFn(...args, hookCtx);
+});
+/**
+ * Create opinionated Apify crawler that uses router for handling requests.
+ *
+ * This is a quality-of-life function that does the following for you:
+ *
+ * 1) TypeScript - Ensure all components use the same Crawler / CrawlerContext.
+ *
+ * 2) Get Actor input from `Actor.getInput` if not given.
+ *
+ * 3) (Optional) Validate Actor input
+ *
+ * 4) Set up router such that requests that reach default route are
+ * redirected to labelled routes based on the "routes" items.
+ *
+ * 5) Register all route handlers for you.
+ *
+ * 6) (Optional) Wrap all route handlers in a wrapper. Use this e.g.
+ * if you want to add a field to the context object, or handle errors
+ * from a single place.
+ */
+const createApifyActor = (config) => __awaiter(void 0, void 0, void 0, function* () {
+    // Initialize actor inputs
+    const input = Object.freeze(config.input
+        ? isFunc(config.input)
+            ? yield config.input(Object.assign({}, config))
+            : config.input
+        : yield apify_1.Actor.getInput());
+    if (config.validateInput)
+        yield config.validateInput(input);
+    // Mutable state that is available to the actor hooks
+    const state = {};
+    // This is context that is available to options that use initialization function
+    const getConfig = () => (Object.assign(Object.assign({}, config), { input, state }));
+    // Set up proxy
+    const defaultProxy = config.proxy == null && process.env.APIFY_IS_AT_HOME
+        ? yield apify_1.Actor.createProxyConfiguration(input === null || input === void 0 ? void 0 : input.proxy)
+        : undefined;
+    const proxy = config.proxy == null
+        ? defaultProxy
+        : isFunc(config.proxy)
+            ? yield config.proxy(getConfig())
+            : config.proxy;
+    // Run initialization functions
+    const router = isRouter(config.router)
+        ? config.router
+        : yield config.router(getConfig());
+    const routes = isFunc(config.routes) ? yield config.routes(getConfig()) : config.routes; // prettier-ignore
+    const routeHandlers = isFunc(config.routeHandlers) ? yield config.routeHandlers(getConfig()) : config.routeHandlers; // prettier-ignore
+    const routerWrappers = isFunc(config.routerWrappers) ? yield config.routerWrappers(getConfig()) : config.routerWrappers; // prettier-ignore
+    // Create Crawlee crawler
+    const getActorCtx = () => ({ router, routes, routeHandlers, proxy, config, input, state });
+    const crawler = yield config.createCrawler(getActorCtx());
+    // Create actor (our custom entity)
+    const preActor = Object.assign({ crawler }, getActorCtx());
+    const runCrawler = createScopedCrawlerRun(preActor);
+    const metamorph = createScopedMetamorph(preActor);
+    const scopedPushData = createScopedPushData(preActor);
+    const actor = Object.assign(Object.assign({}, preActor), { crawler,
+        runCrawler,
+        metamorph, pushData: scopedPushData });
+    // Extra data that we make available to the route handlers
+    const routerContext = { actor, pushData: scopedPushData };
+    // Set up router
+    yield (0, router_1.setupDefaultRoute)({
+        router,
+        routerWrappers,
+        routerContext,
+        routes,
+        routeHandlers,
+    });
+    yield (0, router_1.registerHandlers)({
+        router,
+        routerWrappers,
+        routerContext,
+        routeHandlers,
+    });
+    return actor;
+});
+exports.createApifyActor = createApifyActor;
+/** Create a function that triggers metamorph, using Actor's inputs as defaults. */
+const createScopedMetamorph = (actor) => {
+    // Trigger metamorph if it was set from the input
+    const metamorph = (overrides) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const { metamorphActorId, metamorphActorBuild, metamorphActorInput, } = (0, lodash_1.defaults)({}, overrides, (_a = actor.input) !== null && _a !== void 0 ? _a : {}); // prettier-ignore
+        if (!metamorphActorId)
+            return;
+        yield apify_1.Actor.metamorph(metamorphActorId, metamorphActorInput, { build: metamorphActorBuild });
+    });
+    return metamorph;
+};
+/** pushData wrapper that pre-populates options based on actor input */
+const createScopedPushData = (actor) => {
+    const scopedPushData = (entries, ctx, options) => {
+        var _a;
+        const { includePersonalData, outputTransform, outputFilter, outputDatasetIdOrName, outputPickFields, outputRenameFields, outputCacheStoreIdOrName, outputCachePrimaryKeys, outputCacheActionOnResult, } = ((_a = actor.input) !== null && _a !== void 0 ? _a : {});
+        const mergedOptions = Object.assign({ showPrivate: includePersonalData, pickKeys: outputPickFields, remapKeys: outputRenameFields, transform: outputTransform ? ((item) => evalInputHook(actor, outputTransform, [item])) : undefined, filter: outputFilter ? ((item) => evalInputHook(actor, outputFilter, [item])) : undefined, datasetIdOrName: outputDatasetIdOrName, cacheStoreIdOrName: outputCacheStoreIdOrName, cachePrimaryKeys: outputCachePrimaryKeys, cacheActionOnResult: outputCacheActionOnResult }, options);
+        return (0, pushData_1.pushData)(entries, ctx, mergedOptions);
+    };
+    return scopedPushData;
+};
+/**
+ * Create a function that wraps `crawler.run(requests, runOtions)` with additional
+ * features like:
+ * - Automatically metamorph into another actor after the run finishes
+ */
+const createScopedCrawlerRun = (actor) => {
+    var _a;
+    const { outputTransformBefore, outputTransformAfter, outputFilterBefore, outputFilterAfter, outputCacheStoreIdOrName, outputCacheActionOnResult, } = ((_a = actor.input) !== null && _a !== void 0 ? _a : {});
+    const metamorph = createScopedMetamorph(actor);
+    const runCrawler = (requests, options) => __awaiter(void 0, void 0, void 0, function* () {
+        // Clear cache if it was set from the input
+        if (outputCacheStoreIdOrName && outputCacheActionOnResult === 'overwrite') {
+            const store = yield apify_1.Actor.openKeyValueStore(outputCacheStoreIdOrName);
+            yield store.drop();
+        }
+        yield evalInputHook(actor, outputTransformBefore);
+        yield evalInputHook(actor, outputFilterBefore);
+        const runRes = yield actor.crawler.run(requests, options);
+        yield evalInputHook(actor, outputTransformAfter);
+        yield evalInputHook(actor, outputFilterAfter);
+        // Trigger metamorph if it was set from the input
+        yield metamorph();
+        return runRes;
+    });
+    return runCrawler;
+};
+const actorClassByType = {
+    basic: crawlee_1.BasicCrawler,
+    http: crawlee_1.HttpCrawler,
+    cheerio: crawlee_1.CheerioCrawler,
+    jsdom: crawlee_1.JSDOMCrawler,
+    playwright: crawlee_1.PlaywrightCrawler,
+    puppeteer: crawlee_1.PuppeteerCrawler,
+};
+/**
+ * Create default configuration for an Apify actor
+ * and run the actor within the `Actor.main()` context.
+ */
+const createAndRunApifyActor = ({ actorType, actorName, actorConfig, crawlerConfigDefaults, crawlerConfigOverrides, sentryOptions, onActorReady, }) => __awaiter(void 0, void 0, void 0, function* () {
+    (0, sentry_1.setupSentry)(Object.assign(Object.assign({}, sentryOptions), { serverName: actorName }));
+    // See docs:
+    // - https://docs.apify.com/sdk/js/
+    // - https://docs.apify.com/academy/deploying-your-code/inputs-outputs#accepting-input-with-the-apify-sdk
+    // - https://docs.apify.com/sdk/js/docs/upgrading/upgrading-to-v3#apify-sdk
+    yield apify_1.Actor.main(() => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, _b, _c;
+        const actorDefaults = {
+            router: crawlee_1.Router.create(),
+            routerWrappers: ({ input }) => {
+                var _a;
+                return [
+                    (0, log_1.logLevelHandlerWrapper)((_a = input === null || input === void 0 ? void 0 : input.logLevel) !== null && _a !== void 0 ? _a : 'info'),
+                ];
+            },
+            createCrawler: ({ router, proxy, input }) => {
+                var _a, _b;
+                const options = (0, exports.createHttpCrawlerOptions)({
+                    input,
+                    defaults: crawlerConfigDefaults,
+                    overrides: Object.assign({ requestHandler: router, proxyConfiguration: proxy, 
+                        // Capture errors as a separate Apify/Actor dataset and pass errors to Sentry
+                        failedRequestHandler: (0, errorHandler_1.createErrorHandler)({
+                            reportingDatasetId: (_a = input === null || input === void 0 ? void 0 : input.errorReportingDatasetId) !== null && _a !== void 0 ? _a : 'REPORTING',
+                            sendToSentry: (_b = input === null || input === void 0 ? void 0 : input.errorSendToSentry) !== null && _b !== void 0 ? _b : true,
+                        }) }, crawlerConfigOverrides),
+                });
+                const CrawlerClass = actorClassByType[actorType];
+                return new CrawlerClass(options);
+            },
+            routes: [],
+            routeHandlers: {},
+        };
+        const actor = yield (0, exports.createApifyActor)(Object.assign(Object.assign({}, actorConfig), { router: (_a = actorConfig.router) !== null && _a !== void 0 ? _a : actorDefaults.router, routerWrappers: (_b = actorConfig.routerWrappers) !== null && _b !== void 0 ? _b : actorDefaults.routerWrappers, createCrawler: (_c = actorConfig.createCrawler) !== null && _c !== void 0 ? _c : actorDefaults.createCrawler }));
+        yield (onActorReady === null || onActorReady === void 0 ? void 0 : onActorReady(actor));
+    }), { statusMessage: 'Crawling finished!' });
+});
+exports.createAndRunApifyActor = createAndRunApifyActor;
+/** Given the actor input, create common crawler options. */
+const createHttpCrawlerOptions = ({ input, defaults, overrides, }) => {
+    const pickCrawlerInputFields = (config) => (0, lodash_1.pick)(config, Object.keys(config_1.crawlerInput));
+    return Object.assign(Object.assign(Object.assign({}, (0, lodash_1.omitBy)(defaults !== null && defaults !== void 0 ? defaults : {}, (field) => field === undefined)), (0, lodash_1.omitBy)(pickCrawlerInputFields(input !== null && input !== void 0 ? input : {}), (field) => field === undefined)), (0, lodash_1.omitBy)(overrides !== null && overrides !== void 0 ? overrides : {}, (field) => field === undefined));
+};
+exports.createHttpCrawlerOptions = createHttpCrawlerOptions;
+//# sourceMappingURL=actor.js.map

package/dist/cjs/lib/actor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actor.js","sourceRoot":"","sources":["../../../src/lib/actor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iCAA8B;AAC9B,qCAYiB;AACjB,mCAAgD;AAKhD,qCAMkB;AAClB,qCAQkB;AAClB,uDAA0D;AAC1D,2CAA6C;AAC7C,+BAA+C;AAC/C,iDAA4D;AAI5D,MAAM,QAAQ,GAAG,CAAC,CAAM,EAA2B,EAAE;IACnD,OAAO,CAAC,CAAC,CAAE,CAAmB,CAAC,UAAU,IAAK,CAAmB,CAAC,iBAAiB,CAAC,CAAC;AACvF,CAAC,CAAC;AACF,MAAM,MAAM,GAAG,CAAC,CAAM,EAAgC,EAAE;IACtD,OAAO,OAAO,CAAC,KAAK,UAAU,CAAC;AACjC,CAAC,CAAC;AAEF,kEAAkE;AAClE,MAAM,aAAa,GAAG,CAKpB,KAAgE,EAChE,KAAc,EACd,OAAc,EAAE,EAChB,EAAE;IACF,IAAI,CAAC,KAAK;QAAE,OAAO;IAEnB,MAAM,OAAO,GAAG;QACd,KAAK,EAAL,aAAK;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAZ,uBAAY;KACc,CAAC;IAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,MAAM,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AACjC,CAAC,CAAA,CAAC;AAqJF;;;;;;;;;;;;;;;;;;;GAmBG;AACI,MAAM,gBAAgB,GAAG,CAK9B,MAA2C,EACA,EAAE;IAC7C,0BAA0B;IAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CACzB,MAAM,CAAC,KAAK;QACV,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YACpB,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,mBAAM,MAAM,EAAG;YACnC,CAAC,CAAC,MAAM,CAAC,KAAK;QAChB,CAAC,CAAC,MAAM,aAAK,CAAC,QAAQ,EAAS,CAClC,CAAC;IAEF,IAAI,MAAM,CAAC,aAAa;QAAE,MAAM,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAE5D,qDAAqD;IACrD,MAAM,KAAK,GAAG,EAAE,CAAC;IAEjB,gFAAgF;IAChF,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,iCAAM,MAAM,KAAE,KAAK,EAAE,KAAK,IAAG,CAAC;IAEtD,eAAe;IACf,MAAM,YAAY,GAChB,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAClD,CAAC,CAAC,MAAM,aAAK,CAAC,wBAAwB,CAAC,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,CAAC;QACpD,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,KAAK,GACT,MAAM,CAAC,KAAK,IAAI,IAAI;QAClB,CAAC,CAAC,YAAY;QACd,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YACtB,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IAEnB,+BAA+B;IAC/B,MAAM,MAAM,GAAuB,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;QACxD,CAAC,CAAC,MAAM,CAAC,MAAM;QACf,CAAC,CAAC,MAAO,MAAM,CAAC,MAAc,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,kBAAkB;IAC3G,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,kBAAkB;IACvI,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,kBAAkB;IAE3I,yBAAyB;IACzB,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;IAE1D,mCAAmC;IACnC,MAAM,QAAQ,mBAAK,OAAO,IAAK,WAAW,EAAE,CAAE,CAAC;IAC/C,MAAM,UAAU,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,OAAO;QACP,UAAU;QACV,SAAS,EACT,QAAQ,EAAE,cAAc,GACkB,CAAC;IAE7C,0DAA0D;IAC1D,MAAM,aAAa,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;IAE1D,gBAAgB;IAChB,MAAM,IAAA,0BAAiB,EAAsD;QAC3E,MAAM;QACN,cAAc;QACd,aAAa;QACb,MAAM;QACN,aAAa;KACd,CAAC,CAAC;IACH,MAAM,IAAA,yBAAgB,EAAsD;QAC1E,MAAM;QACN,cAAc;QACd,aAAa;QACb,aAAa;KACd,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC,CAAA,CAAC;AAjFW,QAAA,gBAAgB,oBAiF3B;AAEF,mFAAmF;AACnF,MAAM,qBAAqB,GAAG,CAAC,KAAkC,EAAE,EAAE;IACnE,iDAAiD;IACjD,MAAM,SAAS,GAAc,CAAO,SAA+B,EAAE,EAAE;;QACrE,MAAM,EACJ,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,GACpB,GAAG,IAAA,iBAAQ,EAAC,EAAE,EAAE,SAAS,EAAE,MAAA,KAAK,CAAC,KAAK,mCAAI,EAAE,CAAC,CAAC,CAAC,kBAAkB;QAElE,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAE9B,MAAM,aAAK,CAAC,SAAS,CAAC,gBAAgB,EAAE,mBAAmB,EAAE,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAA,CAAC;IAEF,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,oBAAoB,GAAG,CAAC,KAA4C,EAAE,EAAE;IAC5E,MAAM,cAAc,GAAoB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;;QAChE,MAAM,EACJ,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,kBAAkB,EAClB,wBAAwB,EACxB,sBAAsB,EACtB,yBAAyB,GAC1B,GAAG,CAAC,MAAA,KAAK,CAAC,KAAK,mCAAI,EAAE,CAAyC,CAAC;QAEhE,MAAM,aAAa,mBACjB,WAAW,EAAE,mBAAmB,EAChC,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAClG,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EACzF,eAAe,EAAE,qBAAqB,EACtC,kBAAkB,EAAE,wBAAwB,EAC5C,gBAAgB,EAAE,sBAAsB,EACxC,mBAAmB,EAAE,yBAAyB,IAC3C,OAAO,CACX,CAAC;QAEF,OAAO,IAAA,mBAAQ,EAAC,OAAO,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,CAK7B,KAAsF,EACtF,EAAE;;IACF,MAAM,EACJ,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,wBAAwB,EACxB,yBAAyB,GAC1B,GAAG,CAAC,MAAA,KAAK,CAAC,KAAK,mCAAI,EAAE,CAAqB,CAAC;IAE5C,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAE/C,MAAM,UAAU,GAAoB,CAAO,QAAQ,EAAE,OAAO,EAAE,EAAE;QAC9D,2CAA2C;QAC3C,IAAI,wBAAwB,IAAI,yBAAyB,KAAK,WAAW,EAAE;YACzE,MAAM,KAAK,GAAG,MAAM,aAAK,CAAC,iBAAiB,CAAC,wBAAwB,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,MAAM,aAAa,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;QAClD,MAAM,aAAa,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE1D,MAAM,aAAa,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC;QACjD,MAAM,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;QAE9C,iDAAiD;QACjD,MAAM,SAAS,EAAE,CAAC;QAElB,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG;IACvB,KAAK,EAAE,sBAAY;IACnB,IAAI,EAAE,qBAAW;IACjB,OAAO,EAAE,wBAAc;IACvB,KAAK,EAAE,sBAAY;IACnB,UAAU,EAAE,2BAAiB;IAC7B,SAAS,EAAE,0BAAgB;CAC+C,CAAC;AAS7E;;;GAGG;AACI,MAAM,sBAAsB,GAAG,CAKpC,EACA,SAAS,EACT,SAAS,EACT,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,aAAa,EACb,YAAY,GAgCb,EAAiB,EAAE;IAClB,IAAA,oBAAW,kCAAM,aAAa,KAAE,UAAU,EAAE,SAAS,IAAG,CAAC;IAEzD,YAAY;IACZ,mCAAmC;IACnC,yGAAyG;IACzG,2EAA2E;IAC3E,MAAM,aAAK,CAAC,IAAI,CACd,GAAS,EAAE;;QACT,MAAM,aAAa,GAAoD;YACrE,MAAM,EAAE,gBAAM,CAAC,MAAM,EAAO;YAC5B,cAAc,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;;gBAAC,OAAA;oBAC7B,IAAA,4BAAsB,EAAW,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,mCAAI,MAAM,CAAC;iBAC5D,CAAA;aAAA;YACD,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;;gBAC1C,MAAM,OAAO,GAAG,IAAA,gCAAwB,EAGtC;oBACA,KAAK;oBACL,QAAQ,EAAE,qBAAqB;oBAC/B,SAAS,kBACP,cAAc,EAAE,MAAM,EACtB,kBAAkB,EAAE,KAAK;wBACzB,6EAA6E;wBAC7E,oBAAoB,EAAE,IAAA,iCAAkB,EAAC;4BACvC,kBAAkB,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,uBAAuB,mCAAI,WAAW;4BACjE,YAAY,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,iBAAiB,mCAAI,IAAI;yBAC/C,CAAC,IACC,sBAAsB,CAC1B;iBACF,CAAC,CAAC;gBACH,MAAM,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAQ,CAAC;gBACxD,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAS;SACzB,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,IAAA,wBAAgB,kCAC/B,WAAW,KACd,MAAM,EAAE,MAAA,WAAW,CAAC,MAAM,mCAAK,aAAa,CAAC,MAAc,EAC3D,cAAc,EAAE,MAAA,WAAW,CAAC,cAAc,mCAAK,aAAa,CAAC,cAAsB,EACnF,aAAa,EAAE,MAAA,WAAW,CAAC,aAAa,mCAAK,aAAa,CAAC,aAAqB,IAChF,CAAC;QAEH,MAAM,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAG,KAAK,CAAC,CAAA,CAAC;IAC9B,CAAC,CAAA,EACD,EAAE,aAAa,EAAE,oBAAoB,EAAE,CACxC,CAAC;AACJ,CAAC,CAAA,CAAC;AA9FW,QAAA,sBAAsB,0BA8FjC;AAEF,4DAA4D;AACrD,MAAM,wBAAwB,GAAG,CAGtC,EACA,KAAK,EACL,QAAQ,EACR,SAAS,GAcV,EAAE,EAAE;IACH,MAAM,sBAAsB,GAAG,CAAoC,MAAS,EAAE,EAAE,CAC9E,IAAA,aAAI,EAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,qBAAY,CAAC,CAAC,CAAC;IAE1C,OAAO,8CAEF,IAAA,eAAM,EAAC,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAK,EAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,GAEjE,IAAA,eAAM,EAAC,sBAAsB,CAAC,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,GAE3E,IAAA,eAAM,EAAC,SAAS,aAAT,SAAS,cAAT,SAAS,GAAK,EAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAC7C,CAAC;AAC7B,CAAC,CAAC;AAhCW,QAAA,wBAAwB,4BAgCnC","sourcesContent":["import { Actor } from 'apify';\nimport {\n  BasicCrawler,\n  CrawlingContext,\n  RouterHandler,\n  ProxyConfiguration,\n  BasicCrawlerOptions,\n  CheerioCrawler,\n  Router,\n  HttpCrawler,\n  JSDOMCrawler,\n  PlaywrightCrawler,\n  PuppeteerCrawler,\n} from 'crawlee';\nimport { omitBy, pick, defaults } from 'lodash';\nimport * as Sentry from '@sentry/node';\n\nimport type { CrawlerMeta, CrawlerType } from '../types';\nimport type { MaybePromise } from '../utils/types';\nimport {\n  RouteHandler,\n  CrawlerRouterWrapper,\n  RouteMatcher,\n  registerHandlers,\n  setupDefaultRoute,\n} from './router';\nimport {\n  CrawlerConfigActorInput,\n  LoggingActorInput,\n  OutputActorInput,\n  MetamorphActorInput,\n  PrivacyActorInput,\n  ProxyActorInput,\n  crawlerInput,\n} from './config';\nimport { createErrorHandler } from './error/errorHandler';\nimport { setupSentry } from './error/sentry';\nimport { logLevelHandlerWrapper } from './log';\nimport { itemCacheKey, pushData } from './dataset/pushData';\n\ntype MaybeAsyncFn<R, Args extends any[]> = R | ((...args: Args) => MaybePromise<R>);\n\nconst isRouter = (r: any): r is RouterHandler<any> => {\n  return !!((r as RouterHandler).addHandler && (r as RouterHandler).addDefaultHandler);\n};\nconst isFunc = (f: any): f is (...args: any[]) => any => {\n  return typeof f === 'function';\n};\n\n/** Run a function that was defined as a string via Actor input */\nconst evalInputHook = async <\n  Ctx extends CrawlingContext<any> = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n>(\n  actor: Pick<ActorContext<Ctx, Labels, Input>, 'input' | 'state'>,\n  fnStr?: string,\n  args: any[] = []\n) => {\n  if (!fnStr) return;\n\n  const hookCtx = {\n    Actor,\n    input: actor.input,\n    state: actor.state,\n    itemCacheKey,\n  } satisfies ActorHookContext;\n\n  const hookFn = eval(fnStr);\n  await hookFn(...args, hookCtx);\n};\n\nexport interface ActorDefinition<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n> {\n  // Actor input\n  /**\n   * Actor input which you can get e.g. via `Actor.getInput()`\n   *\n   * Input is automatically retrieved if undefined.\n   */\n  input?: MaybeAsyncFn<Input, [ActorDefinition<Ctx, Labels, Input>]>;\n  /** Validation for the actor input. Should throw error if validation fails. */\n  validateInput?: (input: Input | null) => MaybePromise<void>;\n\n  // Router setup\n  /**\n   * Router instance that redirects the request to handlers.\n   * @example\n   * import { createCheerioRouter } from 'crawlee';\n   *\n   * ({\n   *    ...\n   *   router: createCheerioRouter(),\n   * })\n   */\n  router: MaybeAsyncFn<RouterHandler<Ctx>, [ActorDefinitionWithInput<Ctx, Labels, Input>]>;\n  /**\n   * Criteria that un-labelled requests are matched against.\n   *\n   * E.g. If `match` function returns truthy value,\n   * the request is passed to the `action` function for processing.\n   *\n   * @example\n   * ({\n   *   ...\n   *   routes: [{\n   *     // If match returns true, the request is forwarded to handler\n   *     // with label JOB_DETAIL.\n   *     name: 'Job detail',\n   *     handlerLabel: routeLabels.JOB_DETAIL,\n   *     match: (url) => isUrlOfJobOffer(url),\n   *   }, {\n   *     // Define custom action function:\n   *     // If match returns true, we replace this request with new one\n   *     // pointing to new domain.\n   *     name: 'Main page',\n   *     handlerLabel: null,\n   *     match: (url) => url.match(/example\\.com\\/?(?:[?#~]|$)/i),\n   *     action: async (url, ctx, _, handlers) => {\n   *       ctx.log.info(`Redirecting to https://www.new-domain.com`);\n   *       await ctx.crawler.addRequests(['https://www.new-domain.com'], { forefront: true });\n   *     },\n   *   }],\n   * })\n   */\n  routes: MaybeAsyncFn<\n    RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input>, Labels>[],\n    [ActorDefinitionWithInput<Ctx, Labels, Input>]\n  >;\n  /** Handlers for the labelled requests. The object keys are the labels. */\n  routeHandlers: MaybeAsyncFn<Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input>>>, [ActorDefinitionWithInput<Ctx, Labels, Input>]>; // prettier-ignore\n  /**\n   * Provides the option to modify or extend all router handlers by wrapping\n   * them in these functions.\n   *\n   * Wrappers are applied from right to left. That means that wrappers `[A, B, C]`\n   * will be applied like so `A( B( C( handler ) ) )`.\n   */\n  routerWrappers?: MaybeAsyncFn<CrawlerRouterWrapper<Ctx, ActorRouterContext<Ctx, Labels, Input>>[], [ActorDefinitionWithInput<Ctx, Labels, Input>]>; // prettier-ignore\n\n  // Proxy setup\n  proxy?: MaybeAsyncFn<ProxyConfiguration, [ActorDefinitionWithInput<Ctx, Labels, Input>]>; // prettier-ignore\n\n  // Crawler setup\n  createCrawler: (\n    actorCtx: Omit<\n      ActorContext<Ctx, Labels, Input>,\n      'crawler' | 'runCrawler' | 'metamorph' | 'pushData'\n    >\n  ) => MaybePromise<Ctx['crawler']>;\n}\n\n/** ActorDefinition object where the input is already resolved */\nexport type ActorDefinitionWithInput<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n> = Omit<ActorDefinition<Ctx, Labels, Input>, 'input'> & {\n  input: Input | null;\n  state: Record<string, unknown>;\n};\n\n/** Context available while creating an Apify/Crawlee crawler */\nexport interface ActorContext<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n> {\n  crawler: Ctx['crawler'];\n  /**\n   * This function wraps `crawler.run(requests, runOtions)` with additional\n   * features:\n   * - Automatically metamorph into another actor after the run finishes\n   */\n  runCrawler: RunCrawler<Ctx>;\n  /** Trigger actor metamorph, using actor's inputs as defaults. */\n  metamorph: Metamorph;\n  pushData: typeof pushData;\n  proxy?: ProxyConfiguration;\n  router: RouterHandler<Ctx>;\n  routes: RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input>, Labels>[];\n  routeHandlers: Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input>>>;\n  /** Original config from which this actor context was created */\n  config: ActorDefinition<Ctx, Labels, Input>;\n  /** Read-only inputs passed to the actor */\n  input: Input | null;\n  /** Mutable state that is shared across setup and teardown hooks */\n  state: Record<string, unknown>;\n}\n\ntype OrigRunCrawler<T extends CrawlingContext<any, any>> = BasicCrawler<T>['run'];\n\n/** Extended type of `crawler.run()` function */\nexport type RunCrawler<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>> = (\n  requests?: Parameters<OrigRunCrawler<Ctx>>[0],\n  options?: Parameters<OrigRunCrawler<Ctx>>[1]\n) => ReturnType<OrigRunCrawler<Ctx>>;\n\n/** Trigger actor metamorph, using actor's inputs as defaults. */\nexport type Metamorph = (overrides?: MetamorphActorInput) => Promise<void>;\n\n/** Context passed to user-defined functions passed from input */\nexport type ActorHookContext = Pick<ActorContext, 'input' | 'state'> & {\n  Actor: typeof Actor;\n  itemCacheKey: typeof itemCacheKey;\n};\n\n/** Context passed to route handlers */\nexport type ActorRouterContext<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n> = {\n  actor: ActorContext<Ctx, Labels, Input>;\n};\n\n/**\n * Create opinionated Apify crawler that uses router for handling requests.\n *\n * This is a quality-of-life function that does the following for you:\n *\n * 1) TypeScript - Ensure all components use the same Crawler / CrawlerContext.\n *\n * 2) Get Actor input from `Actor.getInput` if not given.\n *\n * 3) (Optional) Validate Actor input\n *\n * 4) Set up router such that requests that reach default route are\n * redirected to labelled routes based on the \"routes\" items.\n *\n * 5) Register all route handlers for you.\n *\n * 6) (Optional) Wrap all route handlers in a wrapper. Use this e.g.\n * if you want to add a field to the context object, or handle errors\n * from a single place.\n */\nexport const createApifyActor = async <\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n>(\n  config: ActorDefinition<Ctx, Labels, Input>\n): Promise<ActorContext<Ctx, Labels, Input>> => {\n  // Initialize actor inputs\n  const input = Object.freeze(\n    config.input\n      ? isFunc(config.input)\n        ? await config.input({ ...config })\n        : config.input\n      : await Actor.getInput<Input>()\n  );\n\n  if (config.validateInput) await config.validateInput(input);\n\n  // Mutable state that is available to the actor hooks\n  const state = {};\n\n  // This is context that is available to options that use initialization function\n  const getConfig = () => ({ ...config, input, state });\n\n  // Set up proxy\n  const defaultProxy =\n    config.proxy == null && process.env.APIFY_IS_AT_HOME\n      ? await Actor.createProxyConfiguration(input?.proxy)\n      : undefined;\n  const proxy =\n    config.proxy == null\n      ? defaultProxy\n      : isFunc(config.proxy)\n      ? await config.proxy(getConfig())\n      : config.proxy;\n\n  // Run initialization functions\n  const router: RouterHandler<Ctx> = isRouter(config.router)\n    ? config.router\n    : await (config.router as any)(getConfig());\n  const routes = isFunc(config.routes) ? await config.routes(getConfig()) : config.routes; // prettier-ignore\n  const routeHandlers = isFunc(config.routeHandlers) ? await config.routeHandlers(getConfig()) : config.routeHandlers; // prettier-ignore\n  const routerWrappers = isFunc(config.routerWrappers) ? await config.routerWrappers(getConfig()) : config.routerWrappers; // prettier-ignore\n\n  // Create Crawlee crawler\n  const getActorCtx = () => ({ router, routes, routeHandlers, proxy, config, input, state });\n  const crawler = await config.createCrawler(getActorCtx());\n\n  // Create actor (our custom entity)\n  const preActor = { crawler, ...getActorCtx() };\n  const runCrawler = createScopedCrawlerRun(preActor);\n  const metamorph = createScopedMetamorph(preActor);\n  const scopedPushData = createScopedPushData(preActor);\n\n  const actor = {\n    ...preActor,\n    crawler,\n    runCrawler,\n    metamorph,\n    pushData: scopedPushData,\n  } satisfies ActorContext<Ctx, Labels, Input>;\n\n  // Extra data that we make available to the route handlers\n  const routerContext = { actor, pushData: scopedPushData };\n\n  // Set up router\n  await setupDefaultRoute<Ctx, ActorRouterContext<Ctx, Labels, Input>, Labels>({\n    router,\n    routerWrappers,\n    routerContext,\n    routes,\n    routeHandlers,\n  });\n  await registerHandlers<Ctx, ActorRouterContext<Ctx, Labels, Input>, Labels>({\n    router,\n    routerWrappers,\n    routerContext,\n    routeHandlers,\n  });\n\n  return actor;\n};\n\n/** Create a function that triggers metamorph, using Actor's inputs as defaults. */\nconst createScopedMetamorph = (actor: Pick<ActorContext, 'input'>) => {\n  // Trigger metamorph if it was set from the input\n  const metamorph: Metamorph = async (overrides?: MetamorphActorInput) => {\n    const {\n      metamorphActorId,\n      metamorphActorBuild,\n      metamorphActorInput,\n    } = defaults({}, overrides, actor.input ?? {}); // prettier-ignore\n\n    if (!metamorphActorId) return;\n\n    await Actor.metamorph(metamorphActorId, metamorphActorInput, { build: metamorphActorBuild });\n  };\n\n  return metamorph;\n};\n\n/** pushData wrapper that pre-populates options based on actor input */\nconst createScopedPushData = (actor: Pick<ActorContext, 'input' | 'state'>) => {\n  const scopedPushData: typeof pushData = (entries, ctx, options) => {\n    const {\n      includePersonalData,\n      outputTransform,\n      outputFilter,\n      outputDatasetIdOrName,\n      outputPickFields,\n      outputRenameFields,\n      outputCacheStoreIdOrName,\n      outputCachePrimaryKeys,\n      outputCacheActionOnResult,\n    } = (actor.input ?? {}) as OutputActorInput & PrivacyActorInput;\n\n    const mergedOptions = {\n      showPrivate: includePersonalData,\n      pickKeys: outputPickFields,\n      remapKeys: outputRenameFields,\n      transform: outputTransform ? ((item) => evalInputHook(actor, outputTransform, [item])) : undefined, // prettier-ignore\n      filter: outputFilter ? ((item) => evalInputHook(actor, outputFilter, [item])) : undefined, // prettier-ignore\n      datasetIdOrName: outputDatasetIdOrName,\n      cacheStoreIdOrName: outputCacheStoreIdOrName,\n      cachePrimaryKeys: outputCachePrimaryKeys,\n      cacheActionOnResult: outputCacheActionOnResult,\n      ...options,\n    };\n\n    return pushData(entries, ctx, mergedOptions);\n  };\n\n  return scopedPushData;\n};\n\n/**\n * Create a function that wraps `crawler.run(requests, runOtions)` with additional\n * features like:\n * - Automatically metamorph into another actor after the run finishes\n */\nconst createScopedCrawlerRun = <\n  Ctx extends CrawlingContext<any> = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n>(\n  actor: Omit<ActorContext<Ctx, Labels, Input>, 'runCrawler' | 'metamorph' | 'pushData'>\n) => {\n  const {\n    outputTransformBefore,\n    outputTransformAfter,\n    outputFilterBefore,\n    outputFilterAfter,\n    outputCacheStoreIdOrName,\n    outputCacheActionOnResult,\n  } = (actor.input ?? {}) as OutputActorInput;\n\n  const metamorph = createScopedMetamorph(actor);\n\n  const runCrawler: RunCrawler<Ctx> = async (requests, options) => {\n    // Clear cache if it was set from the input\n    if (outputCacheStoreIdOrName && outputCacheActionOnResult === 'overwrite') {\n      const store = await Actor.openKeyValueStore(outputCacheStoreIdOrName);\n      await store.drop();\n    }\n\n    await evalInputHook(actor, outputTransformBefore);\n    await evalInputHook(actor, outputFilterBefore);\n\n    const runRes = await actor.crawler.run(requests, options);\n\n    await evalInputHook(actor, outputTransformAfter);\n    await evalInputHook(actor, outputFilterAfter);\n\n    // Trigger metamorph if it was set from the input\n    await metamorph();\n\n    return runRes;\n  };\n\n  return runCrawler;\n};\n\nconst actorClassByType = {\n  basic: BasicCrawler,\n  http: HttpCrawler,\n  cheerio: CheerioCrawler,\n  jsdom: JSDOMCrawler,\n  playwright: PlaywrightCrawler,\n  puppeteer: PuppeteerCrawler,\n} satisfies Record<CrawlerType, { new (options: Record<string, any>): any }>;\n\ntype AllInputs = CrawlerConfigActorInput &\n  LoggingActorInput &\n  ProxyActorInput &\n  PrivacyActorInput &\n  OutputActorInput &\n  MetamorphActorInput;\n\n/**\n * Create default configuration for an Apify actor\n * and run the actor within the `Actor.main()` context.\n */\nexport const createAndRunApifyActor = async <\n  TCrawlerType extends CrawlerType,\n  Ctx extends CrawlerMeta<TCrawlerType, any>['context'] = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n>({\n  actorType,\n  actorName,\n  actorConfig,\n  crawlerConfigDefaults,\n  crawlerConfigOverrides,\n  sentryOptions,\n  onActorReady,\n}: {\n  /** String idetifying the actor class, e.g. `'cheerio'` */\n  actorType: TCrawlerType;\n  actorName: string;\n  /** Config passed to the {@link createApifyActor} */\n  actorConfig: Omit<ActorDefinition<Ctx, Labels, Input>, 'router' | 'createCrawler'> &\n    Partial<Pick<ActorDefinition<Ctx, Labels, Input>, 'router' | 'createCrawler'>>;\n  /**\n   * If using default `createCrawler` implementation, these are crawler options\n   * that may be overriden by user input.\n   */\n  crawlerConfigDefaults?: CrawlerMeta<TCrawlerType, any>['options'];\n  /**\n   * If using default `createCrawler` implementation, these are crawler options\n   * that will override user input.\n   *\n   * This is useful for testing env.\n   */\n  crawlerConfigOverrides?: CrawlerMeta<TCrawlerType, any>['options'];\n  /**\n   * Sentry configuration. If using default `createCrawler` implementation,\n   * failed requests are optionally reported to Sentry.\n   *\n   * To disable Sentry, set `\"enabled\": false`.\n   */\n  sentryOptions?: Sentry.NodeOptions;\n  /**\n   * Callback with the created actor. The callback is called within\n   * the `Actor.main()` context.\n   */\n  onActorReady?: (actor: ActorContext<Ctx, Labels, Input>) => MaybePromise<void>;\n}): Promise<void> => {\n  setupSentry({ ...sentryOptions, serverName: actorName });\n\n  // See docs:\n  // - https://docs.apify.com/sdk/js/\n  // - https://docs.apify.com/academy/deploying-your-code/inputs-outputs#accepting-input-with-the-apify-sdk\n  // - https://docs.apify.com/sdk/js/docs/upgrading/upgrading-to-v3#apify-sdk\n  await Actor.main(\n    async () => {\n      const actorDefaults: ActorDefinition<Ctx, Labels, Input & AllInputs> = {\n        router: Router.create<Ctx>(),\n        routerWrappers: ({ input }) => [\n          logLevelHandlerWrapper<Ctx, any>(input?.logLevel ?? 'info'),\n        ],\n        createCrawler: ({ router, proxy, input }) => {\n          const options = createHttpCrawlerOptions<\n            CrawlerMeta<TCrawlerType, any>['options'],\n            Input\n          >({\n            input,\n            defaults: crawlerConfigDefaults,\n            overrides: {\n              requestHandler: router,\n              proxyConfiguration: proxy,\n              // Capture errors as a separate Apify/Actor dataset and pass errors to Sentry\n              failedRequestHandler: createErrorHandler({\n                reportingDatasetId: input?.errorReportingDatasetId ?? 'REPORTING',\n                sendToSentry: input?.errorSendToSentry ?? true,\n              }),\n              ...crawlerConfigOverrides,\n            },\n          });\n          const CrawlerClass = actorClassByType[actorType] as any;\n          return new CrawlerClass(options);\n        },\n        routes: [],\n        routeHandlers: {} as any,\n      };\n\n      const actor = await createApifyActor<Ctx, Labels, Input>({\n        ...actorConfig,\n        router: actorConfig.router ?? (actorDefaults.router as any),\n        routerWrappers: actorConfig.routerWrappers ?? (actorDefaults.routerWrappers as any),\n        createCrawler: actorConfig.createCrawler ?? (actorDefaults.createCrawler as any),\n      });\n\n      await onActorReady?.(actor);\n    },\n    { statusMessage: 'Crawling finished!' }\n  );\n};\n\n/** Given the actor input, create common crawler options. */\nexport const createHttpCrawlerOptions = <\n  TOpts extends BasicCrawlerOptions<any> = BasicCrawlerOptions,\n  Input extends Record<string, any> = Record<string, any>\n>({\n  input,\n  defaults,\n  overrides,\n}: {\n  /** Actor input */\n  input: Input | null;\n  /**\n   * Default config options set by us. These may be overriden\n   * by values from actor input (set by user).\n   */\n  defaults?: TOpts;\n  /**\n   * These config options will overwrite both the default and user\n   * options. This is useful for hard-setting values e.g. in tests.\n   */\n  overrides?: TOpts;\n}) => {\n  const pickCrawlerInputFields = <T extends CrawlerConfigActorInput>(config: T) =>\n    pick(config, Object.keys(crawlerInput));\n\n  return {\n    // ----- 1. DEFAULTS -----\n    ...omitBy(defaults ?? ({} as TOpts), (field) => field === undefined),\n    // ----- 2. CONFIG FROM INPUT -----\n    ...omitBy(pickCrawlerInputFields(input ?? {}), (field) => field === undefined),\n    // ----- 3. OVERRIDES - E.G. TEST CONFIG -----\n    ...omitBy(overrides ?? ({} as TOpts), (field) => field === undefined),\n  } satisfies Partial<TOpts>;\n};\n"]}

package/dist/cjs/lib/actorSpec.d.ts

@@ -0,0 +1,20 @@
+import type { DatasetPerfStat, ScraperActorSpec, ScraperDataset } from 'actor-spec';
+/**
+ * Scraper actor spec with additional dataset perf stats info for formatting in tables
+ *
+ * See {@link ScraperActorSpec}
+ */
+export interface CrawleeOneScraperActorSpec extends ScraperActorSpec {
+    datasets: CrawleeOneScraperDataset[];
+}
+/** Dataset with additional perf stats info for formatting in tables */
+export interface CrawleeOneScraperDataset extends ScraperDataset {
+    perfStats: CrawleeOneDatasetPerfStat[];
+    /** Specify which perfTable should render this data */
+    perfTable: string;
+}
+/** Dataset perf stats with additional info for formatting in tables */
+export interface CrawleeOneDatasetPerfStat extends DatasetPerfStat {
+    rowId: string;
+    colId: string;
+}

package/dist/cjs/lib/actorSpec.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=actorSpec.js.map

package/dist/cjs/lib/actorSpec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actorSpec.js","sourceRoot":"","sources":["../../../src/lib/actorSpec.ts"],"names":[],"mappings":"","sourcesContent":["import type { DatasetPerfStat, ScraperActorSpec, ScraperDataset } from 'actor-spec';\n\n/**\n * Scraper actor spec with additional dataset perf stats info for formatting in tables\n *\n * See {@link ScraperActorSpec}\n */\nexport interface CrawleeOneScraperActorSpec extends ScraperActorSpec {\n  datasets: CrawleeOneScraperDataset[];\n}\n\n/** Dataset with additional perf stats info for formatting in tables */\nexport interface CrawleeOneScraperDataset extends ScraperDataset {\n  perfStats: CrawleeOneDatasetPerfStat[];\n  /** Specify which perfTable should render this data */\n  perfTable: string;\n}\n\n/** Dataset perf stats with additional info for formatting in tables */\nexport interface CrawleeOneDatasetPerfStat extends DatasetPerfStat {\n  rowId: string;\n  colId: string;\n}\n"]}