crawlee-one 1.0.0

package/dist/cjs/lib/actor/actor.js

@@ -0,0 +1,306 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createHttpCrawlerOptions = exports.createCrawleeOne = exports.createAndRunCrawleeOne = void 0;
+const crawlee_1 = require("crawlee");
+const lodash_1 = require("lodash");
+const got_scraping_1 = require("got-scraping");
+const errorHandler_1 = require("../error/errorHandler");
+const sentry_1 = require("../error/sentry");
+const pushData_1 = require("../io/pushData");
+const dataset_1 = require("../io/dataset");
+const pushRequests_1 = require("../io/pushRequests");
+const apify_1 = require("../integrations/apify");
+const router_1 = require("../router");
+const config_1 = require("../config");
+const log_1 = require("../log");
+const actorClassByType = {
+    basic: crawlee_1.BasicCrawler,
+    http: crawlee_1.HttpCrawler,
+    cheerio: crawlee_1.CheerioCrawler,
+    jsdom: crawlee_1.JSDOMCrawler,
+    playwright: crawlee_1.PlaywrightCrawler,
+    puppeteer: crawlee_1.PuppeteerCrawler,
+};
+const isRouter = (r) => {
+    return !!(r.addHandler && r.addDefaultHandler);
+};
+const isFunc = (f) => {
+    return typeof f === 'function';
+};
+/** Run a function that was defined as a string via Actor input */
+const genHookFn = (actor, fnStr) => {
+    if (!fnStr)
+        return null;
+    const hookCtx = {
+        io: actor.io,
+        input: actor.input,
+        state: actor.state,
+        itemCacheKey: pushData_1.itemCacheKey,
+        sendRequest: got_scraping_1.gotScraping,
+    };
+    const hookFn = eval(fnStr);
+    if (!hookFn)
+        return null;
+    return (...args) => __awaiter(void 0, void 0, void 0, function* () { return hookFn(...args, hookCtx); });
+};
+/**
+ * Create default configuration for an opinionated Crawlee actor,
+ * and run the actor within Apify's `Actor.main()` context.
+ *
+ * Apify context can be replaced with custom implementation using the `actorConfig.io` option.
+ *
+ * Read more about what this actor does at {@link createCrawleeOne}.
+ */
+const createAndRunCrawleeOne = (input) => __awaiter(void 0, void 0, void 0, function* () {
+    const { actorType, actorName, actorConfig, crawlerConfigDefaults, crawlerConfigOverrides, sentryOptions, onActorReady, } = input;
+    const { io = apify_1.apifyIO } = actorConfig;
+    yield (0, sentry_1.setupSentry)(Object.assign(Object.assign({}, sentryOptions), { serverName: actorName }), { io });
+    // See docs:
+    // - https://docs.apify.com/sdk/js/
+    // - https://docs.apify.com/academy/deploying-your-code/inputs-outputs#accepting-input-with-the-apify-sdk
+    // - https://docs.apify.com/sdk/js/docs/upgrading/upgrading-to-v3#apify-sdk
+    yield io.runInContext(() => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, _b, _c;
+        const actorDefaults = {
+            io,
+            router: crawlee_1.Router.create(),
+            routerWrappers: ({ input }) => {
+                var _a;
+                return [
+                    (0, log_1.logLevelHandlerWrapper)((_a = input === null || input === void 0 ? void 0 : input.logLevel) !== null && _a !== void 0 ? _a : 'info'),
+                ];
+            },
+            createCrawler: ({ router, proxy, input }) => {
+                var _a, _b;
+                const options = (0, exports.createHttpCrawlerOptions)({
+                    input,
+                    defaults: crawlerConfigDefaults,
+                    overrides: Object.assign({ requestHandler: router, proxyConfiguration: proxy, 
+                        // Capture errors in a separate (Apify) Dataset and pass errors to Sentry
+                        failedRequestHandler: (0, errorHandler_1.createErrorHandler)({
+                            io,
+                            reportingDatasetId: (_a = input === null || input === void 0 ? void 0 : input.errorReportingDatasetId) !== null && _a !== void 0 ? _a : 'REPORTING',
+                            sendToSentry: (_b = input === null || input === void 0 ? void 0 : input.errorSendToSentry) !== null && _b !== void 0 ? _b : true,
+                        }) }, crawlerConfigOverrides),
+                });
+                const CrawlerClass = actorClassByType[actorType];
+                return new CrawlerClass(options);
+            },
+            routes: [],
+            routeHandlers: {},
+        };
+        const actor = yield (0, exports.createCrawleeOne)(Object.assign(Object.assign({}, actorConfig), { io, router: (_a = actorConfig.router) !== null && _a !== void 0 ? _a : actorDefaults.router, routerWrappers: (_b = actorConfig.routerWrappers) !== null && _b !== void 0 ? _b : actorDefaults.routerWrappers, createCrawler: (_c = actorConfig.createCrawler) !== null && _c !== void 0 ? _c : actorDefaults.createCrawler }));
+        yield (onActorReady === null || onActorReady === void 0 ? void 0 : onActorReady(actor));
+    }), { statusMessage: 'Crawling finished!' });
+});
+exports.createAndRunCrawleeOne = createAndRunCrawleeOne;
+/**
+ * Create opinionated Crawlee crawler that uses router for handling requests.
+ *
+ * This is a quality-of-life function that does the following for you:
+ *
+ * 1) Full TypeScript coverage - Ensure all components use the same Crawler / CrawlerContext.
+ *
+ * 2) Get Actor input from `Actor.getInput` if not given.
+ *
+ * 3) (Optional) Validate Actor input
+ *
+ * 4) Set up router such that requests that reach default route are
+ * redirected to labelled routes based on which item from "routes" they match.
+ *
+ * 5) Register all route handlers for you.
+ *
+ * 6) (Optional) Wrap all route handlers in a wrapper. Use this e.g.
+ * if you want to add a field to the context object, or handle errors
+ * from a single place.
+ *
+ * 7) (Optional) Support transformation and filtering of (scraped) entries,
+ * configured via Actor input.
+ *
+ * 8) (Optional) Support Actor metamorphing, configured via Actor input.
+ *
+ * 9) Apify context (e.g. calling `Actor.getInput`) can be replaced with custom
+ *  implementation using the `io` option.
+ */
+const createCrawleeOne = (config) => __awaiter(void 0, void 0, void 0, function* () {
+    const { io = apify_1.apifyIO } = config;
+    // Mutable state that is available to the actor hooks
+    const state = {};
+    // Initialize actor inputs
+    const rawInput = config.input
+        ? isFunc(config.input)
+            ? yield config.input(Object.assign(Object.assign({}, config), { io }))
+            : config.input
+        : yield io.getInput();
+    const input = Object.freeze(yield resolveInput(rawInput, state, { io }));
+    if (config.validateInput)
+        yield config.validateInput(input);
+    // This is context that is available to options that use initialization function
+    const getConfig = () => (Object.assign(Object.assign({}, config), { input, state, io }));
+    // Set up proxy
+    const defaultProxy = config.proxy == null ? yield io.createDefaultProxyConfiguration(input !== null && input !== void 0 ? input : undefined) : undefined;
+    const proxy = config.proxy == null
+        ? defaultProxy
+        : isFunc(config.proxy)
+            ? yield config.proxy(getConfig())
+            : config.proxy;
+    // Run initialization functions
+    const router = isRouter(config.router)
+        ? config.router
+        : yield config.router(getConfig());
+    const routes = isFunc(config.routes) ? yield config.routes(getConfig()) : config.routes; // prettier-ignore
+    const routeHandlers = isFunc(config.routeHandlers) ? yield config.routeHandlers(getConfig()) : config.routeHandlers; // prettier-ignore
+    const routerWrappers = isFunc(config.routerWrappers) ? yield config.routerWrappers(getConfig()) : config.routerWrappers; // prettier-ignore
+    // Create Crawlee crawler
+    const getActorCtx = () => ({
+        io,
+        router,
+        routes,
+        routeHandlers,
+        proxy,
+        config,
+        input,
+        state,
+    });
+    const crawler = yield config.createCrawler(getActorCtx());
+    // Create actor (our custom entity)
+    const preActor = Object.assign({ crawler }, getActorCtx());
+    const runCrawler = createScopedCrawlerRun(preActor);
+    const metamorph = createScopedMetamorph(preActor);
+    const scopedPushData = createScopedPushData(preActor);
+    const scopedPushRequest = createScopedPushRequests(preActor);
+    const startUrls = yield getStartUrlsFromInput(preActor);
+    const actor = Object.assign(Object.assign({}, preActor), { crawler,
+        runCrawler,
+        metamorph, pushData: scopedPushData, pushRequests: scopedPushRequest, startUrls });
+    // Extra data that we make available to the route handlers
+    const routerContext = { actor, pushData: scopedPushData };
+    // Set up router
+    yield (0, router_1.setupDefaultRoute)({
+        io,
+        router,
+        routerWrappers,
+        routerContext,
+        routes,
+        routeHandlers,
+        input,
+    });
+    yield (0, router_1.registerHandlers)({
+        router,
+        routerWrappers,
+        routerContext,
+        routeHandlers,
+    });
+    return actor;
+});
+exports.createCrawleeOne = createCrawleeOne;
+const resolveInput = (input, state, options) => __awaiter(void 0, void 0, void 0, function* () {
+    var _d;
+    const { io = apify_1.apifyIO } = options !== null && options !== void 0 ? options : {};
+    const { inputExtendUrl, inputExtendFromFunction } = (input !== null && input !== void 0 ? input : {});
+    const inputFromUrl = inputExtendUrl ? yield got_scraping_1.gotScraping.get(inputExtendUrl).json() : null;
+    const inputFn = genHookFn({ state, input, io }, inputExtendFromFunction);
+    const inputFromFunc = (_d = (yield (inputFn === null || inputFn === void 0 ? void 0 : inputFn()))) !== null && _d !== void 0 ? _d : null;
+    const extendedInput = Object.assign(Object.assign(Object.assign({}, inputFromUrl), inputFromFunc), input);
+    return extendedInput;
+});
+/**
+ * Create a function that wraps `crawler.run(requests, runOtions)` with additional
+ * features like:
+ * - Automatically metamorph into another actor after the run finishes
+ */
+const createScopedCrawlerRun = (actor) => {
+    var _a;
+    const { requestTransformBefore, requestTransformAfter, requestFilterBefore, requestFilterAfter, outputTransformBefore, outputTransformAfter, outputFilterBefore, outputFilterAfter, outputCacheStoreId, outputCacheActionOnResult, } = ((_a = actor.input) !== null && _a !== void 0 ? _a : {});
+    const metamorph = createScopedMetamorph(actor);
+    const runCrawler = (requests, options) => __awaiter(void 0, void 0, void 0, function* () {
+        var _b, _c, _d, _e, _f, _g, _h, _j;
+        // Clear cache if it was set from the input
+        if (outputCacheStoreId && outputCacheActionOnResult === 'overwrite') {
+            const store = yield actor.io.openKeyValueStore(outputCacheStoreId);
+            yield store.drop();
+        }
+        yield ((_b = genHookFn(actor, outputTransformBefore)) === null || _b === void 0 ? void 0 : _b());
+        yield ((_c = genHookFn(actor, outputFilterBefore)) === null || _c === void 0 ? void 0 : _c());
+        yield ((_d = genHookFn(actor, requestTransformBefore)) === null || _d === void 0 ? void 0 : _d());
+        yield ((_e = genHookFn(actor, requestFilterBefore)) === null || _e === void 0 ? void 0 : _e());
+        const runRes = yield actor.crawler.run(requests, options);
+        yield ((_f = genHookFn(actor, outputTransformAfter)) === null || _f === void 0 ? void 0 : _f());
+        yield ((_g = genHookFn(actor, outputFilterAfter)) === null || _g === void 0 ? void 0 : _g());
+        yield ((_h = genHookFn(actor, requestTransformAfter)) === null || _h === void 0 ? void 0 : _h());
+        yield ((_j = genHookFn(actor, requestFilterAfter)) === null || _j === void 0 ? void 0 : _j());
+        // Trigger metamorph if it was set from the input
+        yield metamorph();
+        return runRes;
+    });
+    return runCrawler;
+};
+/** Create a function that triggers metamorph, using Actor's inputs as defaults. */
+const createScopedMetamorph = (actor) => {
+    // Trigger metamorph if it was set from the input
+    const metamorph = (overrides) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const { metamorphActorId, metamorphActorBuild, metamorphActorInput, } = (0, lodash_1.defaults)({}, overrides, (_a = actor.input) !== null && _a !== void 0 ? _a : {}); // prettier-ignore
+        if (!metamorphActorId)
+            return;
+        yield actor.io.triggerDownstreamCrawler(metamorphActorId, metamorphActorInput, {
+            build: metamorphActorBuild,
+        });
+    });
+    return metamorph;
+};
+/** pushData wrapper that pre-populates options based on actor input */
+const createScopedPushData = (actor) => {
+    var _a;
+    const { includePersonalData, requestQueueId, outputMaxEntries, outputTransform, outputFilter, outputDatasetId, outputPickFields, outputRenameFields, outputCacheStoreId, outputCachePrimaryKeys, outputCacheActionOnResult, } = ((_a = actor.input) !== null && _a !== void 0 ? _a : {});
+    const scopedPushData = (entries, ctx, options) => __awaiter(void 0, void 0, void 0, function* () {
+        const transformFn = genHookFn(actor, outputTransform);
+        const filterFn = genHookFn(actor, outputFilter);
+        const mergedOptions = Object.assign({ io: actor.io, showPrivate: includePersonalData, maxCount: outputMaxEntries, pickKeys: outputPickFields, remapKeys: outputRenameFields, transform: transformFn ? (item) => transformFn(item) : undefined, filter: filterFn ? (item) => filterFn(item) : undefined, datasetId: outputDatasetId, requestQueueId, cacheStoreId: outputCacheStoreId, cachePrimaryKeys: outputCachePrimaryKeys, cacheActionOnResult: outputCacheActionOnResult }, options);
+        return (0, pushData_1.pushData)(entries, ctx, mergedOptions);
+    });
+    return scopedPushData;
+};
+/** pushRequests wrapper that pre-populates options based on actor input */
+const createScopedPushRequests = (actor) => {
+    var _a;
+    const { requestQueueId, requestMaxEntries, requestTransform, requestFilter } = ((_a = actor.input) !== null && _a !== void 0 ? _a : {});
+    const scopedPushRequest = (entries, ctx, options) => __awaiter(void 0, void 0, void 0, function* () {
+        const transformFn = genHookFn(actor, requestTransform);
+        const filterFn = genHookFn(actor, requestFilter);
+        const mergedOptions = Object.assign({ io: actor.io, maxCount: requestMaxEntries, transform: transformFn ? (item) => transformFn(item) : undefined, filter: filterFn ? (item) => filterFn(item) : undefined, requestQueueId }, options);
+        return (0, pushRequests_1.pushRequests)(entries, ctx, mergedOptions);
+    });
+    return scopedPushRequest;
+};
+/** Given the actor input, create common crawler options. */
+const createHttpCrawlerOptions = ({ input, defaults, overrides, }) => {
+    const pickCrawlerInputFields = (config) => (0, lodash_1.pick)(config, Object.keys(config_1.crawlerInput));
+    return Object.assign(Object.assign(Object.assign({}, (0, lodash_1.omitBy)(defaults !== null && defaults !== void 0 ? defaults : {}, (field) => field === undefined)), (0, lodash_1.omitBy)(pickCrawlerInputFields(input !== null && input !== void 0 ? input : {}), (field) => field === undefined)), (0, lodash_1.omitBy)(overrides !== null && overrides !== void 0 ? overrides : {}, (field) => field === undefined));
+};
+exports.createHttpCrawlerOptions = createHttpCrawlerOptions;
+const getStartUrlsFromInput = (actor) => __awaiter(void 0, void 0, void 0, function* () {
+    var _e, _f;
+    const { startUrls, startUrlsFromDataset, startUrlsFromFunction } = ((_e = actor.input) !== null && _e !== void 0 ? _e : {});
+    const urlsAgg = [...(startUrls !== null && startUrls !== void 0 ? startUrls : [])];
+    if (startUrlsFromDataset) {
+        const [datasetId, field] = startUrlsFromDataset.split('#');
+        const urlsFromDataset = yield (0, dataset_1.getColumnFromDataset)(datasetId, field, { io: actor.io });
+        urlsAgg.push(...urlsFromDataset);
+    }
+    if (startUrlsFromFunction) {
+        const urlsFromFn = yield ((_f = genHookFn(actor, startUrlsFromFunction)) === null || _f === void 0 ? void 0 : _f());
+        urlsAgg.push(...urlsFromFn);
+    }
+    return urlsAgg;
+});
+//# sourceMappingURL=actor.js.map

package/dist/cjs/lib/actor/actor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actor.js","sourceRoot":"","sources":["../../../../src/lib/actor/actor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAWiB;AACjB,mCAAgD;AAEhD,+CAA2C;AAI3C,wDAA2D;AAC3D,4CAA8C;AAC9C,6CAA8E;AAC9E,2CAAqD;AACrD,qDAAuE;AAEvE,iDAAgD;AAChD,sCAAgE;AAChE,sCAUmB;AACnB,gCAAgD;AAUhD,MAAM,gBAAgB,GAAG;IACvB,KAAK,EAAE,sBAAY;IACnB,IAAI,EAAE,qBAAW;IACjB,OAAO,EAAE,wBAAc;IACvB,KAAK,EAAE,sBAAY;IACnB,UAAU,EAAE,2BAAiB;IAC7B,SAAS,EAAE,0BAAgB;CAC+C,CAAC;AAE7E,MAAM,QAAQ,GAAG,CAAC,CAAM,EAA2B,EAAE;IACnD,OAAO,CAAC,CAAC,CAAE,CAAmB,CAAC,UAAU,IAAK,CAAmB,CAAC,iBAAiB,CAAC,CAAC;AACvF,CAAC,CAAC;AACF,MAAM,MAAM,GAAG,CAAC,CAAM,EAAgC,EAAE;IACtD,OAAO,OAAO,CAAC,KAAK,UAAU,CAAC;AACjC,CAAC,CAAC;AAEF,kEAAkE;AAClE,MAAM,SAAS,GAAG,CAMhB,KAA4E,EAC5E,KAAc,EACd,EAAE;IACF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,OAAO,GAAG;QACd,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAZ,uBAAY;QACZ,WAAW,EAAE,0BAAW;KACO,CAAC;IAElC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,OAAO,CAAO,GAAG,IAAI,EAAE,EAAE,kDAAC,OAAA,MAAM,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAA,GAAA,CAAC;AACrD,CAAC,CAAC;AAEF;;;;;;;GAOG;AACI,MAAM,sBAAsB,GAAG,CAMpC,KAiCD,EAAiB,EAAE;IAClB,MAAM,EACJ,SAAS,EACT,SAAS,EACT,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,aAAa,EACb,YAAY,GACb,GAAG,KAAK,CAAC;IAEV,MAAM,EAAE,EAAE,GAAG,eAAqB,EAAE,GAAG,WAAW,CAAC;IAEnD,MAAM,IAAA,oBAAW,kCAAM,aAAa,KAAE,UAAU,EAAE,SAAS,KAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAEvE,YAAY;IACZ,mCAAmC;IACnC,yGAAyG;IACzG,2EAA2E;IAC3E,MAAM,EAAE,CAAC,YAAY,CACnB,GAAS,EAAE;;QACT,MAAM,aAAa,GAA8D;YAC/E,EAAE;YACF,MAAM,EAAE,gBAAM,CAAC,MAAM,EAAO;YAC5B,cAAc,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;;gBAAC,OAAA;oBAC7B,IAAA,4BAAsB,EAAW,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,mCAAI,MAAM,CAAC;iBAC5D,CAAA;aAAA;YACD,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;;gBAC1C,MAAM,OAAO,GAAG,IAAA,gCAAwB,EAGtC;oBACA,KAAK;oBACL,QAAQ,EAAE,qBAAqB;oBAC/B,SAAS,kBACP,cAAc,EAAE,MAAM,EACtB,kBAAkB,EAAE,KAAK;wBACzB,yEAAyE;wBACzE,oBAAoB,EAAE,IAAA,iCAAkB,EAAC;4BACvC,EAAE;4BACF,kBAAkB,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,uBAAuB,mCAAI,WAAW;4BACjE,YAAY,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,iBAAiB,mCAAI,IAAI;yBAC/C,CAAC,IACC,sBAAsB,CAC1B;iBACF,CAAC,CAAC;gBACH,MAAM,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAQ,CAAC;gBACxD,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAS;SACzB,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,IAAA,wBAAgB,kCAC/B,WAAW,KACd,EAAE,EACF,MAAM,EAAE,MAAA,WAAW,CAAC,MAAM,mCAAK,aAAa,CAAC,MAAc,EAC3D,cAAc,EAAE,MAAA,WAAW,CAAC,cAAc,mCAAK,aAAa,CAAC,cAAsB,EACnF,aAAa,EAAE,MAAA,WAAW,CAAC,aAAa,mCAAK,aAAa,CAAC,aAAqB,IAChF,CAAC;QAEH,MAAM,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAG,KAAK,CAAC,CAAA,CAAC;IAC9B,CAAC,CAAA,EACD,EAAE,aAAa,EAAE,oBAAoB,EAAE,CACxC,CAAC;AACJ,CAAC,CAAA,CAAC;AAxGW,QAAA,sBAAsB,0BAwGjC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACI,MAAM,gBAAgB,GAAG,CAM9B,MAAmE,EACnB,EAAE;IAClD,MAAM,EAAE,EAAE,GAAG,eAAqB,EAAE,GAAG,MAAM,CAAC;IAE9C,qDAAqD;IACrD,MAAM,KAAK,GAAG,EAAE,CAAC;IAEjB,0BAA0B;IAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;QAC3B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YACpB,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,iCAAM,MAAM,KAAE,EAAE,IAAG;YACvC,CAAC,CAAC,MAAM,CAAC,KAAK;QAChB,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAS,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,CAAe,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAEvF,IAAI,MAAM,CAAC,aAAa;QAAE,MAAM,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAE5D,gFAAgF;IAChF,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,iCAAM,MAAM,KAAE,KAAK,EAAE,KAAK,EAAE,EAAE,IAAG,CAAC;IAE1D,eAAe;IACf,MAAM,YAAY,GAChB,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,+BAA+B,CAAC,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClG,MAAM,KAAK,GACT,MAAM,CAAC,KAAK,IAAI,IAAI;QAClB,CAAC,CAAC,YAAY;QACd,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YACtB,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IAEnB,+BAA+B;IAC/B,MAAM,MAAM,GAAuB,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;QACxD,CAAC,CAAC,MAAM,CAAC,MAAM;QACf,CAAC,CAAC,MAAO,MAAM,CAAC,MAAc,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,kBAAkB;IAC3G,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,kBAAkB;IACvI,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,kBAAkB;IAE3I,yBAAyB;IACzB,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,CAAC;QACzB,EAAE;QACF,MAAM;QACN,MAAM;QACN,aAAa;QACb,KAAK;QACL,MAAM;QACN,KAAK;QACL,KAAK;KACN,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;IAE1D,mCAAmC;IACnC,MAAM,QAAQ,mBAAK,OAAO,IAAK,WAAW,EAAE,CAAE,CAAC;IAC/C,MAAM,UAAU,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACtD,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAExD,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,OAAO;QACP,UAAU;QACV,SAAS,EACT,QAAQ,EAAE,cAAc,EACxB,YAAY,EAAE,iBAAiB,EAC/B,SAAS,GACsC,CAAC;IAElD,0DAA0D;IAC1D,MAAM,aAAa,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;IAE1D,gBAAgB;IAChB,MAAM,IAAA,0BAAiB,EAAkE;QACvF,EAAE;QACF,MAAM;QACN,cAAc;QACd,aAAa;QACb,MAAM;QACN,aAAa;QACb,KAAK;KACN,CAAC,CAAC;IACH,MAAM,IAAA,yBAAgB,EAA2D;QAC/E,MAAM;QACN,cAAc;QACd,aAAa;QACb,aAAa;KACd,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC,CAAA,CAAC;AAhGW,QAAA,gBAAgB,oBAgG3B;AAEF,MAAM,YAAY,GAAG,CACnB,KAAoB,EACpB,KAA8B,EAC9B,OAA+B,EAC/B,EAAE;;IACF,MAAM,EAAE,EAAE,GAAG,eAAuB,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IACvD,MAAM,EAAE,cAAc,EAAE,uBAAuB,EAAE,GAAG,CAAC,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,EAAE,CAAoB,CAAC;IAErF,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,0BAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,IAAI,EAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAClG,MAAM,OAAO,GAAG,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,uBAAuB,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,MAAA,CAAC,MAAM,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,EAAI,CAAA,CAAC,mCAAI,IAAI,CAAC;IAClD,MAAM,aAAa,iDAAQ,YAAY,GAAK,aAAa,GAAK,KAAK,CAAE,CAAC;IAEtE,OAAO,aAAkB,CAAC;AAC5B,CAAC,CAAA,CAAC;AAEF;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,CAM7B,KAGC,EACD,EAAE;;IACF,MAAM,EACJ,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,yBAAyB,GAC1B,GAAG,CAAC,MAAA,KAAK,CAAC,KAAK,mCAAI,EAAE,CAAyC,CAAC;IAEhE,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAE/C,MAAM,UAAU,GAAoB,CAAO,QAAQ,EAAE,OAAO,EAAE,EAAE;;QAC9D,2CAA2C;QAC3C,IAAI,kBAAkB,IAAI,yBAAyB,KAAK,WAAW,EAAE;YACnE,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,qBAAqB,CAAC,2CAAI,CAAA,CAAC;QAClD,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,kBAAkB,CAAC,2CAAI,CAAA,CAAC;QAC/C,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,sBAAsB,CAAC,2CAAI,CAAA,CAAC;QACnD,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,mBAAmB,CAAC,2CAAI,CAAA,CAAC;QAEhD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE1D,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,oBAAoB,CAAC,2CAAI,CAAA,CAAC;QACjD,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,iBAAiB,CAAC,2CAAI,CAAA,CAAC;QAC9C,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,qBAAqB,CAAC,2CAAI,CAAA,CAAC;QAClD,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,kBAAkB,CAAC,2CAAI,CAAA,CAAC;QAE/C,iDAAiD;QACjD,MAAM,SAAS,EAAE,CAAC;QAElB,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,qBAAqB,GAAG,CAAC,KAAyC,EAAE,EAAE;IAC1E,iDAAiD;IACjD,MAAM,SAAS,GAAc,CAAO,SAA+B,EAAE,EAAE;;QACrE,MAAM,EACJ,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,GACpB,GAAG,IAAA,iBAAQ,EAAC,EAAE,EAAE,SAAS,EAAE,MAAA,KAAK,CAAC,KAAK,mCAAI,EAAE,CAAC,CAAC,CAAC,kBAAkB;QAElE,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAE9B,MAAM,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,gBAAgB,EAAE,mBAAmB,EAAE;YAC7E,KAAK,EAAE,mBAAmB;SAC3B,CAAC,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,uEAAuE;AACvE,MAAM,oBAAoB,GAAG,CAAC,KAAmD,EAAE,EAAE;;IACnF,MAAM,EACJ,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,GAC1B,GAAG,CAAC,MAAA,KAAK,CAAC,KAAK,mCAAI,EAAE,CAA6D,CAAC;IAEpF,MAAM,cAAc,GAA6B,CAAO,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;QAC/E,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAEhD,MAAM,aAAa,GAAG,gBACpB,EAAE,EAAE,KAAK,CAAC,EAAE,EACZ,WAAW,EAAE,mBAAmB,EAChC,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAChE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EACvD,SAAS,EAAE,eAAe,EAC1B,cAAc,EACd,YAAY,EAAE,kBAAkB,EAChC,gBAAgB,EAAE,sBAAsB,EACxC,mBAAmB,EAAE,yBAAyB,IAC3C,OAAO,CACuB,CAAC;QAEpC,OAAO,IAAA,mBAAQ,EAAC,OAAO,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC,CAAA,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,wBAAwB,GAAG,CAAC,KAAmD,EAAE,EAAE;;IACvF,MAAM,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,CAAC,MAAA,KAAK,CAAC,KAAK,mCACzF,EAAE,CAAsB,CAAC;IAE3B,MAAM,iBAAiB,GAAiC,CAAO,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;QACtF,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,gBACpB,EAAE,EAAE,KAAK,CAAC,EAAE,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAChE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EACvD,cAAc,IACX,OAAO,CACwB,CAAC;QAErC,OAAO,IAAA,2BAAY,EAAC,OAAO,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IACnD,CAAC,CAAA,CAAC;IAEF,OAAO,iBAAiB,CAAC;AAC3B,CAAC,CAAC;AAEF,4DAA4D;AACrD,MAAM,wBAAwB,GAAG,CAGtC,EACA,KAAK,EACL,QAAQ,EACR,SAAS,GAcV,EAAE,EAAE;IACH,MAAM,sBAAsB,GAAG,CAAoC,MAAS,EAAE,EAAE,CAC9E,IAAA,aAAI,EAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,qBAAY,CAAC,CAAC,CAAC;IAE1C,OAAO,8CAEF,IAAA,eAAM,EAAC,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAK,EAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,GAEjE,IAAA,eAAM,EAAC,sBAAsB,CAAC,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,GAE3E,IAAA,eAAM,EAAC,SAAS,aAAT,SAAS,cAAT,SAAS,GAAK,EAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAC7C,CAAC;AAC7B,CAAC,CAAC;AAhCW,QAAA,wBAAwB,4BAgCnC;AAEF,MAAM,qBAAqB,GAAG,CAAO,KAAmD,EAAE,EAAE;;IAC1F,MAAM,EAAE,SAAS,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,GAAG,CAAC,MAAA,KAAK,CAAC,KAAK,mCAC7E,EAAE,CAAwB,CAAC;IAE7B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,EAAE,CAAC,CAAC,CAAC;IAEvC,IAAI,oBAAoB,EAAE;QACxB,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAA,8BAAoB,EAAM,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5F,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;KAClC;IAED,IAAI,qBAAqB,EAAE;QACzB,MAAM,UAAU,GAAG,MAAM,CAAA,MAAA,SAAS,CAAC,KAAK,EAAE,qBAAqB,CAAC,2CAAI,CAAA,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;KAC7B;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAA,CAAC","sourcesContent":["import {\n  BasicCrawler,\n  CrawlingContext,\n  RouterHandler,\n  BasicCrawlerOptions,\n  CheerioCrawler,\n  Router,\n  HttpCrawler,\n  JSDOMCrawler,\n  PlaywrightCrawler,\n  PuppeteerCrawler,\n} from 'crawlee';\nimport { omitBy, pick, defaults } from 'lodash';\nimport * as Sentry from '@sentry/node';\nimport { gotScraping } from 'got-scraping';\n\nimport type { CrawlerMeta, CrawlerType } from '../../types';\nimport type { MaybePromise, PickPartial } from '../../utils/types';\nimport { createErrorHandler } from '../error/errorHandler';\nimport { setupSentry } from '../error/sentry';\nimport { type PushDataOptions, itemCacheKey, pushData } from '../io/pushData';\nimport { getColumnFromDataset } from '../io/dataset';\nimport { PushRequestsOptions, pushRequests } from '../io/pushRequests';\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\nimport { registerHandlers, setupDefaultRoute } from '../router';\nimport {\n  CrawlerConfigActorInput,\n  OutputActorInput,\n  MetamorphActorInput,\n  PrivacyActorInput,\n  crawlerInput,\n  StartUrlsActorInput,\n  InputActorInput,\n  RequestActorInput,\n  AllActorInputs,\n} from '../config';\nimport { logLevelHandlerWrapper } from '../log';\nimport type {\n  ActorContext,\n  ActorDefinition,\n  ActorHookContext,\n  ActorRouterContext,\n  Metamorph,\n  RunCrawler,\n} from './types';\n\nconst actorClassByType = {\n  basic: BasicCrawler,\n  http: HttpCrawler,\n  cheerio: CheerioCrawler,\n  jsdom: JSDOMCrawler,\n  playwright: PlaywrightCrawler,\n  puppeteer: PuppeteerCrawler,\n} satisfies Record<CrawlerType, { new (options: Record<string, any>): any }>;\n\nconst isRouter = (r: any): r is RouterHandler<any> => {\n  return !!((r as RouterHandler).addHandler && (r as RouterHandler).addDefaultHandler);\n};\nconst isFunc = (f: any): f is (...args: any[]) => any => {\n  return typeof f === 'function';\n};\n\n/** Run a function that was defined as a string via Actor input */\nconst genHookFn = <\n  Ctx extends CrawlingContext<any> = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n>(\n  actor: Pick<ActorContext<Ctx, Labels, Input, TIO>, 'input' | 'state' | 'io'>,\n  fnStr?: string\n) => {\n  if (!fnStr) return null;\n\n  const hookCtx = {\n    io: actor.io,\n    input: actor.input,\n    state: actor.state,\n    itemCacheKey,\n    sendRequest: gotScraping,\n  } satisfies ActorHookContext<TIO>;\n\n  const hookFn = eval(fnStr);\n  if (!hookFn) return null;\n\n  return async (...args) => hookFn(...args, hookCtx);\n};\n\n/**\n * Create default configuration for an opinionated Crawlee actor,\n * and run the actor within Apify's `Actor.main()` context.\n *\n * Apify context can be replaced with custom implementation using the `actorConfig.io` option.\n *\n * Read more about what this actor does at {@link createCrawleeOne}.\n */\nexport const createAndRunCrawleeOne = async <\n  TCrawlerType extends CrawlerType,\n  Ctx extends CrawlerMeta<TCrawlerType, any>['context'] = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n>(input: {\n  /** String idetifying the actor class, e.g. `'cheerio'` */\n  actorType: TCrawlerType;\n  actorName: string;\n  /** Config passed to the {@link createCrawleeOne} */\n  actorConfig: PickPartial<\n    ActorDefinition<Ctx, Labels, Input, TIO>,\n    'router' | 'createCrawler' | 'io'\n  >;\n  /**\n   * If using default `createCrawler` implementation, these are crawler options\n   * that may be overriden by user input.\n   */\n  crawlerConfigDefaults?: CrawlerMeta<TCrawlerType, any>['options'];\n  /**\n   * If using default `createCrawler` implementation, these are crawler options\n   * that will override user input.\n   *\n   * This is useful for testing env.\n   */\n  crawlerConfigOverrides?: CrawlerMeta<TCrawlerType, any>['options'];\n  /**\n   * Sentry configuration. If using default `createCrawler` implementation,\n   * failed requests are optionally reported to Sentry.\n   *\n   * To disable Sentry, set `\"enabled\": false`.\n   */\n  sentryOptions?: Sentry.NodeOptions;\n  /**\n   * Callback with the created actor. The callback is called within\n   * the `Actor.main()` context.\n   */\n  onActorReady?: (actor: ActorContext<Ctx, Labels, Input, TIO>) => MaybePromise<void>;\n}): Promise<void> => {\n  const {\n    actorType,\n    actorName,\n    actorConfig,\n    crawlerConfigDefaults,\n    crawlerConfigOverrides,\n    sentryOptions,\n    onActorReady,\n  } = input;\n\n  const { io = apifyIO as any as TIO } = actorConfig;\n\n  await setupSentry({ ...sentryOptions, serverName: actorName }, { io });\n\n  // See docs:\n  // - https://docs.apify.com/sdk/js/\n  // - https://docs.apify.com/academy/deploying-your-code/inputs-outputs#accepting-input-with-the-apify-sdk\n  // - https://docs.apify.com/sdk/js/docs/upgrading/upgrading-to-v3#apify-sdk\n  await io.runInContext(\n    async () => {\n      const actorDefaults: ActorDefinition<Ctx, Labels, Input & AllActorInputs, TIO> = {\n        io,\n        router: Router.create<Ctx>(),\n        routerWrappers: ({ input }) => [\n          logLevelHandlerWrapper<Ctx, any>(input?.logLevel ?? 'info'),\n        ],\n        createCrawler: ({ router, proxy, input }) => {\n          const options = createHttpCrawlerOptions<\n            CrawlerMeta<TCrawlerType, any>['options'],\n            Input\n          >({\n            input,\n            defaults: crawlerConfigDefaults,\n            overrides: {\n              requestHandler: router,\n              proxyConfiguration: proxy,\n              // Capture errors in a separate (Apify) Dataset and pass errors to Sentry\n              failedRequestHandler: createErrorHandler({\n                io,\n                reportingDatasetId: input?.errorReportingDatasetId ?? 'REPORTING',\n                sendToSentry: input?.errorSendToSentry ?? true,\n              }),\n              ...crawlerConfigOverrides,\n            },\n          });\n          const CrawlerClass = actorClassByType[actorType] as any;\n          return new CrawlerClass(options);\n        },\n        routes: [],\n        routeHandlers: {} as any,\n      };\n\n      const actor = await createCrawleeOne<Ctx, Labels, Input, TIO>({\n        ...actorConfig,\n        io,\n        router: actorConfig.router ?? (actorDefaults.router as any),\n        routerWrappers: actorConfig.routerWrappers ?? (actorDefaults.routerWrappers as any),\n        createCrawler: actorConfig.createCrawler ?? (actorDefaults.createCrawler as any),\n      });\n\n      await onActorReady?.(actor);\n    },\n    { statusMessage: 'Crawling finished!' }\n  );\n};\n\n/**\n * Create opinionated Crawlee crawler that uses router for handling requests.\n *\n * This is a quality-of-life function that does the following for you:\n *\n * 1) Full TypeScript coverage - Ensure all components use the same Crawler / CrawlerContext.\n *\n * 2) Get Actor input from `Actor.getInput` if not given.\n *\n * 3) (Optional) Validate Actor input\n *\n * 4) Set up router such that requests that reach default route are\n * redirected to labelled routes based on which item from \"routes\" they match.\n *\n * 5) Register all route handlers for you.\n *\n * 6) (Optional) Wrap all route handlers in a wrapper. Use this e.g.\n * if you want to add a field to the context object, or handle errors\n * from a single place.\n *\n * 7) (Optional) Support transformation and filtering of (scraped) entries,\n * configured via Actor input.\n *\n * 8) (Optional) Support Actor metamorphing, configured via Actor input.\n *\n * 9) Apify context (e.g. calling `Actor.getInput`) can be replaced with custom\n *  implementation using the `io` option.\n */\nexport const createCrawleeOne = async <\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n>(\n  config: PickPartial<ActorDefinition<Ctx, Labels, Input, TIO>, 'io'>\n): Promise<ActorContext<Ctx, Labels, Input, TIO>> => {\n  const { io = apifyIO as any as TIO } = config;\n\n  // Mutable state that is available to the actor hooks\n  const state = {};\n\n  // Initialize actor inputs\n  const rawInput = config.input\n    ? isFunc(config.input)\n      ? await config.input({ ...config, io })\n      : config.input\n    : await io.getInput<Input>();\n  const input = Object.freeze(await resolveInput<Input | null>(rawInput, state, { io }));\n\n  if (config.validateInput) await config.validateInput(input);\n\n  // This is context that is available to options that use initialization function\n  const getConfig = () => ({ ...config, input, state, io });\n\n  // Set up proxy\n  const defaultProxy =\n    config.proxy == null ? await io.createDefaultProxyConfiguration(input ?? undefined) : undefined;\n  const proxy =\n    config.proxy == null\n      ? defaultProxy\n      : isFunc(config.proxy)\n      ? await config.proxy(getConfig())\n      : config.proxy;\n\n  // Run initialization functions\n  const router: RouterHandler<Ctx> = isRouter(config.router)\n    ? config.router\n    : await (config.router as any)(getConfig());\n  const routes = isFunc(config.routes) ? await config.routes(getConfig()) : config.routes; // prettier-ignore\n  const routeHandlers = isFunc(config.routeHandlers) ? await config.routeHandlers(getConfig()) : config.routeHandlers; // prettier-ignore\n  const routerWrappers = isFunc(config.routerWrappers) ? await config.routerWrappers(getConfig()) : config.routerWrappers; // prettier-ignore\n\n  // Create Crawlee crawler\n  const getActorCtx = () => ({\n    io,\n    router,\n    routes,\n    routeHandlers,\n    proxy,\n    config,\n    input,\n    state,\n  });\n  const crawler = await config.createCrawler(getActorCtx());\n\n  // Create actor (our custom entity)\n  const preActor = { crawler, ...getActorCtx() };\n  const runCrawler = createScopedCrawlerRun(preActor);\n  const metamorph = createScopedMetamorph(preActor);\n  const scopedPushData = createScopedPushData(preActor);\n  const scopedPushRequest = createScopedPushRequests(preActor);\n  const startUrls = await getStartUrlsFromInput(preActor);\n\n  const actor = {\n    ...preActor,\n    crawler,\n    runCrawler,\n    metamorph,\n    pushData: scopedPushData,\n    pushRequests: scopedPushRequest,\n    startUrls,\n  } satisfies ActorContext<Ctx, Labels, Input, TIO>;\n\n  // Extra data that we make available to the route handlers\n  const routerContext = { actor, pushData: scopedPushData };\n\n  // Set up router\n  await setupDefaultRoute<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>, Labels, Input>({\n    io,\n    router,\n    routerWrappers,\n    routerContext,\n    routes,\n    routeHandlers,\n    input,\n  });\n  await registerHandlers<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>, Labels>({\n    router,\n    routerWrappers,\n    routerContext,\n    routeHandlers,\n  });\n\n  return actor;\n};\n\nconst resolveInput = async <T extends Record<string, any> | null>(\n  input: object | null,\n  state: Record<string, unknown>,\n  options?: { io?: CrawleeOneIO }\n) => {\n  const { io = apifyIO as CrawleeOneIO } = options ?? {};\n  const { inputExtendUrl, inputExtendFromFunction } = (input ?? {}) as InputActorInput;\n\n  const inputFromUrl = inputExtendUrl ? await gotScraping.get(inputExtendUrl).json<object>() : null;\n  const inputFn = genHookFn({ state, input, io }, inputExtendFromFunction);\n  const inputFromFunc = (await inputFn?.()) ?? null;\n  const extendedInput = { ...inputFromUrl, ...inputFromFunc, ...input };\n\n  return extendedInput as T;\n};\n\n/**\n * Create a function that wraps `crawler.run(requests, runOtions)` with additional\n * features like:\n * - Automatically metamorph into another actor after the run finishes\n */\nconst createScopedCrawlerRun = <\n  Ctx extends CrawlingContext<any> = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n>(\n  actor: Omit<\n    ActorContext<Ctx, Labels, Input, TIO>,\n    'runCrawler' | 'metamorph' | 'pushData' | 'pushRequests' | 'startUrls'\n  >\n) => {\n  const {\n    requestTransformBefore,\n    requestTransformAfter,\n    requestFilterBefore,\n    requestFilterAfter,\n    outputTransformBefore,\n    outputTransformAfter,\n    outputFilterBefore,\n    outputFilterAfter,\n    outputCacheStoreId,\n    outputCacheActionOnResult,\n  } = (actor.input ?? {}) as OutputActorInput & RequestActorInput;\n\n  const metamorph = createScopedMetamorph(actor);\n\n  const runCrawler: RunCrawler<Ctx> = async (requests, options) => {\n    // Clear cache if it was set from the input\n    if (outputCacheStoreId && outputCacheActionOnResult === 'overwrite') {\n      const store = await actor.io.openKeyValueStore(outputCacheStoreId);\n      await store.drop();\n    }\n\n    await genHookFn(actor, outputTransformBefore)?.();\n    await genHookFn(actor, outputFilterBefore)?.();\n    await genHookFn(actor, requestTransformBefore)?.();\n    await genHookFn(actor, requestFilterBefore)?.();\n\n    const runRes = await actor.crawler.run(requests, options);\n\n    await genHookFn(actor, outputTransformAfter)?.();\n    await genHookFn(actor, outputFilterAfter)?.();\n    await genHookFn(actor, requestTransformAfter)?.();\n    await genHookFn(actor, requestFilterAfter)?.();\n\n    // Trigger metamorph if it was set from the input\n    await metamorph();\n\n    return runRes;\n  };\n\n  return runCrawler;\n};\n\n/** Create a function that triggers metamorph, using Actor's inputs as defaults. */\nconst createScopedMetamorph = (actor: Pick<ActorContext, 'input' | 'io'>) => {\n  // Trigger metamorph if it was set from the input\n  const metamorph: Metamorph = async (overrides?: MetamorphActorInput) => {\n    const {\n      metamorphActorId,\n      metamorphActorBuild,\n      metamorphActorInput,\n    } = defaults({}, overrides, actor.input ?? {}); // prettier-ignore\n\n    if (!metamorphActorId) return;\n\n    await actor.io.triggerDownstreamCrawler(metamorphActorId, metamorphActorInput, {\n      build: metamorphActorBuild,\n    });\n  };\n\n  return metamorph;\n};\n\n/** pushData wrapper that pre-populates options based on actor input */\nconst createScopedPushData = (actor: Pick<ActorContext, 'input' | 'state' | 'io'>) => {\n  const {\n    includePersonalData,\n    requestQueueId,\n    outputMaxEntries,\n    outputTransform,\n    outputFilter,\n    outputDatasetId,\n    outputPickFields,\n    outputRenameFields,\n    outputCacheStoreId,\n    outputCachePrimaryKeys,\n    outputCacheActionOnResult,\n  } = (actor.input ?? {}) as OutputActorInput & PrivacyActorInput & RequestActorInput;\n\n  const scopedPushData: ActorContext['pushData'] = async (entries, ctx, options) => {\n    const transformFn = genHookFn(actor, outputTransform);\n    const filterFn = genHookFn(actor, outputFilter);\n\n    const mergedOptions = {\n      io: actor.io,\n      showPrivate: includePersonalData,\n      maxCount: outputMaxEntries,\n      pickKeys: outputPickFields,\n      remapKeys: outputRenameFields,\n      transform: transformFn ? (item) => transformFn(item) : undefined,\n      filter: filterFn ? (item) => filterFn(item) : undefined,\n      datasetId: outputDatasetId,\n      requestQueueId,\n      cacheStoreId: outputCacheStoreId,\n      cachePrimaryKeys: outputCachePrimaryKeys,\n      cacheActionOnResult: outputCacheActionOnResult,\n      ...options,\n    } satisfies PushDataOptions<object>;\n\n    return pushData(entries, ctx, mergedOptions);\n  };\n\n  return scopedPushData;\n};\n\n/** pushRequests wrapper that pre-populates options based on actor input */\nconst createScopedPushRequests = (actor: Pick<ActorContext, 'input' | 'state' | 'io'>) => {\n  const { requestQueueId, requestMaxEntries, requestTransform, requestFilter } = (actor.input ??\n    {}) as RequestActorInput;\n\n  const scopedPushRequest: ActorContext['pushRequests'] = async (entries, ctx, options) => {\n    const transformFn = genHookFn(actor, requestTransform);\n    const filterFn = genHookFn(actor, requestFilter);\n\n    const mergedOptions = {\n      io: actor.io,\n      maxCount: requestMaxEntries,\n      transform: transformFn ? (item) => transformFn(item) : undefined,\n      filter: filterFn ? (item) => filterFn(item) : undefined,\n      requestQueueId,\n      ...options,\n    } satisfies PushRequestsOptions<any>;\n\n    return pushRequests(entries, ctx, mergedOptions);\n  };\n\n  return scopedPushRequest;\n};\n\n/** Given the actor input, create common crawler options. */\nexport const createHttpCrawlerOptions = <\n  TOpts extends BasicCrawlerOptions<any> = BasicCrawlerOptions,\n  Input extends Record<string, any> = Record<string, any>\n>({\n  input,\n  defaults,\n  overrides,\n}: {\n  /** Actor input */\n  input: Input | null;\n  /**\n   * Default config options set by us. These may be overriden\n   * by values from actor input (set by user).\n   */\n  defaults?: TOpts;\n  /**\n   * These config options will overwrite both the default and user\n   * options. This is useful for hard-setting values e.g. in tests.\n   */\n  overrides?: TOpts;\n}) => {\n  const pickCrawlerInputFields = <T extends CrawlerConfigActorInput>(config: T) =>\n    pick(config, Object.keys(crawlerInput));\n\n  return {\n    // ----- 1. DEFAULTS -----\n    ...omitBy(defaults ?? ({} as TOpts), (field) => field === undefined),\n    // ----- 2. CONFIG FROM INPUT -----\n    ...omitBy(pickCrawlerInputFields(input ?? {}), (field) => field === undefined),\n    // ----- 3. OVERRIDES - E.G. TEST CONFIG -----\n    ...omitBy(overrides ?? ({} as TOpts), (field) => field === undefined),\n  } satisfies Partial<TOpts>;\n};\n\nconst getStartUrlsFromInput = async (actor: Pick<ActorContext, 'input' | 'state' | 'io'>) => {\n  const { startUrls, startUrlsFromDataset, startUrlsFromFunction } = (actor.input ??\n    {}) as StartUrlsActorInput;\n\n  const urlsAgg = [...(startUrls ?? [])];\n\n  if (startUrlsFromDataset) {\n    const [datasetId, field] = startUrlsFromDataset.split('#');\n    const urlsFromDataset = await getColumnFromDataset<any>(datasetId, field, { io: actor.io });\n    urlsAgg.push(...urlsFromDataset);\n  }\n\n  if (startUrlsFromFunction) {\n    const urlsFromFn = await genHookFn(actor, startUrlsFromFunction)?.();\n    urlsAgg.push(...urlsFromFn);\n  }\n\n  return urlsAgg;\n};\n"]}

package/dist/cjs/lib/actor/types.d.ts

@@ -0,0 +1,162 @@
+import type { BasicCrawler, CrawlingContext, ProxyConfiguration, RouterHandler } from 'crawlee';
+import type { gotScraping } from 'got-scraping';
+import type { MaybePromise, PickPartial } from '../../utils/types';
+import type { CrawlerUrl } from '../../types';
+import type { itemCacheKey, pushData } from '../io/pushData';
+import type { pushRequests } from '../io/pushRequests';
+import type { RouteHandler, RouteMatcher, CrawlerRouterWrapper } from '../router';
+import type { MetamorphActorInput } from '../config';
+import type { CrawleeOneIO } from '../integrations/types';
+type MaybeAsyncFn<R, Args extends any[]> = R | ((...args: Args) => MaybePromise<R>);
+type OrigRunCrawler<T extends CrawlingContext<any, any>> = BasicCrawler<T>['run'];
+/** Extended type of `crawler.run()` function */
+export type RunCrawler<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>> = (requests?: CrawlerUrl[], options?: Parameters<OrigRunCrawler<Ctx>>[1]) => ReturnType<OrigRunCrawler<Ctx>>;
+/** Trigger actor metamorph, using actor's inputs as defaults. */
+export type Metamorph = (overrides?: MetamorphActorInput) => Promise<void>;
+/** Context passed to route handlers */
+export type ActorRouterContext<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>, TIO extends CrawleeOneIO = CrawleeOneIO> = {
+    actor: ActorContext<Ctx, Labels, Input, TIO>;
+};
+/** Context passed to user-defined functions passed from input */
+export type ActorHookContext<TIO extends CrawleeOneIO> = Pick<ActorContext, 'input' | 'state'> & {
+    io: TIO;
+    itemCacheKey: typeof itemCacheKey;
+    sendRequest: typeof gotScraping;
+};
+export interface ActorDefinition<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>, TIO extends CrawleeOneIO = CrawleeOneIO> {
+    /** Client for communicating with cloud/local storage. */
+    io: TIO;
+    /**
+     * Actor input which you can get e.g. via `Actor.getInput()`
+     *
+     * Input is automatically retrieved if undefined.
+     */
+    input?: MaybeAsyncFn<Input, [ActorDefinition<Ctx, Labels, Input, TIO>]>;
+    /** Validation for the actor input. Should throw error if validation fails. */
+    validateInput?: (input: Input | null) => MaybePromise<void>;
+    /**
+     * Router instance that redirects the request to handlers.
+     * @example
+     * import { createCheerioRouter } from 'crawlee';
+     *
+     * ({
+     *    ...
+     *   router: createCheerioRouter(),
+     * })
+     */
+    router: MaybeAsyncFn<RouterHandler<Ctx>, [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>;
+    /**
+     * Criteria that un-labelled requests are matched against.
+     *
+     * E.g. If `match` function returns truthy value,
+     * the request is passed to the `action` function for processing.
+     *
+     * @example
+     * ({
+     *   ...
+     *   routes: [{
+     *     // If match returns true, the request is forwarded to handler
+     *     // with label JOB_DETAIL.
+     *     name: 'Job detail',
+     *     handlerLabel: routeLabels.JOB_DETAIL,
+     *     match: (url) => isUrlOfJobOffer(url),
+     *   }, {
+     *     // Define custom action function:
+     *     // If match returns true, we replace this request with new one
+     *     // pointing to new domain.
+     *     name: 'Main page',
+     *     handlerLabel: null,
+     *     match: (url) => url.match(/example\.com\/?(?:[?#~]|$)/i),
+     *     action: async (url, ctx, _, handlers) => {
+     *       ctx.log.info(`Redirecting to https://www.new-domain.com`);
+     *       await ctx.crawler.addRequests(['https://www.new-domain.com'], { forefront: true });
+     *     },
+     *   }],
+     * })
+     */
+    routes: MaybeAsyncFn<RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>, Labels>[], [
+        ActorDefinitionWithInput<Ctx, Labels, Input, TIO>
+    ]>;
+    /** Handlers for the labelled requests. The object keys are the labels. */
+    routeHandlers: MaybeAsyncFn<Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>>>, [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>;
+    /**
+     * Provides the option to modify or extend all router handlers by wrapping
+     * them in these functions.
+     *
+     * Wrappers are applied from right to left. That means that wrappers `[A, B, C]`
+     * will be applied like so `A( B( C( handler ) ) )`.
+     *
+     * Default `routerWrappers`:
+     * ```js
+     * {
+     *   ...
+     *   routerWrappers: ({ input }) => [
+     *     logLevelHandlerWrapper<Ctx, any>(input?.logLevel ?? 'info'),
+     *   ],
+     * }
+     * ```
+     */
+    routerWrappers?: MaybeAsyncFn<CrawlerRouterWrapper<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>>[], [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>;
+    proxy?: MaybeAsyncFn<ProxyConfiguration, [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>;
+    createCrawler: (actorCtx: Omit<ActorContext<Ctx, Labels, Input, TIO>, 'crawler' | 'runCrawler' | 'metamorph' | 'pushData' | 'pushRequests' | 'startUrls'>) => MaybePromise<Ctx['crawler']>;
+}
+/** ActorDefinition object where the input is already resolved */
+export type ActorDefinitionWithInput<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>, TIO extends CrawleeOneIO = CrawleeOneIO> = Omit<ActorDefinition<Ctx, Labels, Input, TIO>, 'input'> & {
+    input: Input | null;
+    state: Record<string, unknown>;
+};
+/** Context available while creating a Crawlee crawler/actor */
+export interface ActorContext<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>, TIO extends CrawleeOneIO = CrawleeOneIO> {
+    crawler: Ctx['crawler'];
+    /**
+     * This function wraps `crawler.run(requests, runOtions)` with additional
+     * features:
+     * - Automatically metamorph into another actor after the run finishes
+     */
+    runCrawler: RunCrawler<Ctx>;
+    /** Trigger actor metamorph, using actor's inputs as defaults. */
+    metamorph: Metamorph;
+    /**
+     * `Actor.pushData` with extra optional features:
+     *
+     * - Limit the number of entries pushed to the Dataset based on the Actor input
+     * - Transform and filter entries via Actor input.
+     * - Add metadata to entries before they are pushed to Dataset.
+     * - Set which (nested) properties are personal data optionally redact them for privacy compliance.
+     */
+    pushData: typeof pushData;
+    /**
+     * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:
+     *
+     * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.
+     * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.
+     */
+    pushRequests: typeof pushRequests;
+    /**
+     * A list of resolved Requests to be scraped.
+     *
+     * This list is a combination of 3 Actor inputs:
+     * - `startUrls` - Static list of URLs to scrape.
+     * - `startUrlsFromDataset` - From a specific field from a Dataset (e.g. "dataset123#fieldName" - Dataset: "dataset123", field: "fieldName").
+     * - `startUrlsFromFunction` - A function that is evaulated to generate the Requests.
+     */
+    startUrls: CrawlerUrl[];
+    proxy?: ProxyConfiguration;
+    router: RouterHandler<Ctx>;
+    routes: RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>, Labels>[];
+    routeHandlers: Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>>>;
+    /** Original config from which this actor context was created */
+    config: PickPartial<ActorDefinition<Ctx, Labels, Input, TIO>, 'io'>;
+    /** Read-only inputs passed to the actor */
+    input: Input | null;
+    /** Mutable state that is shared across setup and teardown hooks */
+    state: Record<string, unknown>;
+    /**
+     * Instance managing communication with databases - storage & retrieval
+     * (Dataset, RequestQueue, KeyValueStore).
+     *
+     * This is modelled and similar to Apify's `Actor` static class.
+     */
+    io: TIO;
+}
+export {};

package/dist/cjs/lib/actor/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/cjs/lib/actor/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/lib/actor/types.ts"],"names":[],"mappings":"","sourcesContent":["import type { BasicCrawler, CrawlingContext, ProxyConfiguration, RouterHandler } from 'crawlee';\nimport type { gotScraping } from 'got-scraping';\n\nimport type { MaybePromise, PickPartial } from '../../utils/types';\nimport type { CrawlerUrl } from '../../types';\nimport type { itemCacheKey, pushData } from '../io/pushData';\nimport type { pushRequests } from '../io/pushRequests';\nimport type { RouteHandler, RouteMatcher, CrawlerRouterWrapper } from '../router';\nimport type { MetamorphActorInput } from '../config';\nimport type { CrawleeOneIO } from '../integrations/types';\n\ntype MaybeAsyncFn<R, Args extends any[]> = R | ((...args: Args) => MaybePromise<R>);\n\ntype OrigRunCrawler<T extends CrawlingContext<any, any>> = BasicCrawler<T>['run'];\n\n/** Extended type of `crawler.run()` function */\nexport type RunCrawler<Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>> = (\n  requests?: CrawlerUrl[],\n  options?: Parameters<OrigRunCrawler<Ctx>>[1]\n) => ReturnType<OrigRunCrawler<Ctx>>;\n\n/** Trigger actor metamorph, using actor's inputs as defaults. */\nexport type Metamorph = (overrides?: MetamorphActorInput) => Promise<void>;\n\n/** Context passed to route handlers */\nexport type ActorRouterContext<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n> = {\n  actor: ActorContext<Ctx, Labels, Input, TIO>;\n};\n\n/** Context passed to user-defined functions passed from input */\nexport type ActorHookContext<TIO extends CrawleeOneIO> = Pick<ActorContext, 'input' | 'state'> & {\n  io: TIO;\n  itemCacheKey: typeof itemCacheKey;\n  sendRequest: typeof gotScraping;\n};\n\nexport interface ActorDefinition<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n> {\n  /** Client for communicating with cloud/local storage. */\n  io: TIO;\n\n  // Actor input\n  /**\n   * Actor input which you can get e.g. via `Actor.getInput()`\n   *\n   * Input is automatically retrieved if undefined.\n   */\n  input?: MaybeAsyncFn<Input, [ActorDefinition<Ctx, Labels, Input, TIO>]>;\n  /** Validation for the actor input. Should throw error if validation fails. */\n  validateInput?: (input: Input | null) => MaybePromise<void>;\n\n  // Router setup\n  /**\n   * Router instance that redirects the request to handlers.\n   * @example\n   * import { createCheerioRouter } from 'crawlee';\n   *\n   * ({\n   *    ...\n   *   router: createCheerioRouter(),\n   * })\n   */\n  router: MaybeAsyncFn<RouterHandler<Ctx>, [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>;\n  /**\n   * Criteria that un-labelled requests are matched against.\n   *\n   * E.g. If `match` function returns truthy value,\n   * the request is passed to the `action` function for processing.\n   *\n   * @example\n   * ({\n   *   ...\n   *   routes: [{\n   *     // If match returns true, the request is forwarded to handler\n   *     // with label JOB_DETAIL.\n   *     name: 'Job detail',\n   *     handlerLabel: routeLabels.JOB_DETAIL,\n   *     match: (url) => isUrlOfJobOffer(url),\n   *   }, {\n   *     // Define custom action function:\n   *     // If match returns true, we replace this request with new one\n   *     // pointing to new domain.\n   *     name: 'Main page',\n   *     handlerLabel: null,\n   *     match: (url) => url.match(/example\\.com\\/?(?:[?#~]|$)/i),\n   *     action: async (url, ctx, _, handlers) => {\n   *       ctx.log.info(`Redirecting to https://www.new-domain.com`);\n   *       await ctx.crawler.addRequests(['https://www.new-domain.com'], { forefront: true });\n   *     },\n   *   }],\n   * })\n   */\n  routes: MaybeAsyncFn<\n    RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>, Labels>[],\n    [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]\n  >;\n  /** Handlers for the labelled requests. The object keys are the labels. */\n  routeHandlers: MaybeAsyncFn<Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>>>, [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>; // prettier-ignore\n  /**\n   * Provides the option to modify or extend all router handlers by wrapping\n   * them in these functions.\n   *\n   * Wrappers are applied from right to left. That means that wrappers `[A, B, C]`\n   * will be applied like so `A( B( C( handler ) ) )`.\n   *\n   * Default `routerWrappers`:\n   * ```js\n   * {\n   *   ...\n   *   routerWrappers: ({ input }) => [\n   *     logLevelHandlerWrapper<Ctx, any>(input?.logLevel ?? 'info'),\n   *   ],\n   * }\n   * ```\n   */\n  routerWrappers?: MaybeAsyncFn<CrawlerRouterWrapper<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>>[], [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>; // prettier-ignore\n\n  // Proxy setup\n  proxy?: MaybeAsyncFn<ProxyConfiguration, [ActorDefinitionWithInput<Ctx, Labels, Input, TIO>]>; // prettier-ignore\n\n  // Crawler setup\n  createCrawler: (\n    actorCtx: Omit<\n      ActorContext<Ctx, Labels, Input, TIO>,\n      'crawler' | 'runCrawler' | 'metamorph' | 'pushData' | 'pushRequests' | 'startUrls'\n    >\n  ) => MaybePromise<Ctx['crawler']>;\n}\n\n/** ActorDefinition object where the input is already resolved */\nexport type ActorDefinitionWithInput<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n> = Omit<ActorDefinition<Ctx, Labels, Input, TIO>, 'input'> & {\n  input: Input | null;\n  state: Record<string, unknown>;\n};\n\n/** Context available while creating a Crawlee crawler/actor */\nexport interface ActorContext<\n  Ctx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>,\n  TIO extends CrawleeOneIO = CrawleeOneIO\n> {\n  crawler: Ctx['crawler'];\n  /**\n   * This function wraps `crawler.run(requests, runOtions)` with additional\n   * features:\n   * - Automatically metamorph into another actor after the run finishes\n   */\n  runCrawler: RunCrawler<Ctx>;\n  /** Trigger actor metamorph, using actor's inputs as defaults. */\n  metamorph: Metamorph;\n  /**\n   * `Actor.pushData` with extra optional features:\n   *\n   * - Limit the number of entries pushed to the Dataset based on the Actor input\n   * - Transform and filter entries via Actor input.\n   * - Add metadata to entries before they are pushed to Dataset.\n   * - Set which (nested) properties are personal data optionally redact them for privacy compliance.\n   */\n  pushData: typeof pushData;\n  /**\n   * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:\n   *\n   * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.\n   * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.\n   */\n  pushRequests: typeof pushRequests;\n  /**\n   * A list of resolved Requests to be scraped.\n   *\n   * This list is a combination of 3 Actor inputs:\n   * - `startUrls` - Static list of URLs to scrape.\n   * - `startUrlsFromDataset` - From a specific field from a Dataset (e.g. \"dataset123#fieldName\" - Dataset: \"dataset123\", field: \"fieldName\").\n   * - `startUrlsFromFunction` - A function that is evaulated to generate the Requests.\n   */\n  startUrls: CrawlerUrl[];\n  proxy?: ProxyConfiguration;\n  router: RouterHandler<Ctx>;\n  routes: RouteMatcher<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>, Labels>[];\n  routeHandlers: Record<Labels, RouteHandler<Ctx, ActorRouterContext<Ctx, Labels, Input, TIO>>>;\n  /** Original config from which this actor context was created */\n  config: PickPartial<ActorDefinition<Ctx, Labels, Input, TIO>, 'io'>;\n  /** Read-only inputs passed to the actor */\n  input: Input | null;\n  /** Mutable state that is shared across setup and teardown hooks */\n  state: Record<string, unknown>;\n  /**\n   * Instance managing communication with databases - storage & retrieval\n   * (Dataset, RequestQueue, KeyValueStore).\n   *\n   * This is modelled and similar to Apify's `Actor` static class.\n   */\n  io: TIO;\n}\n"]}