crawlee-one 1.0.0

package/dist/cjs/lib/error/errorHandler.d.ts

@@ -0,0 +1,112 @@
+import type { BasicCrawlingContext, CheerioCrawlingContext, CrawlingContext, ErrorHandler, HttpCrawlingContext, JSDOMCrawlingContext, PlaywrightCrawlingContext, PuppeteerCrawlingContext } from 'crawlee';
+import type { MaybePromise, PickRequired } from '../../utils/types';
+import type { RouteHandler, RouterHandlerCtx } from '../router';
+import type { CrawleeOneErrorHandlerInput, CrawleeOneErrorHandlerOptions } from '../integrations/types';
+export type CaptureErrorInput = PickRequired<Partial<CrawleeOneErrorHandlerInput>, 'error'>;
+export type CaptureError = (input: CaptureErrorInput) => MaybePromise<void>;
+/**
+ * Error handling for CrawleeOne crawlers.
+ *
+ * By default, error reports are saved to Apify Dataset.
+ *
+ * See https://docs.apify.com/academy/node-js/analyzing-pages-and-fixing-errors#error-reporting
+ */
+export declare const captureError: <TEnv extends object = object, TReport extends object = object>(input: CaptureErrorInput, options: CrawleeOneErrorHandlerOptions<TEnv, TReport>) => Promise<never>;
+/**
+ * Error handling for Crawlers as a function wrapper
+ *
+ * By default, error reports are saved to Apify Dataset.
+ */
+export declare const captureErrorWrapper: <TEnv extends object = object, TReport extends object = object>(fn: (input: {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<TEnv, TReport>) => Promise<void>;
+/**
+ * Drop-in replacement for regular request handler callback for Crawlee route
+ * that automatically tracks errors.
+ *
+ * By default, error reports are saved to Apify Dataset.
+ *
+ * @example
+ *
+ * router.addDefaultHandler(
+ *  captureErrorRouteHandler(async (ctx) => {
+ *    const { page, crawler } = ctx;
+ *    const url = page.url();
+ *    ...
+ *  })
+ * );
+ */
+export declare const captureErrorRouteHandler: <Ctx extends CrawlingContext<unknown, import("crawlee").Dictionary>, TEnv extends object = object, TReport extends object = object>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<TEnv, TReport>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+export declare const basicCaptureErrorRouteHandler: <Ctx extends BasicCrawlingContext<import("crawlee").Dictionary>>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<object, object>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+export declare const httpCaptureErrorRouteHandler: <Ctx extends HttpCrawlingContext<any, any>>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<object, object>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+export declare const jsdomCaptureErrorRouteHandler: <Ctx extends JSDOMCrawlingContext<any, any>>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<object, object>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+export declare const playwrightCaptureErrorRouteHandler: <Ctx extends PlaywrightCrawlingContext<import("crawlee").Dictionary>>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<object, object>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+export declare const cheerioCaptureErrorRouteHandler: <Ctx extends CheerioCrawlingContext<any, any>>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<object, object>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+export declare const puppeteerCaptureErrorRouteHandler: <Ctx extends PuppeteerCrawlingContext<import("crawlee").Dictionary>>(handler: (ctx: Omit<Ctx, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+} & {
+    captureError: CaptureError;
+}) => MaybePromise<void>, options: CrawleeOneErrorHandlerOptions<object, object>) => (ctx: Omit<Omit<Ctx & Record<string, any>, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}, "request"> & {
+    request: import("crawlee").Request<import("crawlee").Dictionary>;
+}) => Promise<void>;
+/**
+ * Create an `ErrorHandler` function that can be assigned to
+ * `failedRequestHandler` option of `BasicCrawlerOptions`.
+ *
+ * The function saves error to a Dataset, and optionally forwards it to Sentry.
+ *
+ * By default, error reports are saved to Apify Dataset.
+ */
+export declare const createErrorHandler: <Ctx extends CrawlingContext<unknown, import("crawlee").Dictionary>>(options: CrawleeOneErrorHandlerOptions & {
+    sendToSentry?: boolean;
+}) => ErrorHandler<Ctx>;

package/dist/cjs/lib/error/errorHandler.js

@@ -0,0 +1,164 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createErrorHandler = exports.puppeteerCaptureErrorRouteHandler = exports.cheerioCaptureErrorRouteHandler = exports.playwrightCaptureErrorRouteHandler = exports.jsdomCaptureErrorRouteHandler = exports.httpCaptureErrorRouteHandler = exports.basicCaptureErrorRouteHandler = exports.captureErrorRouteHandler = exports.captureErrorWrapper = exports.captureError = void 0;
+const Sentry = __importStar(require("@sentry/node"));
+const apify_1 = require("../integrations/apify");
+/**
+ * Error handling for CrawleeOne crawlers.
+ *
+ * By default, error reports are saved to Apify Dataset.
+ *
+ * See https://docs.apify.com/academy/node-js/analyzing-pages-and-fixing-errors#error-reporting
+ */
+const captureError = (input, options) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a, _b, _c;
+    const { error, log: parentLog } = input;
+    const { io = apify_1.apifyIO, reportingDatasetId, onErrorCapture, } = options;
+    const log = (_a = parentLog === null || parentLog === void 0 ? void 0 : parentLog.child({ prefix: '[Error capture] ' })) !== null && _a !== void 0 ? _a : null;
+    log === null || log === void 0 ? void 0 : log.error(`ERROR ${error.name}: ${error.message}`, error);
+    console.error(`ERROR ${error.name}: ${error.message}`, error);
+    // Let's create reporting dataset
+    // If you already have one, this will continue adding to it
+    const reportingDataset = reportingDatasetId ? yield io.openDataset(reportingDatasetId) : null;
+    const report = yield io.generateErrorReport({ error, page: (_b = input.page) !== null && _b !== void 0 ? _b : null, url: (_c = input.url) !== null && _c !== void 0 ? _c : null, log }, Object.assign(Object.assign({}, options), { io }));
+    log === null || log === void 0 ? void 0 : log.error('[Error capture] Error captured', report);
+    // And we push the report
+    if (reportingDatasetId) {
+        log === null || log === void 0 ? void 0 : log.info(`[Error capture] Pushing error to dataset ${reportingDatasetId}`);
+        yield (reportingDataset === null || reportingDataset === void 0 ? void 0 : reportingDataset.pushData(report));
+        log === null || log === void 0 ? void 0 : log.info(`[Error capture] DONE pushing error to dataset ${reportingDatasetId}`);
+    }
+    log === null || log === void 0 ? void 0 : log.error('[Error capture] Calling onErrorCapture');
+    yield (onErrorCapture === null || onErrorCapture === void 0 ? void 0 : onErrorCapture({ error, report }));
+    log === null || log === void 0 ? void 0 : log.error('[Error capture] Done calling onErrorCapture');
+    // @ts-expect-error Tag the error, so we don't capture it twice.
+    error._crawleeOneErrorCaptured = true;
+    // Propagate the error
+    throw error;
+});
+exports.captureError = captureError;
+/**
+ * Error handling for Crawlers as a function wrapper
+ *
+ * By default, error reports are saved to Apify Dataset.
+ */
+const captureErrorWrapper = (fn, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const captureErrorWithArgs = (input) => (0, exports.captureError)(input, options);
+    try {
+        // Pass the error capturing function to the wrapped function, so it can trigger it by itself
+        yield fn({ captureError: captureErrorWithArgs });
+    }
+    catch (error) {
+        if (!error._crawleeOneErrorCaptured) {
+            // And if the wrapped function fails, we capture error for them
+            yield captureErrorWithArgs({ error, url: null, page: null, log: null });
+        }
+    }
+});
+exports.captureErrorWrapper = captureErrorWrapper;
+/**
+ * Drop-in replacement for regular request handler callback for Crawlee route
+ * that automatically tracks errors.
+ *
+ * By default, error reports are saved to Apify Dataset.
+ *
+ * @example
+ *
+ * router.addDefaultHandler(
+ *  captureErrorRouteHandler(async (ctx) => {
+ *    const { page, crawler } = ctx;
+ *    const url = page.url();
+ *    ...
+ *  })
+ * );
+ */
+const captureErrorRouteHandler = (handler, options) => {
+    // Wrap the original handler, so we can additionally pass it the captureError function
+    const wrapperHandler = (ctx) => {
+        return (0, exports.captureErrorWrapper)(({ captureError }) => {
+            return handler(Object.assign(Object.assign({}, ctx), { 
+                // And automatically feed contextual args (page, url, log) to captureError
+                captureError: (input) => {
+                    var _a, _b;
+                    return captureError({
+                        error: input.error,
+                        page: (_a = input.page) !== null && _a !== void 0 ? _a : ctx.page,
+                        url: input.url || ctx.request.url,
+                        log: (_b = input.log) !== null && _b !== void 0 ? _b : ctx.log,
+                    });
+                } }));
+        }, options);
+    };
+    return wrapperHandler;
+};
+exports.captureErrorRouteHandler = captureErrorRouteHandler;
+const basicCaptureErrorRouteHandler = (...args) => (0, exports.captureErrorRouteHandler)(...args); // prettier-ignore
+exports.basicCaptureErrorRouteHandler = basicCaptureErrorRouteHandler;
+const httpCaptureErrorRouteHandler = (...args) => (0, exports.captureErrorRouteHandler)(...args); // prettier-ignore
+exports.httpCaptureErrorRouteHandler = httpCaptureErrorRouteHandler;
+const jsdomCaptureErrorRouteHandler = (...args) => (0, exports.captureErrorRouteHandler)(...args); // prettier-ignore
+exports.jsdomCaptureErrorRouteHandler = jsdomCaptureErrorRouteHandler;
+const playwrightCaptureErrorRouteHandler = (...args) => (0, exports.captureErrorRouteHandler)(...args); // prettier-ignore
+exports.playwrightCaptureErrorRouteHandler = playwrightCaptureErrorRouteHandler;
+const cheerioCaptureErrorRouteHandler = (...args) => (0, exports.captureErrorRouteHandler)(...args); // prettier-ignore
+exports.cheerioCaptureErrorRouteHandler = cheerioCaptureErrorRouteHandler;
+const puppeteerCaptureErrorRouteHandler = (...args) => (0, exports.captureErrorRouteHandler)(...args); // prettier-ignore
+exports.puppeteerCaptureErrorRouteHandler = puppeteerCaptureErrorRouteHandler;
+/**
+ * Create an `ErrorHandler` function that can be assigned to
+ * `failedRequestHandler` option of `BasicCrawlerOptions`.
+ *
+ * The function saves error to a Dataset, and optionally forwards it to Sentry.
+ *
+ * By default, error reports are saved to Apify Dataset.
+ */
+const createErrorHandler = (options) => {
+    return ({ request, log, page }, error) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const url = request.loadedUrl || request.url;
+        (0, exports.captureError)({ error, url, log, page: page }, {
+            io: options.io,
+            reportingDatasetId: options.reportingDatasetId,
+            allowScreenshot: (_a = options.allowScreenshot) !== null && _a !== void 0 ? _a : true,
+            onErrorCapture: ({ error, report }) => {
+                if (!options.sendToSentry)
+                    return;
+                Sentry.captureException(error, { extra: report });
+            },
+        });
+    });
+};
+exports.createErrorHandler = createErrorHandler;
+//# sourceMappingURL=errorHandler.js.map

package/dist/cjs/lib/error/errorHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errorHandler.js","sourceRoot":"","sources":["../../../../src/lib/error/errorHandler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAUA,qDAAuC;AAUvC,iDAAgD;AAKhD;;;;;;GAMG;AACI,MAAM,YAAY,GAAG,CAC1B,KAAwB,EACxB,OAAqD,EACrD,EAAE;;IACF,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;IACxC,MAAM,EACJ,EAAE,GAAG,eAA6C,EAClD,kBAAkB,EAClB,cAAc,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,GAAG,GAAG,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,mCAAI,IAAI,CAAC;IAErE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;IAC3D,OAAO,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;IAE9D,iCAAiC;IACjC,2DAA2D;IAC3D,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,mBAAmB,CACzC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAA,KAAK,CAAC,IAAI,mCAAI,IAAI,EAAE,GAAG,EAAE,MAAA,KAAK,CAAC,GAAG,mCAAI,IAAI,EAAE,GAAG,EAAE,kCAC3D,OAAO,KAAE,EAAE,IACjB,CAAC;IAEF,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;IAErD,yBAAyB;IACzB,IAAI,kBAAkB,EAAE;QACtB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC,4CAA4C,kBAAkB,EAAE,CAAC,CAAC;QAC5E,MAAM,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,QAAQ,CAAC,MAAM,CAAC,CAAA,CAAC;QACzC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC,iDAAiD,kBAAkB,EAAE,CAAC,CAAC;KAClF;IAED,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACrD,MAAM,CAAA,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA,CAAC;IAC1C,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAE1D,gEAAgE;IAChE,KAAK,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACtC,sBAAsB;IACtB,MAAM,KAAK,CAAC;AACd,CAAC,CAAA,CAAC;AAzCW,QAAA,YAAY,gBAyCvB;AAEF;;;;GAIG;AACI,MAAM,mBAAmB,GAAG,CAIjC,EAAiE,EACjE,OAAqD,EACrD,EAAE;IACF,MAAM,oBAAoB,GAAiB,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,oBAAY,EAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAEnF,IAAI;QACF,4FAA4F;QAC5F,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,oBAAoB,EAAE,CAAC,CAAC;KAClD;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE;YACnC,+DAA+D;YAC/D,MAAM,oBAAoB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SACzE;KACF;AACH,CAAC,CAAA,CAAC;AAlBW,QAAA,mBAAmB,uBAkB9B;AAEF;;;;;;;;;;;;;;;GAeG;AACI,MAAM,wBAAwB,GAAG,CAKtC,OAA4F,EAC5F,OAAqD,EACrD,EAAE;IACF,sFAAsF;IACtF,MAAM,cAAc,GAAG,CAAC,GAAqC,EAAE,EAAE;QAC/D,OAAO,IAAA,2BAAmB,EAAC,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE;YAC9C,OAAO,OAAO,iCACR,GAAW;gBACf,0EAA0E;gBAC1E,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;;oBACtB,OAAA,YAAY,CAAC;wBACX,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,IAAI,EAAE,MAAA,KAAK,CAAC,IAAI,mCAAI,GAAG,CAAC,IAAI;wBAC5B,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG;wBACjC,GAAG,EAAE,MAAA,KAAK,CAAC,GAAG,mCAAI,GAAG,CAAC,GAAG;qBAC1B,CAAC,CAAA;iBAAA,IACJ,CAAC;QACL,CAAC,EAAE,OAAO,CAAC,CAAC;IACd,CAAC,CAAC;IACF,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAzBW,QAAA,wBAAwB,4BAyBnC;AAEK,MAAM,6BAA6B,GAAG,CAAmC,GAAG,IAAsD,EAAE,EAAE,CAAC,IAAA,gCAAwB,EAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB;AAA3L,QAAA,6BAA6B,iCAA2I;AAC9K,MAAM,4BAA4B,GAAG,CAAkC,GAAG,IAAsD,EAAE,EAAE,CAAC,IAAA,gCAAwB,EAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB;AAAzL,QAAA,4BAA4B,gCAA0I;AAC5K,MAAM,6BAA6B,GAAG,CAAmC,GAAG,IAAsD,EAAE,EAAE,CAAC,IAAA,gCAAwB,EAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB;AAA3L,QAAA,6BAA6B,iCAA2I;AAC9K,MAAM,kCAAkC,GAAG,CAAwC,GAAG,IAAsD,EAAE,EAAE,CAAC,IAAA,gCAAwB,EAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB;AAArM,QAAA,kCAAkC,sCAAgJ;AACxL,MAAM,+BAA+B,GAAG,CAAqC,GAAG,IAAsD,EAAE,EAAE,CAAC,IAAA,gCAAwB,EAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB;AAA/L,QAAA,+BAA+B,mCAA6I;AAClL,MAAM,iCAAiC,GAAG,CAAuC,GAAG,IAAsD,EAAE,EAAE,CAAC,IAAA,gCAAwB,EAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB;AAAnM,QAAA,iCAAiC,qCAA+I;AAE7L;;;;;;;GAOG;AACI,MAAM,kBAAkB,GAAG,CAChC,OAAmE,EAChD,EAAE;IACrB,OAAO,CAAO,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE;;QAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC;QAC7C,IAAA,oBAAY,EACV,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAY,EAAE,EACvC;YACE,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;YAC9C,eAAe,EAAE,MAAA,OAAO,CAAC,eAAe,mCAAI,IAAI;YAChD,cAAc,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;gBACpC,IAAI,CAAC,OAAO,CAAC,YAAY;oBAAE,OAAO;gBAElC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,MAAa,EAAE,CAAC,CAAC;YAC3D,CAAC;SACF,CACF,CAAC;IACJ,CAAC,CAAA,CAAC;AACJ,CAAC,CAAC;AAnBW,QAAA,kBAAkB,sBAmB7B","sourcesContent":["import type {\n  BasicCrawlingContext,\n  CheerioCrawlingContext,\n  CrawlingContext,\n  ErrorHandler,\n  HttpCrawlingContext,\n  JSDOMCrawlingContext,\n  PlaywrightCrawlingContext,\n  PuppeteerCrawlingContext,\n} from 'crawlee';\nimport * as Sentry from '@sentry/node';\nimport type { Page } from 'playwright';\n\nimport type { MaybePromise, PickRequired } from '../../utils/types';\nimport type { RouteHandler, RouterHandlerCtx } from '../router';\nimport type {\n  CrawleeOneErrorHandlerInput,\n  CrawleeOneErrorHandlerOptions,\n  CrawleeOneIO,\n} from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\n\nexport type CaptureErrorInput = PickRequired<Partial<CrawleeOneErrorHandlerInput>, 'error'>;\nexport type CaptureError = (input: CaptureErrorInput) => MaybePromise<void>;\n\n/**\n * Error handling for CrawleeOne crawlers.\n *\n * By default, error reports are saved to Apify Dataset.\n *\n * See https://docs.apify.com/academy/node-js/analyzing-pages-and-fixing-errors#error-reporting\n */\nexport const captureError = async <TEnv extends object = object, TReport extends object = object>(\n  input: CaptureErrorInput,\n  options: CrawleeOneErrorHandlerOptions<TEnv, TReport>\n) => {\n  const { error, log: parentLog } = input;\n  const {\n    io = apifyIO as any as CrawleeOneIO<TEnv, TReport>,\n    reportingDatasetId,\n    onErrorCapture,\n  } = options;\n\n  const log = parentLog?.child({ prefix: '[Error capture] ' }) ?? null;\n\n  log?.error(`ERROR ${error.name}: ${error.message}`, error);\n  console.error(`ERROR ${error.name}: ${error.message}`, error);\n\n  // Let's create reporting dataset\n  // If you already have one, this will continue adding to it\n  const reportingDataset = reportingDatasetId ? await io.openDataset(reportingDatasetId) : null;\n  const report = await io.generateErrorReport(\n    { error, page: input.page ?? null, url: input.url ?? null, log },\n    { ...options, io }\n  );\n\n  log?.error('[Error capture] Error captured', report);\n\n  // And we push the report\n  if (reportingDatasetId) {\n    log?.info(`[Error capture] Pushing error to dataset ${reportingDatasetId}`);\n    await reportingDataset?.pushData(report);\n    log?.info(`[Error capture] DONE pushing error to dataset ${reportingDatasetId}`);\n  }\n\n  log?.error('[Error capture] Calling onErrorCapture');\n  await onErrorCapture?.({ error, report });\n  log?.error('[Error capture] Done calling onErrorCapture');\n\n  // @ts-expect-error Tag the error, so we don't capture it twice.\n  error._crawleeOneErrorCaptured = true;\n  // Propagate the error\n  throw error;\n};\n\n/**\n * Error handling for Crawlers as a function wrapper\n *\n * By default, error reports are saved to Apify Dataset.\n */\nexport const captureErrorWrapper = async <\n  TEnv extends object = object,\n  TReport extends object = object\n>(\n  fn: (input: { captureError: CaptureError }) => MaybePromise<void>,\n  options: CrawleeOneErrorHandlerOptions<TEnv, TReport>\n) => {\n  const captureErrorWithArgs: CaptureError = (input) => captureError(input, options);\n\n  try {\n    // Pass the error capturing function to the wrapped function, so it can trigger it by itself\n    await fn({ captureError: captureErrorWithArgs });\n  } catch (error: any) {\n    if (!error._crawleeOneErrorCaptured) {\n      // And if the wrapped function fails, we capture error for them\n      await captureErrorWithArgs({ error, url: null, page: null, log: null });\n    }\n  }\n};\n\n/**\n * Drop-in replacement for regular request handler callback for Crawlee route\n * that automatically tracks errors.\n *\n * By default, error reports are saved to Apify Dataset.\n *\n * @example\n *\n * router.addDefaultHandler(\n *  captureErrorRouteHandler(async (ctx) => {\n *    const { page, crawler } = ctx;\n *    const url = page.url();\n *    ...\n *  })\n * );\n */\nexport const captureErrorRouteHandler = <\n  Ctx extends CrawlingContext,\n  TEnv extends object = object,\n  TReport extends object = object\n>(\n  handler: (ctx: RouterHandlerCtx<Ctx> & { captureError: CaptureError }) => MaybePromise<void>,\n  options: CrawleeOneErrorHandlerOptions<TEnv, TReport>\n) => {\n  // Wrap the original handler, so we can additionally pass it the captureError function\n  const wrapperHandler = (ctx: Parameters<RouteHandler<Ctx>>[0]) => {\n    return captureErrorWrapper(({ captureError }) => {\n      return handler({\n        ...(ctx as any),\n        // And automatically feed contextual args (page, url, log) to captureError\n        captureError: (input) =>\n          captureError({\n            error: input.error,\n            page: input.page ?? ctx.page,\n            url: input.url || ctx.request.url,\n            log: input.log ?? ctx.log,\n          }),\n      });\n    }, options);\n  };\n  return wrapperHandler;\n};\n\nexport const basicCaptureErrorRouteHandler = <Ctx extends BasicCrawlingContext>(...args: Parameters<typeof captureErrorRouteHandler<Ctx>>) => captureErrorRouteHandler<Ctx>(...args); // prettier-ignore\nexport const httpCaptureErrorRouteHandler = <Ctx extends HttpCrawlingContext>(...args: Parameters<typeof captureErrorRouteHandler<Ctx>>) => captureErrorRouteHandler<Ctx>(...args); // prettier-ignore\nexport const jsdomCaptureErrorRouteHandler = <Ctx extends JSDOMCrawlingContext>(...args: Parameters<typeof captureErrorRouteHandler<Ctx>>) => captureErrorRouteHandler<Ctx>(...args); // prettier-ignore\nexport const playwrightCaptureErrorRouteHandler = <Ctx extends PlaywrightCrawlingContext>(...args: Parameters<typeof captureErrorRouteHandler<Ctx>>) => captureErrorRouteHandler<Ctx>(...args); // prettier-ignore\nexport const cheerioCaptureErrorRouteHandler = <Ctx extends CheerioCrawlingContext>(...args: Parameters<typeof captureErrorRouteHandler<Ctx>>) => captureErrorRouteHandler<Ctx>(...args); // prettier-ignore\nexport const puppeteerCaptureErrorRouteHandler = <Ctx extends PuppeteerCrawlingContext>(...args: Parameters<typeof captureErrorRouteHandler<Ctx>>) => captureErrorRouteHandler<Ctx>(...args); // prettier-ignore\n\n/**\n * Create an `ErrorHandler` function that can be assigned to\n * `failedRequestHandler` option of `BasicCrawlerOptions`.\n *\n * The function saves error to a Dataset, and optionally forwards it to Sentry.\n *\n * By default, error reports are saved to Apify Dataset.\n */\nexport const createErrorHandler = <Ctx extends CrawlingContext>(\n  options: CrawleeOneErrorHandlerOptions & { sendToSentry?: boolean }\n): ErrorHandler<Ctx> => {\n  return async ({ request, log, page }, error) => {\n    const url = request.loadedUrl || request.url;\n    captureError(\n      { error, url, log, page: page as Page },\n      {\n        io: options.io,\n        reportingDatasetId: options.reportingDatasetId,\n        allowScreenshot: options.allowScreenshot ?? true,\n        onErrorCapture: ({ error, report }) => {\n          if (!options.sendToSentry) return;\n\n          Sentry.captureException(error, { extra: report as any });\n        },\n      }\n    );\n  };\n};\n"]}

package/dist/cjs/lib/error/sentry.d.ts

@@ -0,0 +1,11 @@
+import * as Sentry from '@sentry/node';
+import type { CrawleeOneIO } from '../integrations/types';
+/**
+ * Sentry configuration common to all crawlers.
+ *
+ * By default, sentry is enabled only on the server.
+ * In Apify, whis is when `process.env.APIFY_IS_AT_HOME` is true.
+ */
+export declare const setupSentry: (sentryOptions?: Sentry.NodeOptions, options?: {
+    io?: CrawleeOneIO;
+}) => Promise<void>;

package/dist/cjs/lib/error/sentry.js

@@ -0,0 +1,60 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setupSentry = void 0;
+const Sentry = __importStar(require("@sentry/node"));
+const apify_1 = require("../integrations/apify");
+/**
+ * Sentry configuration common to all crawlers.
+ *
+ * By default, sentry is enabled only on the server.
+ * In Apify, whis is when `process.env.APIFY_IS_AT_HOME` is true.
+ */
+const setupSentry = (sentryOptions, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { io = apify_1.apifyIO } = options !== null && options !== void 0 ? options : {};
+    // As default, enable sentry only on Apify server
+    const enabled = (sentryOptions === null || sentryOptions === void 0 ? void 0 : sentryOptions.enabled) != null ? sentryOptions.enabled : yield io.isTelemetryEnabled();
+    if (!enabled)
+        return;
+    // We use this field for identification in UI, so it's required.
+    if (!(sentryOptions === null || sentryOptions === void 0 ? void 0 : sentryOptions.serverName))
+        throw Error('Sentry setup is missing "serverName" property.'); // prettier-ignore
+    Sentry.init(Object.assign({ dsn: 'https://5b2e0562b4ec4ef6805a3fbbf4ff8acd@o470159.ingest.sentry.io/4505019830370304', 
+        // Set tracesSampleRate to 1.0 to capture 100%
+        // of transactions for performance monitoring.
+        // We recommend adjusting this value in production
+        tracesSampleRate: 1.0 }, sentryOptions));
+});
+exports.setupSentry = setupSentry;
+//# sourceMappingURL=sentry.js.map

package/dist/cjs/lib/error/sentry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentry.js","sourceRoot":"","sources":["../../../../src/lib/error/sentry.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qDAAuC;AAGvC,iDAAgD;AAEhD;;;;;GAKG;AACI,MAAM,WAAW,GAAG,CACzB,aAAkC,EAClC,OAA+B,EAC/B,EAAE;IACF,MAAM,EAAE,EAAE,GAAG,eAAO,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAEvC,iDAAiD;IACjD,MAAM,OAAO,GACX,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,OAAO,KAAI,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,kBAAkB,EAAE,CAAC;IAEzF,IAAI,CAAC,OAAO;QAAE,OAAO;IAErB,gEAAgE;IAChE,IAAI,CAAC,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,UAAU,CAAA;QAAE,MAAM,KAAK,CAAC,gDAAgD,CAAC,CAAC,CAAC,kBAAkB;IAEjH,MAAM,CAAC,IAAI,iBACT,GAAG,EAAE,oFAAoF;QAEzF,8CAA8C;QAC9C,8CAA8C;QAC9C,kDAAkD;QAClD,gBAAgB,EAAE,GAAG,IAElB,aAAa,EAChB,CAAC;AACL,CAAC,CAAA,CAAC;AAzBW,QAAA,WAAW,eAyBtB","sourcesContent":["import * as Sentry from '@sentry/node';\n\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\n\n/**\n * Sentry configuration common to all crawlers.\n *\n * By default, sentry is enabled only on the server.\n * In Apify, whis is when `process.env.APIFY_IS_AT_HOME` is true.\n */\nexport const setupSentry = async (\n  sentryOptions?: Sentry.NodeOptions,\n  options?: { io?: CrawleeOneIO }\n) => {\n  const { io = apifyIO } = options ?? {};\n\n  // As default, enable sentry only on Apify server\n  const enabled =\n    sentryOptions?.enabled != null ? sentryOptions.enabled : await io.isTelemetryEnabled();\n\n  if (!enabled) return;\n\n  // We use this field for identification in UI, so it's required.\n  if (!sentryOptions?.serverName) throw Error('Sentry setup is missing \"serverName\" property.'); // prettier-ignore\n\n  Sentry.init({\n    dsn: 'https://5b2e0562b4ec4ef6805a3fbbf4ff8acd@o470159.ingest.sentry.io/4505019830370304',\n\n    // Set tracesSampleRate to 1.0 to capture 100%\n    // of transactions for performance monitoring.\n    // We recommend adjusting this value in production\n    tracesSampleRate: 1.0,\n\n    ...sentryOptions,\n  });\n};\n"]}

package/dist/cjs/lib/integrations/apify.d.ts

@@ -0,0 +1,67 @@
+import { ApifyEnv } from 'apify';
+import { CrawlingContext } from 'crawlee';
+import type { CrawleeOneIO } from './types';
+export interface ApifyErrorReport {
+    actorId: string | null;
+    actorRunId: string | null;
+    actorRunUrl: string;
+    errorName: string;
+    errorMessage: string;
+    pageUrl: string | null;
+    pageHtmlSnapshot: string | null;
+    pageScreenshot: string | null;
+}
+export interface ApifyEntryMetadata {
+    actorId: string | null;
+    actorRunId: string | null;
+    actorRunUrl: string | null;
+    contextId: string;
+    requestId: string | null;
+    /** The URL given to the crawler */
+    originalUrl: string | null;
+    /** The URL given to the crawler after possible redirects */
+    loadedUrl: string | null;
+    /** ISO datetime string that indicates the time when the request has been processed. */
+    dateHandled: string;
+    numberOfRetries: number;
+}
+/**
+ * Integration between CrawleeOne and Apify.
+ *
+ * This is the default integration.
+ */
+export type ApifyCrawleeOneIO = CrawleeOneIO<ApifyEnv, ApifyErrorReport, ApifyEntryMetadata>;
+/**
+ * Integration between CrawleeOne and Apify.
+ *
+ * This is the default integration.
+ */
+export declare const apifyIO: {
+    openDataset: (id?: string | null | undefined) => Promise<{
+        pushData: (data: import("crawlee").Dictionary | import("crawlee").Dictionary[]) => Promise<void>;
+        getItems: (options?: Pick<import("apify").DatasetDataOptions, "offset" | "desc" | "limit" | "fields"> | undefined) => import("../../utils/types").MaybePromise<object[]>;
+        getItemCount: () => Promise<number | null>;
+    }>;
+    openRequestQueue: (id?: string | null | undefined) => Promise<import("apify").RequestQueue>;
+    openKeyValueStore: (id?: string | null | undefined) => Promise<import("apify").KeyValueStore>;
+    getEnv: () => ApifyEnv;
+    getInput: <Input extends object>() => Promise<Input | null>;
+    runInContext: (userFunc: () => unknown, options?: import("apify").ExitOptions | undefined) => Promise<void>;
+    triggerDownstreamCrawler: <TInput extends object>(targetActorId: string, input?: TInput | undefined, options?: {
+        build?: string | undefined;
+    } | undefined) => Promise<void>;
+    createDefaultProxyConfiguration: (input: any) => Promise<import("apify").ProxyConfiguration | undefined>;
+    isTelemetryEnabled: () => boolean;
+    generateErrorReport: (input: import("./types").CrawleeOneErrorHandlerInput, options: import("../../utils/types").PickRequired<import("./types").CrawleeOneErrorHandlerOptions<ApifyEnv, ApifyErrorReport>, "io">) => import("../../utils/types").MaybePromise<ApifyErrorReport>;
+    generateEntryMetadata: <Ctx extends CrawlingContext<unknown, import("crawlee").Dictionary>>(ctx: Ctx) => {
+        actorId: string | null;
+        actorRunId: string | null;
+        actorRunUrl: string | null;
+        contextId: string;
+        requestId: string | null;
+        originalUrl: string;
+        loadedUrl: string | null;
+        dateHandled: string;
+        numberOfRetries: number;
+    };
+};

package/dist/cjs/lib/integrations/apify.js

@@ -0,0 +1,106 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.apifyIO = void 0;
+const apify_1 = require("apify");
+const crawlee_1 = require("crawlee");
+const generateApifyErrorReport = (input, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { error, page, url, log } = input;
+    const { io, allowScreenshot } = options;
+    // storeId is ID of current key-value store, where we save snapshots
+    // We can also capture actor and run IDs
+    // to have easy access in the reporting dataset
+    const { actorId, actorRunId, defaultKeyValueStoreId: storeId } = yield io.getEnv();
+    const actorRunUrl = `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`;
+    const randomNumber = Math.random();
+    const key = `ERROR-${randomNumber}`;
+    let pageScreenshot = null;
+    let pageHtmlSnapshot = null;
+    let pageUrl = url !== null && url !== void 0 ? url : null;
+    if (page && allowScreenshot) {
+        pageUrl = pageUrl || page.url();
+        log === null || log === void 0 ? void 0 : log.info('Capturing page snapshot');
+        yield crawlee_1.playwrightUtils.saveSnapshot(page, { key });
+        log === null || log === void 0 ? void 0 : log.info('DONE capturing page snapshot');
+        // You will have to adjust the keys if you save them in a non-standard way
+        pageScreenshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.jpg?disableRedirect=true`;
+        pageHtmlSnapshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.html?disableRedirect=true`;
+    }
+    // We create a report object
+    const report = {
+        actorId,
+        actorRunId,
+        actorRunUrl,
+        errorName: error.name,
+        errorMessage: error.toString(),
+        pageUrl,
+        pageHtmlSnapshot,
+        pageScreenshot,
+    };
+    return report;
+});
+const generateApifyEntryMetadata = (ctx) => {
+    var _a, _b, _c;
+    const { actorId, actorRunId } = apify_1.Actor.getEnv();
+    const actorRunUrl = actorId != null && actorRunId != null
+        ? `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`
+        : null;
+    const handledAt = new Date().toISOString();
+    const metadata = {
+        actorId,
+        actorRunId,
+        actorRunUrl,
+        contextId: ctx.id,
+        requestId: (_a = ctx.request.id) !== null && _a !== void 0 ? _a : null,
+        originalUrl: (_b = ctx.request.url) !== null && _b !== void 0 ? _b : null,
+        loadedUrl: (_c = ctx.request.loadedUrl) !== null && _c !== void 0 ? _c : null,
+        dateHandled: ctx.request.handledAt || handledAt,
+        numberOfRetries: ctx.request.retryCount,
+    };
+    return metadata;
+};
+/**
+ * Integration between CrawleeOne and Apify.
+ *
+ * This is the default integration.
+ */
+exports.apifyIO = {
+    openDataset: (...args) => __awaiter(void 0, void 0, void 0, function* () {
+        const dataset = yield apify_1.Actor.openDataset(...args);
+        const getItemCount = () => __awaiter(void 0, void 0, void 0, function* () { var _a, _b; return (_b = (_a = (yield dataset.getInfo())) === null || _a === void 0 ? void 0 : _a.itemCount) !== null && _b !== void 0 ? _b : null; });
+        const getItems = (options) => __awaiter(void 0, void 0, void 0, function* () {
+            const result = yield dataset.getData(Object.assign(Object.assign({}, options), { skipEmpty: true }));
+            return result.items;
+        });
+        return {
+            pushData: dataset.pushData.bind(dataset),
+            getItems,
+            getItemCount,
+        };
+    }),
+    openRequestQueue: (...args) => apify_1.Actor.openRequestQueue(...args),
+    openKeyValueStore: (...args) => apify_1.Actor.openKeyValueStore(...args),
+    getEnv: (...args) => apify_1.Actor.getEnv(...args),
+    getInput: (...args) => apify_1.Actor.getInput(...args),
+    runInContext: (...args) => __awaiter(void 0, void 0, void 0, function* () {
+        yield apify_1.Actor.main(...args);
+    }),
+    triggerDownstreamCrawler: (...args) => apify_1.Actor.metamorph(...args),
+    createDefaultProxyConfiguration: (input) => __awaiter(void 0, void 0, void 0, function* () {
+        return process.env.APIFY_IS_AT_HOME
+            ? yield apify_1.Actor.createProxyConfiguration(input === null || input === void 0 ? void 0 : input.proxy)
+            : undefined;
+    }),
+    isTelemetryEnabled: () => !!process.env.APIFY_IS_AT_HOME,
+    generateErrorReport: generateApifyErrorReport,
+    generateEntryMetadata: generateApifyEntryMetadata,
+};
+//# sourceMappingURL=apify.js.map

package/dist/cjs/lib/integrations/apify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apify.js","sourceRoot":"","sources":["../../../../src/lib/integrations/apify.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iCAAwC;AACxC,qCAA2D;AAuC3D,MAAM,wBAAwB,GAA6C,CACzE,KAAK,EACL,OAAO,EACP,EAAE;IACF,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IACxC,MAAM,EAAE,EAAE,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC;IAExC,oEAAoE;IACpE,wCAAwC;IACxC,+CAA+C;IAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC;IAEnF,MAAM,WAAW,GAAG,oCAAoC,OAAO,SAAS,UAAU,EAAE,CAAC;IAErF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,SAAS,YAAY,EAAE,CAAC;IAEpC,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,gBAAgB,GAAkB,IAAI,CAAC;IAC3C,IAAI,OAAO,GAAkB,GAAG,aAAH,GAAG,cAAH,GAAG,GAAI,IAAI,CAAC;IACzC,IAAI,IAAI,IAAI,eAAe,EAAE;QAC3B,OAAO,GAAG,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QAChC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACrC,MAAM,yBAAe,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC1C,0EAA0E;QAC1E,cAAc,GAAG,6CAA6C,OAAO,YAAY,GAAG,2BAA2B,CAAC;QAChH,gBAAgB,GAAG,6CAA6C,OAAO,YAAY,GAAG,4BAA4B,CAAC;KACpH;IAED,4BAA4B;IAC5B,MAAM,MAAM,GAAG;QACb,OAAO;QACP,UAAU;QACV,WAAW;QACX,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,YAAY,EAAE,KAAK,CAAC,QAAQ,EAAE;QAE9B,OAAO;QACP,gBAAgB;QAChB,cAAc;KACY,CAAC;IAE7B,OAAO,MAAM,CAAC;AAChB,CAAC,CAAA,CAAC;AAEF,MAAM,0BAA0B,GAAG,CAA8B,GAAQ,EAAE,EAAE;;IAC3E,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,aAAK,CAAC,MAAM,EAAE,CAAC;IAC/C,MAAM,WAAW,GACf,OAAO,IAAI,IAAI,IAAI,UAAU,IAAI,IAAI;QACnC,CAAC,CAAC,oCAAoC,OAAO,SAAS,UAAU,EAAE;QAClE,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,MAAM,QAAQ,GAAG;QACf,OAAO;QACP,UAAU;QACV,WAAW;QACX,SAAS,EAAE,GAAG,CAAC,EAAE;QACjB,SAAS,EAAE,MAAA,GAAG,CAAC,OAAO,CAAC,EAAE,mCAAI,IAAI;QAEjC,WAAW,EAAE,MAAA,GAAG,CAAC,OAAO,CAAC,GAAG,mCAAI,IAAI;QACpC,SAAS,EAAE,MAAA,GAAG,CAAC,OAAO,CAAC,SAAS,mCAAI,IAAI;QAExC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,SAAS;QAC/C,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,UAAU;KACX,CAAC;IAE/B,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF;;;;GAIG;AACU,QAAA,OAAO,GAAG;IACrB,WAAW,EAAE,CAAO,GAAG,IAAI,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,MAAM,aAAK,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,GAAS,EAAE,8DAAC,OAAA,MAAA,MAAA,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,0CAAE,SAAS,mCAAI,IAAI,CAAA,GAAA,CAAC;QAC9E,MAAM,QAAQ,GAAkC,CAAO,OAAO,EAAE,EAAE;YAChE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,iCAC/B,OAAO,KACV,SAAS,EAAE,IAAI,IACf,CAAC;YACH,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC,CAAA,CAAC;QAEF,OAAO;YACL,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;YACxC,QAAQ;YACR,YAAY;SACb,CAAC;IACJ,CAAC,CAAA;IACD,gBAAgB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,aAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAC9D,iBAAiB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,aAAK,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAChE,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,aAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAC1C,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,aAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IAC9C,YAAY,EAAE,CAAO,GAAG,IAAI,EAAE,EAAE;QAC9B,MAAM,aAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAC5B,CAAC,CAAA;IACD,wBAAwB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,aAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;IAC/D,+BAA+B,EAAE,CAAO,KAAU,EAAE,EAAE;QACpD,OAAO,OAAO,CAAC,GAAG,CAAC,gBAAgB;YACjC,CAAC,CAAC,MAAM,aAAK,CAAC,wBAAwB,CAAC,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,CAAC;YACpD,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC,CAAA;IACD,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB;IACxD,mBAAmB,EAAE,wBAAwB;IAC7C,qBAAqB,EAAE,0BAA0B;CACtB,CAAC","sourcesContent":["import { Actor, ApifyEnv } from 'apify';\nimport { CrawlingContext, playwrightUtils } from 'crawlee';\n\nimport type { CrawleeOneDataset, CrawleeOneIO } from './types';\n\nexport interface ApifyErrorReport {\n  actorId: string | null;\n  actorRunId: string | null;\n  actorRunUrl: string;\n  errorName: string;\n  errorMessage: string;\n  pageUrl: string | null;\n  pageHtmlSnapshot: string | null;\n  pageScreenshot: string | null;\n}\n\nexport interface ApifyEntryMetadata {\n  actorId: string | null;\n  actorRunId: string | null;\n  actorRunUrl: string | null;\n  contextId: string;\n  requestId: string | null;\n\n  /** The URL given to the crawler */\n  originalUrl: string | null;\n  /** The URL given to the crawler after possible redirects */\n  loadedUrl: string | null;\n\n  /** ISO datetime string that indicates the time when the request has been processed. */\n  dateHandled: string;\n  numberOfRetries: number;\n}\n\n/**\n * Integration between CrawleeOne and Apify.\n *\n * This is the default integration.\n */\nexport type ApifyCrawleeOneIO = CrawleeOneIO<ApifyEnv, ApifyErrorReport, ApifyEntryMetadata>;\n\nconst generateApifyErrorReport: ApifyCrawleeOneIO['generateErrorReport'] = async (\n  input,\n  options\n) => {\n  const { error, page, url, log } = input;\n  const { io, allowScreenshot } = options;\n\n  // storeId is ID of current key-value store, where we save snapshots\n  // We can also capture actor and run IDs\n  // to have easy access in the reporting dataset\n  const { actorId, actorRunId, defaultKeyValueStoreId: storeId } = await io.getEnv();\n\n  const actorRunUrl = `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`;\n\n  const randomNumber = Math.random();\n  const key = `ERROR-${randomNumber}`;\n\n  let pageScreenshot: string | null = null;\n  let pageHtmlSnapshot: string | null = null;\n  let pageUrl: string | null = url ?? null;\n  if (page && allowScreenshot) {\n    pageUrl = pageUrl || page.url();\n    log?.info('Capturing page snapshot');\n    await playwrightUtils.saveSnapshot(page, { key });\n    log?.info('DONE capturing page snapshot');\n    // You will have to adjust the keys if you save them in a non-standard way\n    pageScreenshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.jpg?disableRedirect=true`;\n    pageHtmlSnapshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.html?disableRedirect=true`;\n  }\n\n  // We create a report object\n  const report = {\n    actorId,\n    actorRunId,\n    actorRunUrl,\n    errorName: error.name,\n    errorMessage: error.toString(),\n\n    pageUrl,\n    pageHtmlSnapshot,\n    pageScreenshot,\n  } satisfies ApifyErrorReport;\n\n  return report;\n};\n\nconst generateApifyEntryMetadata = <Ctx extends CrawlingContext>(ctx: Ctx) => {\n  const { actorId, actorRunId } = Actor.getEnv();\n  const actorRunUrl =\n    actorId != null && actorRunId != null\n      ? `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`\n      : null;\n  const handledAt = new Date().toISOString();\n\n  const metadata = {\n    actorId,\n    actorRunId,\n    actorRunUrl,\n    contextId: ctx.id,\n    requestId: ctx.request.id ?? null,\n\n    originalUrl: ctx.request.url ?? null,\n    loadedUrl: ctx.request.loadedUrl ?? null,\n\n    dateHandled: ctx.request.handledAt || handledAt,\n    numberOfRetries: ctx.request.retryCount,\n  } satisfies ApifyEntryMetadata;\n\n  return metadata;\n};\n\n/**\n * Integration between CrawleeOne and Apify.\n *\n * This is the default integration.\n */\nexport const apifyIO = {\n  openDataset: async (...args) => {\n    const dataset = await Actor.openDataset(...args);\n    const getItemCount = async () => (await dataset.getInfo())?.itemCount ?? null;\n    const getItems: CrawleeOneDataset['getItems'] = async (options) => {\n      const result = await dataset.getData({\n        ...options,\n        skipEmpty: true,\n      });\n      return result.items;\n    };\n\n    return {\n      pushData: dataset.pushData.bind(dataset),\n      getItems,\n      getItemCount,\n    };\n  },\n  openRequestQueue: (...args) => Actor.openRequestQueue(...args),\n  openKeyValueStore: (...args) => Actor.openKeyValueStore(...args),\n  getEnv: (...args) => Actor.getEnv(...args),\n  getInput: (...args) => Actor.getInput(...args),\n  runInContext: async (...args) => {\n    await Actor.main(...args);\n  },\n  triggerDownstreamCrawler: (...args) => Actor.metamorph(...args),\n  createDefaultProxyConfiguration: async (input: any) => {\n    return process.env.APIFY_IS_AT_HOME\n      ? await Actor.createProxyConfiguration(input?.proxy)\n      : undefined;\n  },\n  isTelemetryEnabled: () => !!process.env.APIFY_IS_AT_HOME,\n  generateErrorReport: generateApifyErrorReport,\n  generateEntryMetadata: generateApifyEntryMetadata,\n} satisfies ApifyCrawleeOneIO;\n"]}