crawlee-one 2.0.4 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/{cjs/api.d.ts → api.d.ts} +6 -6
- package/dist/apify.js +7 -0
- package/dist/apify.js.map +1 -0
- package/dist/chunk-P2JHMDHF.js +116 -0
- package/dist/chunk-P2JHMDHF.js.map +1 -0
- package/dist/chunk-UUDYXYJW.js +317 -0
- package/dist/chunk-UUDYXYJW.js.map +1 -0
- package/dist/chunk-V524BXEG.js +23 -0
- package/dist/chunk-V524BXEG.js.map +1 -0
- package/dist/{cjs/cli → cli}/commands/codegen.d.ts +1 -1
- package/dist/{cjs/cli → cli}/commands/config.d.ts +1 -1
- package/dist/cli.js +75 -0
- package/dist/cli.js.map +1 -0
- package/dist/{cjs/constants.d.ts → constants.d.ts} +3 -1
- package/dist/index.d.ts +27 -0
- package/dist/index.js +2331 -0
- package/dist/index.js.map +1 -0
- package/dist/{cjs/lib → lib}/actions/scrapeListing.d.ts +1 -1
- package/dist/{cjs/lib → lib}/actor/actor.d.ts +7 -9
- package/dist/{cjs/lib → lib}/actor/types.d.ts +8 -8
- package/dist/lib/error/errorHandler.d.ts +60 -0
- package/dist/{cjs/lib → lib}/input.d.ts +104 -100
- package/dist/{cjs/lib → lib}/integrations/apify.d.ts +1 -1
- package/dist/{cjs/lib → lib}/integrations/types.d.ts +1 -1
- package/dist/{cjs/lib → lib}/io/dataset.d.ts +4 -4
- package/dist/{cjs/lib → lib}/io/pushData.d.ts +3 -3
- package/dist/{cjs/lib → lib}/io/pushRequests.d.ts +4 -4
- package/dist/{cjs/lib → lib}/io/requestQueue.d.ts +3 -3
- package/dist/{cjs/lib → lib}/log.d.ts +4 -4
- package/dist/{cjs/lib → lib}/migrate/types.d.ts +1 -1
- package/dist/{cjs/lib → lib}/router/router.d.ts +8 -8
- package/dist/{cjs/lib → lib}/router/types.d.ts +2 -2
- package/dist/lib/telemetry/sentry.d.ts +4 -0
- package/dist/{cjs/lib → lib}/telemetry/types.d.ts +3 -3
- package/dist/{cjs/lib → lib}/test/actor.d.ts +2 -2
- package/dist/{cjs/lib → lib}/test/mockApifyClient.d.ts +7 -7
- package/dist/sentry.js +7 -0
- package/dist/sentry.js.map +1 -0
- package/dist/{cjs/types → types}/config.d.ts +1 -1
- package/dist/{cjs/types → types}/index.d.ts +8 -6
- package/dist/{cjs/utils → utils}/async.d.ts +1 -1
- package/dist/utils/package.d.ts +5 -0
- package/dist/{cjs/utils → utils}/types.d.ts +1 -1
- package/dist/{cjs/utils → utils}/valueMonitor.d.ts +1 -1
- package/package.json +57 -56
- package/README.md +0 -931
- package/dist/cjs/api/composer.d.ts +0 -0
- package/dist/cjs/api/composer.js +0 -248
- package/dist/cjs/api/composer.js.map +0 -1
- package/dist/cjs/api/types.d.ts +0 -1
- package/dist/cjs/api/types.js +0 -121
- package/dist/cjs/api/types.js.map +0 -1
- package/dist/cjs/api.js +0 -57
- package/dist/cjs/api.js.map +0 -1
- package/dist/cjs/cli/cli.js +0 -88
- package/dist/cjs/cli/cli.js.map +0 -1
- package/dist/cjs/cli/commands/codegen.js +0 -177
- package/dist/cjs/cli/commands/codegen.js.map +0 -1
- package/dist/cjs/cli/commands/config.js +0 -65
- package/dist/cjs/cli/commands/config.js.map +0 -1
- package/dist/cjs/cli/index.js +0 -6
- package/dist/cjs/cli/index.js.map +0 -1
- package/dist/cjs/composer.d.ts +0 -0
- package/dist/cjs/composer.js +0 -248
- package/dist/cjs/composer.js.map +0 -1
- package/dist/cjs/config/config.d.ts +0 -8
- package/dist/cjs/config/config.js +0 -161
- package/dist/cjs/config/config.js.map +0 -1
- package/dist/cjs/config/types.d.ts +0 -35
- package/dist/cjs/config/types.js +0 -3
- package/dist/cjs/config/types.js.map +0 -1
- package/dist/cjs/constants.js +0 -21
- package/dist/cjs/constants.js.map +0 -1
- package/dist/cjs/index.d.ts +0 -27
- package/dist/cjs/index.js +0 -45
- package/dist/cjs/index.js.map +0 -1
- package/dist/cjs/lib/actions/dom.d.ts +0 -102
- package/dist/cjs/lib/actions/dom.js +0 -743
- package/dist/cjs/lib/actions/dom.js.map +0 -1
- package/dist/cjs/lib/actions/domUtils.d.ts +0 -42
- package/dist/cjs/lib/actions/domUtils.js +0 -126
- package/dist/cjs/lib/actions/domUtils.js.map +0 -1
- package/dist/cjs/lib/actions/page.d.ts +0 -69
- package/dist/cjs/lib/actions/page.js +0 -205
- package/dist/cjs/lib/actions/page.js.map +0 -1
- package/dist/cjs/lib/actions/scrapeListing.js +0 -242
- package/dist/cjs/lib/actions/scrapeListing.js.map +0 -1
- package/dist/cjs/lib/actor/actor.js +0 -386
- package/dist/cjs/lib/actor/actor.js.map +0 -1
- package/dist/cjs/lib/actor/types.js +0 -3
- package/dist/cjs/lib/actor/types.js.map +0 -1
- package/dist/cjs/lib/actor.d.ts +0 -189
- package/dist/cjs/lib/actor.js +0 -225
- package/dist/cjs/lib/actor.js.map +0 -1
- package/dist/cjs/lib/actorSpec.d.ts +0 -20
- package/dist/cjs/lib/actorSpec.js +0 -3
- package/dist/cjs/lib/actorSpec.js.map +0 -1
- package/dist/cjs/lib/config.d.ts +0 -561
- package/dist/cjs/lib/config.js +0 -707
- package/dist/cjs/lib/config.js.map +0 -1
- package/dist/cjs/lib/dataset/maxCount.d.ts +0 -30
- package/dist/cjs/lib/dataset/maxCount.js +0 -55
- package/dist/cjs/lib/dataset/maxCount.js.map +0 -1
- package/dist/cjs/lib/dataset/pushData.d.ts +0 -123
- package/dist/cjs/lib/dataset/pushData.js +0 -182
- package/dist/cjs/lib/dataset/pushData.js.map +0 -1
- package/dist/cjs/lib/dataset.d.ts +0 -98
- package/dist/cjs/lib/dataset.js +0 -122
- package/dist/cjs/lib/dataset.js.map +0 -1
- package/dist/cjs/lib/dom.d.ts +0 -78
- package/dist/cjs/lib/dom.js +0 -243
- package/dist/cjs/lib/dom.js.map +0 -1
- package/dist/cjs/lib/error/errorHandler.d.ts +0 -84
- package/dist/cjs/lib/error/errorHandler.js +0 -142
- package/dist/cjs/lib/error/errorHandler.js.map +0 -1
- package/dist/cjs/lib/error/sentry.d.ts +0 -11
- package/dist/cjs/lib/error/sentry.js +0 -60
- package/dist/cjs/lib/error/sentry.js.map +0 -1
- package/dist/cjs/lib/input.js +0 -707
- package/dist/cjs/lib/input.js.map +0 -1
- package/dist/cjs/lib/integrations/apify.js +0 -136
- package/dist/cjs/lib/integrations/apify.js.map +0 -1
- package/dist/cjs/lib/integrations/types.js +0 -3
- package/dist/cjs/lib/integrations/types.js.map +0 -1
- package/dist/cjs/lib/io/dataset.js +0 -86
- package/dist/cjs/lib/io/dataset.js.map +0 -1
- package/dist/cjs/lib/io/maxCount.d.ts +0 -30
- package/dist/cjs/lib/io/maxCount.js +0 -55
- package/dist/cjs/lib/io/maxCount.js.map +0 -1
- package/dist/cjs/lib/io/pushData.js +0 -197
- package/dist/cjs/lib/io/pushData.js.map +0 -1
- package/dist/cjs/lib/io/pushRequests.js +0 -64
- package/dist/cjs/lib/io/pushRequests.js.map +0 -1
- package/dist/cjs/lib/io/requestQueue.js +0 -40
- package/dist/cjs/lib/io/requestQueue.js.map +0 -1
- package/dist/cjs/lib/log.js +0 -54
- package/dist/cjs/lib/log.js.map +0 -1
- package/dist/cjs/lib/migrate/localMigrator.js +0 -57
- package/dist/cjs/lib/migrate/localMigrator.js.map +0 -1
- package/dist/cjs/lib/migrate/localState.js +0 -43
- package/dist/cjs/lib/migrate/localState.js.map +0 -1
- package/dist/cjs/lib/migrate/types.js +0 -3
- package/dist/cjs/lib/migrate/types.js.map +0 -1
- package/dist/cjs/lib/readme/apify/readme.d.ts +0 -65
- package/dist/cjs/lib/readme/apify/readme.js +0 -534
- package/dist/cjs/lib/readme/apify/readme.js.map +0 -1
- package/dist/cjs/lib/readme/apify/types.d.ts +0 -260
- package/dist/cjs/lib/readme/apify/types.js +0 -54
- package/dist/cjs/lib/readme/apify/types.js.map +0 -1
- package/dist/cjs/lib/readme/readme.d.ts +0 -65
- package/dist/cjs/lib/readme/readme.js +0 -534
- package/dist/cjs/lib/readme/readme.js.map +0 -1
- package/dist/cjs/lib/readme/types.d.ts +0 -260
- package/dist/cjs/lib/readme/types.js +0 -54
- package/dist/cjs/lib/readme/types.js.map +0 -1
- package/dist/cjs/lib/router/router.js +0 -274
- package/dist/cjs/lib/router/router.js.map +0 -1
- package/dist/cjs/lib/router/types.js +0 -3
- package/dist/cjs/lib/router/types.js.map +0 -1
- package/dist/cjs/lib/router.d.ts +0 -132
- package/dist/cjs/lib/router.js +0 -171
- package/dist/cjs/lib/router.js.map +0 -1
- package/dist/cjs/lib/scraper/scrapeListing.d.ts +0 -78
- package/dist/cjs/lib/scraper/scrapeListing.js +0 -242
- package/dist/cjs/lib/scraper/scrapeListing.js.map +0 -1
- package/dist/cjs/lib/telemetry/sentry.d.ts +0 -4
- package/dist/cjs/lib/telemetry/sentry.js +0 -64
- package/dist/cjs/lib/telemetry/sentry.js.map +0 -1
- package/dist/cjs/lib/telemetry/types.js +0 -3
- package/dist/cjs/lib/telemetry/types.js.map +0 -1
- package/dist/cjs/lib/test/actor.js +0 -56
- package/dist/cjs/lib/test/actor.js.map +0 -1
- package/dist/cjs/lib/test/mockApifyClient.js +0 -176
- package/dist/cjs/lib/test/mockApifyClient.js.map +0 -1
- package/dist/cjs/types/config.js +0 -3
- package/dist/cjs/types/config.js.map +0 -1
- package/dist/cjs/types/index.js +0 -13
- package/dist/cjs/types/index.js.map +0 -1
- package/dist/cjs/types.d.ts +0 -52
- package/dist/cjs/types.js +0 -13
- package/dist/cjs/types.js.map +0 -1
- package/dist/cjs/utils/async.js +0 -74
- package/dist/cjs/utils/async.js.map +0 -1
- package/dist/cjs/utils/error.js +0 -10
- package/dist/cjs/utils/error.js.map +0 -1
- package/dist/cjs/utils/format.js +0 -19
- package/dist/cjs/utils/format.js.map +0 -1
- package/dist/cjs/utils/package.d.ts +0 -15
- package/dist/cjs/utils/package.js +0 -25
- package/dist/cjs/utils/package.js.map +0 -1
- package/dist/cjs/utils/types.js +0 -11
- package/dist/cjs/utils/types.js.map +0 -1
- package/dist/cjs/utils/url.js +0 -32
- package/dist/cjs/utils/url.js.map +0 -1
- package/dist/cjs/utils/valueMonitor.js +0 -91
- package/dist/cjs/utils/valueMonitor.js.map +0 -1
- /package/dist/{cjs/cli → cli}/cli.d.ts +0 -0
- /package/dist/{cjs/cli → cli}/index.d.ts +0 -0
- /package/dist/{cjs/lib → lib}/migrate/localMigrator.d.ts +0 -0
- /package/dist/{cjs/lib → lib}/migrate/localState.d.ts +0 -0
- /package/dist/{cjs/utils → utils}/error.d.ts +0 -0
- /package/dist/{cjs/utils → utils}/format.d.ts +0 -0
- /package/dist/{cjs/utils → utils}/url.d.ts +0 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023-present Juro Oravec
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { CrawleeOneActorInst, CrawleeOneActorDef, CrawleeOneActorRouterCtx, CrawleeOneCtx } from './lib/actor/types';
|
|
2
|
-
import type { AllActorInputs } from './lib/input';
|
|
3
|
-
import type { CrawleeOneRouteHandler, CrawleeOneRoute } from './lib/router/types';
|
|
4
|
-
import type { CrawlerMeta, CrawlerType } from './types';
|
|
5
|
-
import type { MaybePromise } from './utils/types';
|
|
1
|
+
import type { CrawleeOneActorInst, CrawleeOneActorDef, CrawleeOneActorRouterCtx, CrawleeOneCtx } from './lib/actor/types.js';
|
|
2
|
+
import type { AllActorInputs } from './lib/input.js';
|
|
3
|
+
import type { CrawleeOneRouteHandler, CrawleeOneRoute } from './lib/router/types.js';
|
|
4
|
+
import type { CrawlerMeta, CrawlerType } from './types/index.js';
|
|
5
|
+
import type { MaybePromise } from './utils/types.js';
|
|
6
6
|
/** Args object passed to `crawleeOne` */
|
|
7
7
|
export interface CrawleeOneArgs<TType extends CrawlerType, T extends CrawleeOneCtx<CrawlerMeta<TType>['context']>> {
|
|
8
8
|
/** Type specifying the Crawlee crawler class, input options, and more. */
|
|
@@ -104,4 +104,4 @@ export interface CrawleeOneArgs<TType extends CrawlerType, T extends CrawleeOneC
|
|
|
104
104
|
};
|
|
105
105
|
routes: Record<T['labels'], CrawleeOneRoute<T, CrawleeOneActorRouterCtx<T>>>;
|
|
106
106
|
}
|
|
107
|
-
export declare const crawleeOne: <TType extends
|
|
107
|
+
export declare const crawleeOne: <TType extends CrawlerType, T extends CrawleeOneCtx<CrawlerMeta<TType>["context"]> = CrawleeOneCtx<CrawlerMeta<TType>["context"]>>(args: CrawleeOneArgs<TType, T>) => Promise<void>;
|
package/dist/apify.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// src/lib/integrations/apify.ts
|
|
2
|
+
import { Actor } from "apify";
|
|
3
|
+
import { playwrightUtils } from "crawlee";
|
|
4
|
+
var generateApifyErrorReport = async (input, options) => {
|
|
5
|
+
const { error, page, url, log } = input;
|
|
6
|
+
const { allowScreenshot } = options;
|
|
7
|
+
const { actorId, actorRunId, defaultKeyValueStoreId: storeId } = await Actor.getEnv();
|
|
8
|
+
const actorRunUrl = `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`;
|
|
9
|
+
const randomNumber = Math.random();
|
|
10
|
+
const key = `ERROR-${randomNumber}`;
|
|
11
|
+
let pageScreenshot = null;
|
|
12
|
+
let pageHtmlSnapshot = null;
|
|
13
|
+
let pageUrl = url ?? null;
|
|
14
|
+
if (page && allowScreenshot) {
|
|
15
|
+
pageUrl = pageUrl || page.url();
|
|
16
|
+
log?.info("Capturing page snapshot");
|
|
17
|
+
await playwrightUtils.saveSnapshot(page, { key });
|
|
18
|
+
log?.info("DONE capturing page snapshot");
|
|
19
|
+
pageScreenshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.jpg?disableRedirect=true`;
|
|
20
|
+
pageHtmlSnapshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.html?disableRedirect=true`;
|
|
21
|
+
}
|
|
22
|
+
const report = {
|
|
23
|
+
actorId,
|
|
24
|
+
actorRunId,
|
|
25
|
+
actorRunUrl,
|
|
26
|
+
errorName: error.name,
|
|
27
|
+
errorMessage: error.toString(),
|
|
28
|
+
pageUrl,
|
|
29
|
+
pageHtmlSnapshot,
|
|
30
|
+
pageScreenshot
|
|
31
|
+
};
|
|
32
|
+
return report;
|
|
33
|
+
};
|
|
34
|
+
var generateApifyEntryMetadata = (ctx) => {
|
|
35
|
+
const { actorId, actorRunId } = Actor.getEnv();
|
|
36
|
+
const actorRunUrl = actorId != null && actorRunId != null ? `https://console.apify.com/actors/${actorId}/runs/${actorRunId}` : null;
|
|
37
|
+
const handledAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
38
|
+
const metadata = {
|
|
39
|
+
actorId,
|
|
40
|
+
actorRunId,
|
|
41
|
+
actorRunUrl,
|
|
42
|
+
contextId: ctx.id,
|
|
43
|
+
requestId: ctx.request.id ?? null,
|
|
44
|
+
originalUrl: ctx.request.url ?? null,
|
|
45
|
+
loadedUrl: ctx.request.loadedUrl ?? null,
|
|
46
|
+
dateHandled: ctx.request.handledAt || handledAt,
|
|
47
|
+
numberOfRetries: ctx.request.retryCount
|
|
48
|
+
};
|
|
49
|
+
return metadata;
|
|
50
|
+
};
|
|
51
|
+
var apifyIO = {
|
|
52
|
+
openDataset: async (...args) => {
|
|
53
|
+
const dataset = await Actor.openDataset(...args);
|
|
54
|
+
const getItemCount = async () => (await dataset.getInfo())?.itemCount ?? null;
|
|
55
|
+
const getItems = async (options) => {
|
|
56
|
+
const result = await dataset.getData({
|
|
57
|
+
...options,
|
|
58
|
+
skipEmpty: true
|
|
59
|
+
});
|
|
60
|
+
return result.items;
|
|
61
|
+
};
|
|
62
|
+
return {
|
|
63
|
+
pushData: dataset.pushData.bind(dataset),
|
|
64
|
+
getItems,
|
|
65
|
+
getItemCount
|
|
66
|
+
};
|
|
67
|
+
},
|
|
68
|
+
openRequestQueue: async (...args) => {
|
|
69
|
+
const queue = await Actor.openRequestQueue(...args);
|
|
70
|
+
const clear = async () => {
|
|
71
|
+
let req;
|
|
72
|
+
do {
|
|
73
|
+
req = await queue.fetchNextRequest();
|
|
74
|
+
if (req) await queue.markRequestHandled(req);
|
|
75
|
+
} while (req);
|
|
76
|
+
};
|
|
77
|
+
return {
|
|
78
|
+
addRequests: (...args2) => queue.addRequests(...args2),
|
|
79
|
+
markRequestHandled: (...args2) => queue.markRequestHandled(...args2),
|
|
80
|
+
fetchNextRequest: (...args2) => queue.fetchNextRequest(...args2),
|
|
81
|
+
reclaimRequest: (...args2) => queue.reclaimRequest(...args2),
|
|
82
|
+
isFinished: (...args2) => queue.isFinished(...args2),
|
|
83
|
+
handledCount: (...args2) => queue.handledCount(...args2),
|
|
84
|
+
drop: (...args2) => queue.drop(...args2),
|
|
85
|
+
clear
|
|
86
|
+
};
|
|
87
|
+
},
|
|
88
|
+
openKeyValueStore: async (...args) => {
|
|
89
|
+
const store = await Actor.openKeyValueStore(...args);
|
|
90
|
+
const clear = async () => {
|
|
91
|
+
await store.forEachKey((key) => store.setValue(key, null));
|
|
92
|
+
};
|
|
93
|
+
return {
|
|
94
|
+
getValue: (...args2) => store.getValue(...args2),
|
|
95
|
+
setValue: (...args2) => store.setValue(...args2),
|
|
96
|
+
drop: (...args2) => store.drop(...args2),
|
|
97
|
+
clear
|
|
98
|
+
};
|
|
99
|
+
},
|
|
100
|
+
getInput: (...args) => Actor.getInput(...args),
|
|
101
|
+
runInContext: async (...args) => {
|
|
102
|
+
await Actor.main(...args);
|
|
103
|
+
},
|
|
104
|
+
triggerDownstreamCrawler: (...args) => Actor.metamorph(...args),
|
|
105
|
+
createDefaultProxyConfiguration: async (input) => {
|
|
106
|
+
return process.env.APIFY_IS_AT_HOME ? await Actor.createProxyConfiguration(input?.proxy) : void 0;
|
|
107
|
+
},
|
|
108
|
+
isTelemetryEnabled: () => !!process.env.APIFY_IS_AT_HOME,
|
|
109
|
+
generateErrorReport: generateApifyErrorReport,
|
|
110
|
+
generateEntryMetadata: generateApifyEntryMetadata
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export {
|
|
114
|
+
apifyIO
|
|
115
|
+
};
|
|
116
|
+
//# sourceMappingURL=chunk-P2JHMDHF.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/integrations/apify.ts"],"sourcesContent":["import { Actor, ApifyEnv } from 'apify';\nimport { CrawlingContext, Request as CrawleeRequest, playwrightUtils } from 'crawlee';\n\nimport type { CrawleeOneDataset, CrawleeOneIO } from './types.js';\n\nexport interface ApifyErrorReport {\n actorId: string | null;\n actorRunId: string | null;\n actorRunUrl: string;\n errorName: string;\n errorMessage: string;\n pageUrl: string | null;\n pageHtmlSnapshot: string | null;\n pageScreenshot: string | null;\n}\n\nexport interface ApifyEntryMetadata {\n actorId: string | null;\n actorRunId: string | null;\n actorRunUrl: string | null;\n contextId: string;\n requestId: string | null;\n\n /** The URL given to the crawler */\n originalUrl: string | null;\n /** The URL given to the crawler after possible redirects */\n loadedUrl: string | null;\n\n /** ISO datetime string that indicates the time when the request has been processed. */\n dateHandled: string;\n numberOfRetries: number;\n}\n\n/**\n * Integration between CrawleeOne and Apify.\n *\n * This is the default integration.\n */\nexport type ApifyCrawleeOneIO = CrawleeOneIO<ApifyEnv, ApifyErrorReport, ApifyEntryMetadata>;\n\nconst generateApifyErrorReport: ApifyCrawleeOneIO['generateErrorReport'] = async (\n input,\n options\n) => {\n const { error, page, url, log } = input;\n const { allowScreenshot } = options;\n\n // storeId is ID of current key-value store, where we save snapshots\n // We can also capture actor and run IDs\n // to have easy access in the reporting dataset\n const { actorId, actorRunId, defaultKeyValueStoreId: storeId } = await Actor.getEnv();\n\n const actorRunUrl = `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`;\n\n const randomNumber = Math.random();\n const key = `ERROR-${randomNumber}`;\n\n let pageScreenshot: string | null = null;\n let pageHtmlSnapshot: string | null = null;\n let pageUrl: string | null = url ?? null;\n if (page && allowScreenshot) {\n pageUrl = pageUrl || page.url();\n log?.info('Capturing page snapshot');\n await playwrightUtils.saveSnapshot(page, { key });\n log?.info('DONE capturing page snapshot');\n // You will have to adjust the keys if you save them in a non-standard way\n pageScreenshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.jpg?disableRedirect=true`;\n pageHtmlSnapshot = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${key}.html?disableRedirect=true`;\n }\n\n // We create a report object\n const report = {\n actorId,\n actorRunId,\n actorRunUrl,\n errorName: error.name,\n errorMessage: error.toString(),\n\n pageUrl,\n pageHtmlSnapshot,\n pageScreenshot,\n } satisfies ApifyErrorReport;\n\n return report;\n};\n\nconst generateApifyEntryMetadata = <Ctx extends CrawlingContext>(ctx: Ctx) => {\n const { actorId, actorRunId } = Actor.getEnv();\n const actorRunUrl =\n actorId != null && actorRunId != null\n ? `https://console.apify.com/actors/${actorId}/runs/${actorRunId}`\n : null;\n const handledAt = new Date().toISOString();\n\n const metadata = {\n actorId,\n actorRunId,\n actorRunUrl,\n contextId: ctx.id,\n requestId: ctx.request.id ?? null,\n\n originalUrl: ctx.request.url ?? null,\n loadedUrl: ctx.request.loadedUrl ?? null,\n\n dateHandled: ctx.request.handledAt || handledAt,\n numberOfRetries: ctx.request.retryCount,\n } satisfies ApifyEntryMetadata;\n\n return metadata;\n};\n\n/**\n * Integration between CrawleeOne and Apify.\n *\n * This is the default integration.\n */\nexport const apifyIO: ApifyCrawleeOneIO = {\n openDataset: async (...args) => {\n const dataset = await Actor.openDataset(...args);\n const getItemCount = async () => (await dataset.getInfo())?.itemCount ?? null;\n const getItems: CrawleeOneDataset['getItems'] = async (options) => {\n const result = await dataset.getData({\n ...options,\n skipEmpty: true,\n });\n return result.items;\n };\n\n return {\n pushData: dataset.pushData.bind(dataset),\n getItems,\n getItemCount,\n };\n },\n openRequestQueue: async (...args) => {\n const queue = await Actor.openRequestQueue(...args);\n const clear = async () => {\n let req: CrawleeRequest | null;\n do {\n req = await queue.fetchNextRequest();\n if (req) await queue.markRequestHandled(req);\n } while (req);\n };\n\n return {\n addRequests: (...args) => queue.addRequests(...args),\n markRequestHandled: (...args) => queue.markRequestHandled(...args),\n fetchNextRequest: (...args) => queue.fetchNextRequest(...args),\n reclaimRequest: (...args) => queue.reclaimRequest(...args),\n isFinished: (...args) => queue.isFinished(...args),\n handledCount: (...args) => queue.handledCount(...args),\n drop: (...args) => queue.drop(...args),\n clear,\n };\n },\n openKeyValueStore: async (...args) => {\n const store = await Actor.openKeyValueStore(...args);\n const clear = async () => {\n await store.forEachKey((key) => store.setValue(key, null));\n };\n\n return {\n getValue: (...args) => store.getValue(...args),\n setValue: (...args) => store.setValue(...args),\n drop: (...args) => store.drop(...args),\n clear,\n };\n },\n getInput: (...args) => Actor.getInput(...args),\n runInContext: async (...args) => {\n await Actor.main(...args);\n },\n triggerDownstreamCrawler: (...args) => Actor.metamorph(...args),\n createDefaultProxyConfiguration: async (input: any) => {\n return process.env.APIFY_IS_AT_HOME\n ? await Actor.createProxyConfiguration(input?.proxy)\n : undefined;\n },\n isTelemetryEnabled: () => !!process.env.APIFY_IS_AT_HOME,\n generateErrorReport: generateApifyErrorReport,\n generateEntryMetadata: generateApifyEntryMetadata,\n} satisfies ApifyCrawleeOneIO;\n"],"mappings":";AAAA,SAAS,aAAuB;AAChC,SAAqD,uBAAuB;AAuC5E,IAAM,2BAAqE,OACzE,OACA,YACG;AACH,QAAM,EAAE,OAAO,MAAM,KAAK,IAAI,IAAI;AAClC,QAAM,EAAE,gBAAgB,IAAI;AAK5B,QAAM,EAAE,SAAS,YAAY,wBAAwB,QAAQ,IAAI,MAAM,MAAM,OAAO;AAEpF,QAAM,cAAc,oCAAoC,OAAO,SAAS,UAAU;AAElF,QAAM,eAAe,KAAK,OAAO;AACjC,QAAM,MAAM,SAAS,YAAY;AAEjC,MAAI,iBAAgC;AACpC,MAAI,mBAAkC;AACtC,MAAI,UAAyB,OAAO;AACpC,MAAI,QAAQ,iBAAiB;AAC3B,cAAU,WAAW,KAAK,IAAI;AAC9B,SAAK,KAAK,yBAAyB;AACnC,UAAM,gBAAgB,aAAa,MAAM,EAAE,IAAI,CAAC;AAChD,SAAK,KAAK,8BAA8B;AAExC,qBAAiB,6CAA6C,OAAO,YAAY,GAAG;AACpF,uBAAmB,6CAA6C,OAAO,YAAY,GAAG;AAAA,EACxF;AAGA,QAAM,SAAS;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,cAAc,MAAM,SAAS;AAAA,IAE7B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AACT;AAEA,IAAM,6BAA6B,CAA8B,QAAa;AAC5E,QAAM,EAAE,SAAS,WAAW,IAAI,MAAM,OAAO;AAC7C,QAAM,cACJ,WAAW,QAAQ,cAAc,OAC7B,oCAAoC,OAAO,SAAS,UAAU,KAC9D;AACN,QAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AAEzC,QAAM,WAAW;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,IAAI;AAAA,IACf,WAAW,IAAI,QAAQ,MAAM;AAAA,IAE7B,aAAa,IAAI,QAAQ,OAAO;AAAA,IAChC,WAAW,IAAI,QAAQ,aAAa;AAAA,IAEpC,aAAa,IAAI,QAAQ,aAAa;AAAA,IACtC,iBAAiB,IAAI,QAAQ;AAAA,EAC/B;AAEA,SAAO;AACT;AAOO,IAAM,UAA6B;AAAA,EACxC,aAAa,UAAU,SAAS;AAC9B,UAAM,UAAU,MAAM,MAAM,YAAY,GAAG,IAAI;AAC/C,UAAM,eAAe,aAAa,MAAM,QAAQ,QAAQ,IAAI,aAAa;AACzE,UAAM,WAA0C,OAAO,YAAY;AACjE,YAAM,SAAS,MAAM,QAAQ,QAAQ;AAAA,QACnC,GAAG;AAAA,QACH,WAAW;AAAA,MACb,CAAC;AACD,aAAO,OAAO;AAAA,IAChB;AAEA,WAAO;AAAA,MACL,UAAU,QAAQ,SAAS,KAAK,OAAO;AAAA,MACvC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EACA,kBAAkB,UAAU,SAAS;AACnC,UAAM,QAAQ,MAAM,MAAM,iBAAiB,GAAG,IAAI;AAClD,UAAM,QAAQ,YAAY;AACxB,UAAI;AACJ,SAAG;AACD,cAAM,MAAM,MAAM,iBAAiB;AACnC,YAAI,IAAK,OAAM,MAAM,mBAAmB,GAAG;AAAA,MAC7C,SAAS;AAAA,IACX;AAEA,WAAO;AAAA,MACL,aAAa,IAAIA,UAAS,MAAM,YAAY,GAAGA,KAAI;AAAA,MACnD,oBAAoB,IAAIA,UAAS,MAAM,mBAAmB,GAAGA,KAAI;AAAA,MACjE,kBAAkB,IAAIA,UAAS,MAAM,iBAAiB,GAAGA,KAAI;AAAA,MAC7D,gBAAgB,IAAIA,UAAS,MAAM,eAAe,GAAGA,KAAI;AAAA,MACzD,YAAY,IAAIA,UAAS,MAAM,WAAW,GAAGA,KAAI;AAAA,MACjD,cAAc,IAAIA,UAAS,MAAM,aAAa,GAAGA,KAAI;AAAA,MACrD,MAAM,IAAIA,UAAS,MAAM,KAAK,GAAGA,KAAI;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAAA,EACA,mBAAmB,UAAU,SAAS;AACpC,UAAM,QAAQ,MAAM,MAAM,kBAAkB,GAAG,IAAI;AACnD,UAAM,QAAQ,YAAY;AACxB,YAAM,MAAM,WAAW,CAAC,QAAQ,MAAM,SAAS,KAAK,IAAI,CAAC;AAAA,IAC3D;AAEA,WAAO;AAAA,MACL,UAAU,IAAIA,UAAS,MAAM,SAAS,GAAGA,KAAI;AAAA,MAC7C,UAAU,IAAIA,UAAS,MAAM,SAAS,GAAGA,KAAI;AAAA,MAC7C,MAAM,IAAIA,UAAS,MAAM,KAAK,GAAGA,KAAI;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAAA,EACA,UAAU,IAAI,SAAS,MAAM,SAAS,GAAG,IAAI;AAAA,EAC7C,cAAc,UAAU,SAAS;AAC/B,UAAM,MAAM,KAAK,GAAG,IAAI;AAAA,EAC1B;AAAA,EACA,0BAA0B,IAAI,SAAS,MAAM,UAAU,GAAG,IAAI;AAAA,EAC9D,iCAAiC,OAAO,UAAe;AACrD,WAAO,QAAQ,IAAI,mBACf,MAAM,MAAM,yBAAyB,OAAO,KAAK,IACjD;AAAA,EACN;AAAA,EACA,oBAAoB,MAAM,CAAC,CAAC,QAAQ,IAAI;AAAA,EACxC,qBAAqB;AAAA,EACrB,uBAAuB;AACzB;","names":["args"]}
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
// src/lib/migrate/localMigrator.ts
|
|
2
|
+
import path from "path";
|
|
3
|
+
import dotenv from "dotenv";
|
|
4
|
+
import { glob } from "glob";
|
|
5
|
+
import { ApifyClient } from "apify-client";
|
|
6
|
+
var createLocalMigrator = ({
|
|
7
|
+
migrationsDir,
|
|
8
|
+
extension = ".js",
|
|
9
|
+
delimeter = "_"
|
|
10
|
+
}) => {
|
|
11
|
+
const findLocalMigrationFileByVersion = async (version) => {
|
|
12
|
+
const files = await glob(path.join(migrationsDir, `${version}${delimeter}*${extension}`));
|
|
13
|
+
if (!files.length) {
|
|
14
|
+
throw Error(`No migration file matched version "${version}"`);
|
|
15
|
+
}
|
|
16
|
+
if (files.length > 1) {
|
|
17
|
+
throw Error(
|
|
18
|
+
`Ambiguous migration version. Version "${version}" matched multiple migration files`
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
return files[0];
|
|
22
|
+
};
|
|
23
|
+
const setup = () => {
|
|
24
|
+
dotenv.config();
|
|
25
|
+
const client = new ApifyClient({
|
|
26
|
+
token: process.env.APIFY_TOKEN
|
|
27
|
+
});
|
|
28
|
+
return { client };
|
|
29
|
+
};
|
|
30
|
+
const migrate = async (version) => {
|
|
31
|
+
const migFile = await findLocalMigrationFileByVersion(version);
|
|
32
|
+
const { client } = setup();
|
|
33
|
+
const { migrate: migrate2 } = (await import(migFile)).default;
|
|
34
|
+
await migrate2(client);
|
|
35
|
+
};
|
|
36
|
+
const unmigrate = async (version) => {
|
|
37
|
+
const migFile = await findLocalMigrationFileByVersion(version);
|
|
38
|
+
const { client } = setup();
|
|
39
|
+
const { unmigrate: unmigrate2 } = (await import(migFile)).default;
|
|
40
|
+
await unmigrate2(client);
|
|
41
|
+
};
|
|
42
|
+
return {
|
|
43
|
+
migrate,
|
|
44
|
+
unmigrate
|
|
45
|
+
};
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// src/cli/commands/config.ts
|
|
49
|
+
import { cosmiconfig } from "cosmiconfig";
|
|
50
|
+
import Joi from "joi";
|
|
51
|
+
|
|
52
|
+
// src/utils/package.ts
|
|
53
|
+
import { readFileSync } from "fs";
|
|
54
|
+
import { fileURLToPath } from "url";
|
|
55
|
+
import path2 from "path";
|
|
56
|
+
var getPackageJsonInfo = (importMetaUrl, fields) => {
|
|
57
|
+
let dir = path2.dirname(fileURLToPath(importMetaUrl));
|
|
58
|
+
while (true) {
|
|
59
|
+
try {
|
|
60
|
+
const pkg = JSON.parse(readFileSync(path2.join(dir, "package.json"), "utf-8"));
|
|
61
|
+
const result = {};
|
|
62
|
+
for (const field of fields) result[field] = pkg[field];
|
|
63
|
+
return result;
|
|
64
|
+
} catch {
|
|
65
|
+
const parent = path2.dirname(dir);
|
|
66
|
+
if (parent === dir) throw new Error("package.json not found");
|
|
67
|
+
dir = parent;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
// src/types/index.ts
|
|
73
|
+
var CRAWLER_TYPE = [
|
|
74
|
+
"basic",
|
|
75
|
+
"http",
|
|
76
|
+
"jsdom",
|
|
77
|
+
"cheerio",
|
|
78
|
+
"playwright",
|
|
79
|
+
"adaptive-playwright",
|
|
80
|
+
"puppeteer"
|
|
81
|
+
];
|
|
82
|
+
|
|
83
|
+
// src/cli/commands/config.ts
|
|
84
|
+
var varNamePattern = /^[a-z_][a-z0-9_]*$/i;
|
|
85
|
+
var configSchemaCrawlerJoiSchema = Joi.object({
|
|
86
|
+
type: Joi.string().allow(...CRAWLER_TYPE).required(),
|
|
87
|
+
routes: Joi.array().items(Joi.string().required()).required()
|
|
88
|
+
}).required().unknown(false);
|
|
89
|
+
var configSchemaJoiSchema = Joi.object({
|
|
90
|
+
crawlers: Joi.object().pattern(Joi.string().pattern(varNamePattern), configSchemaCrawlerJoiSchema).required().unknown(false)
|
|
91
|
+
}).required().unknown(false);
|
|
92
|
+
var configJoiSchema = Joi.object({
|
|
93
|
+
version: Joi.number().allow(1).required(),
|
|
94
|
+
schema: configSchemaJoiSchema
|
|
95
|
+
}).required().unknown(false);
|
|
96
|
+
var validateConfig = (config) => {
|
|
97
|
+
Joi.assert(config, configJoiSchema);
|
|
98
|
+
};
|
|
99
|
+
var loadConfig = async (configFilePath) => {
|
|
100
|
+
const pkgJson = getPackageJsonInfo(import.meta.url, ["name"]);
|
|
101
|
+
const explorer = cosmiconfig(pkgJson.name);
|
|
102
|
+
const result = configFilePath ? await explorer.load(configFilePath) : await explorer.search();
|
|
103
|
+
const config = result?.config ?? null;
|
|
104
|
+
return config;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
// src/cli/commands/codegen.ts
|
|
108
|
+
import fsp from "fs/promises";
|
|
109
|
+
import path3 from "path";
|
|
110
|
+
|
|
111
|
+
// src/constants.ts
|
|
112
|
+
import {
|
|
113
|
+
AdaptivePlaywrightCrawler,
|
|
114
|
+
BasicCrawler,
|
|
115
|
+
CheerioCrawler,
|
|
116
|
+
HttpCrawler,
|
|
117
|
+
JSDOMCrawler,
|
|
118
|
+
PlaywrightCrawler,
|
|
119
|
+
PuppeteerCrawler
|
|
120
|
+
} from "crawlee";
|
|
121
|
+
var actorClassByType = {
|
|
122
|
+
basic: BasicCrawler,
|
|
123
|
+
http: HttpCrawler,
|
|
124
|
+
cheerio: CheerioCrawler,
|
|
125
|
+
jsdom: JSDOMCrawler,
|
|
126
|
+
playwright: PlaywrightCrawler,
|
|
127
|
+
"adaptive-playwright": AdaptivePlaywrightCrawler,
|
|
128
|
+
puppeteer: PuppeteerCrawler
|
|
129
|
+
};
|
|
130
|
+
var crawlingContextNameByType = {
|
|
131
|
+
basic: "BasicCrawlingContext",
|
|
132
|
+
http: "HttpCrawlingContext",
|
|
133
|
+
cheerio: "CheerioCrawlingContext",
|
|
134
|
+
jsdom: "JSDOMCrawlingContext",
|
|
135
|
+
playwright: "PlaywrightCrawlingContext",
|
|
136
|
+
"adaptive-playwright": "PlaywrightCrawlingContext",
|
|
137
|
+
puppeteer: "PuppeteerCrawlingContext"
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
// src/cli/commands/codegen.ts
|
|
141
|
+
var makeUnion = (items) => items.map((s) => `"${s}"`).join(` | `);
|
|
142
|
+
var makeEnum = (items) => "{\n" + items.map((s) => ` '${s}' = '${s}'`).join(`,
|
|
143
|
+
`) + "\n}";
|
|
144
|
+
var formatters = {
|
|
145
|
+
type: (name, value, typeArgs) => {
|
|
146
|
+
const typeArgsStr = typeArgs?.length ? `<${typeArgs.join(", ")}>` : "";
|
|
147
|
+
return `export type ${name}${typeArgsStr} = ${value};`;
|
|
148
|
+
},
|
|
149
|
+
typeFunc: (name, value, typeArgs) => {
|
|
150
|
+
const typeArgsStr = typeArgs?.length ? `<${typeArgs.join(", ")}>` : "";
|
|
151
|
+
return `export type ${name} = ${typeArgsStr}${value};`;
|
|
152
|
+
},
|
|
153
|
+
func: (name, value, typeArgs) => {
|
|
154
|
+
const typeArgsStr = typeArgs?.length ? `<${typeArgs.join(", ")}>` : "";
|
|
155
|
+
return `export const ${name} = ${typeArgsStr}${value};`;
|
|
156
|
+
},
|
|
157
|
+
// enum `CrawlerName`LabelEnum { "detailPage" = "detailPage", "otherLabel" = "otherLabel", ... };
|
|
158
|
+
enum: (name, value, typeArgs) => {
|
|
159
|
+
return `export enum ${name} ${value}`;
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
var parseTypesFromSchema = (schema) => {
|
|
163
|
+
const definitions = {};
|
|
164
|
+
const imports = {};
|
|
165
|
+
const addImports = (pkg, newEntries, options) => {
|
|
166
|
+
const { typeOnly } = options ?? {};
|
|
167
|
+
const entries = imports[pkg] = imports[pkg] || /* @__PURE__ */ new Set();
|
|
168
|
+
newEntries.forEach((name) => entries.add({ name, typeOnly }));
|
|
169
|
+
return newEntries.reduce((agg, key) => {
|
|
170
|
+
agg[key] = key;
|
|
171
|
+
return agg;
|
|
172
|
+
}, {});
|
|
173
|
+
};
|
|
174
|
+
const define = (key, value, options) => {
|
|
175
|
+
const kind = options?.kind ?? "type";
|
|
176
|
+
const typeArgs = options?.typeArgs ?? [];
|
|
177
|
+
if (!definitions[key]) {
|
|
178
|
+
const resolvedVal = typeof value === "function" ? value() : value;
|
|
179
|
+
const formatter = formatters[kind];
|
|
180
|
+
const valFormatted = formatter(key, resolvedVal, typeArgs);
|
|
181
|
+
definitions[key] = valFormatted;
|
|
182
|
+
}
|
|
183
|
+
return key;
|
|
184
|
+
};
|
|
185
|
+
const {
|
|
186
|
+
AllActorInputs: actorInput,
|
|
187
|
+
CrawleeOneActorRouterCtx: actorRouterCtx,
|
|
188
|
+
CrawleeOneActorInst: actorCtx,
|
|
189
|
+
CrawleeOneRoute: routeType,
|
|
190
|
+
CrawleeOneRouteHandler: routeHandler,
|
|
191
|
+
CrawleeOneRouteWrapper: routeWrapper,
|
|
192
|
+
CrawleeOneRouteMatcher: routeMatcher,
|
|
193
|
+
CrawleeOneRouteMatcherFn: routeMatcherFn,
|
|
194
|
+
CrawleeOneIO: ioType,
|
|
195
|
+
CrawleeOneTelemetry: telemType,
|
|
196
|
+
CrawleeOneCtx: ctxType,
|
|
197
|
+
CrawleeOneArgs: argsType,
|
|
198
|
+
crawleeOne: crawleeOneFn
|
|
199
|
+
} = addImports("crawlee-one", [
|
|
200
|
+
"AllActorInputs",
|
|
201
|
+
"CrawleeOneActorRouterCtx",
|
|
202
|
+
"CrawleeOneActorInst",
|
|
203
|
+
"CrawleeOneRoute",
|
|
204
|
+
"CrawleeOneRouteHandler",
|
|
205
|
+
"CrawleeOneRouteWrapper",
|
|
206
|
+
"CrawleeOneRouteMatcher",
|
|
207
|
+
"CrawleeOneRouteMatcherFn",
|
|
208
|
+
"CrawleeOneIO",
|
|
209
|
+
"CrawleeOneTelemetry",
|
|
210
|
+
"CrawleeOneCtx",
|
|
211
|
+
"CrawleeOneArgs",
|
|
212
|
+
"crawleeOne"
|
|
213
|
+
]);
|
|
214
|
+
addImports("crawlee", Object.values(crawlingContextNameByType), { typeOnly: true });
|
|
215
|
+
const maybeP = define("MaybePromise", "T | Promise<T>", { typeArgs: ["T"] });
|
|
216
|
+
Object.entries(schema.crawlers).forEach(([crawlerName, crawler]) => {
|
|
217
|
+
const crawlerType = crawler.type;
|
|
218
|
+
const crawlingContextTypeName = crawlingContextNameByType[crawlerType];
|
|
219
|
+
const labelKey = define(`${crawlerName}Label`, () => makeUnion(crawler.routes));
|
|
220
|
+
const labelEnumKey = define(`${crawlerName}LabelEnum`, () => makeEnum(crawler.routes), {
|
|
221
|
+
kind: "enum"
|
|
222
|
+
});
|
|
223
|
+
const ctxTypeArgs = [
|
|
224
|
+
`TInput extends Record<string, any> = ${actorInput}`,
|
|
225
|
+
`TIO extends ${ioType} = ${ioType}`,
|
|
226
|
+
`Telem extends ${telemType}<any, any> = ${telemType}<any, any>`
|
|
227
|
+
];
|
|
228
|
+
const ctxKey = define(
|
|
229
|
+
`${crawlerName}Ctx`,
|
|
230
|
+
`${ctxType}<${crawlingContextTypeName}, ${labelKey}, TInput, TIO, Telem>`,
|
|
231
|
+
{ typeArgs: ctxTypeArgs }
|
|
232
|
+
);
|
|
233
|
+
const crawlerKey = define(
|
|
234
|
+
`${crawlerName}Crawler`,
|
|
235
|
+
`(args: Omit<${argsType}<"${crawlerType}", ${ctxKey}<TInput, TIO, Telem>>, 'type'>) => ${crawleeOneFn}<"${crawlerType}", ${ctxKey}<TInput, TIO, Telem>>({ ...args, type: "${crawlerType}"});`,
|
|
236
|
+
{ kind: "func", typeArgs: ctxTypeArgs }
|
|
237
|
+
);
|
|
238
|
+
const routerCtxKey = define(
|
|
239
|
+
`${crawlerName}RouterContext`,
|
|
240
|
+
`${actorRouterCtx}<${ctxKey}<TInput, TIO, Telem>>`,
|
|
241
|
+
{ typeArgs: ctxTypeArgs }
|
|
242
|
+
);
|
|
243
|
+
const actorCtxKey = define(
|
|
244
|
+
`${crawlerName}ActorCtx`,
|
|
245
|
+
`${actorCtx}<${ctxKey}<TInput, TIO, Telem>>`,
|
|
246
|
+
{ typeArgs: ctxTypeArgs }
|
|
247
|
+
);
|
|
248
|
+
const routeKey = define(
|
|
249
|
+
`${crawlerName}Route`,
|
|
250
|
+
`${routeType}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,
|
|
251
|
+
{
|
|
252
|
+
typeArgs: ctxTypeArgs
|
|
253
|
+
}
|
|
254
|
+
);
|
|
255
|
+
const routeHandlerValue = `${routeHandler}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`;
|
|
256
|
+
const routeHandlerKey = define(`${crawlerName}RouteHandler`, routeHandlerValue, {
|
|
257
|
+
typeArgs: ctxTypeArgs
|
|
258
|
+
});
|
|
259
|
+
const routeWrapperKey = define(
|
|
260
|
+
`${crawlerName}RouteWrapper`,
|
|
261
|
+
`${routeWrapper}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,
|
|
262
|
+
{ typeArgs: ctxTypeArgs }
|
|
263
|
+
);
|
|
264
|
+
const routeMatcherKey = define(
|
|
265
|
+
`${crawlerName}RouteMatcher`,
|
|
266
|
+
`${routeMatcher}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,
|
|
267
|
+
{ typeArgs: ctxTypeArgs }
|
|
268
|
+
);
|
|
269
|
+
const routeMatcherFnKey = define(
|
|
270
|
+
`${crawlerName}RouteMatcherFn`,
|
|
271
|
+
`${routeMatcherFn}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,
|
|
272
|
+
{ typeArgs: ctxTypeArgs }
|
|
273
|
+
);
|
|
274
|
+
const onBeforeHandlerKey = define(`${crawlerName}OnBeforeHandler`, routeHandlerValue, {
|
|
275
|
+
typeArgs: ctxTypeArgs
|
|
276
|
+
});
|
|
277
|
+
const onAfterHandlerKey = define(`${crawlerName}OnAfterHandler`, routeHandlerValue, {
|
|
278
|
+
typeArgs: ctxTypeArgs
|
|
279
|
+
});
|
|
280
|
+
const onReadyKey = define(
|
|
281
|
+
`${crawlerName}OnReady`,
|
|
282
|
+
`(actor: ${actorCtxKey}<TInput, TIO, Telem>) => ${maybeP}<void>;`,
|
|
283
|
+
{ kind: "typeFunc", typeArgs: ctxTypeArgs }
|
|
284
|
+
);
|
|
285
|
+
});
|
|
286
|
+
const finalImports = Object.entries(imports).reduce(
|
|
287
|
+
(agg, [pkg, entriesSet]) => {
|
|
288
|
+
const entries = [...entriesSet.values()];
|
|
289
|
+
const formattedEntries = entries.map((e) => e.name).join(", ");
|
|
290
|
+
const typeStr = entries.every((e) => e.typeOnly) ? "type " : "";
|
|
291
|
+
agg[pkg] = `import ${typeStr}{ ${formattedEntries} } from "${pkg}"`;
|
|
292
|
+
return agg;
|
|
293
|
+
},
|
|
294
|
+
{}
|
|
295
|
+
);
|
|
296
|
+
return { imports: finalImports, definitions };
|
|
297
|
+
};
|
|
298
|
+
var generateTypes = async (outfile, configOrPath) => {
|
|
299
|
+
const config = !configOrPath || typeof configOrPath === "string" ? await loadConfig(configOrPath) : configOrPath;
|
|
300
|
+
validateConfig(config);
|
|
301
|
+
const { imports, definitions } = await parseTypesFromSchema(config.schema);
|
|
302
|
+
const fileContent = Object.values(imports).join("\n") + "\n\n\n" + Object.values(definitions).join("\n\n");
|
|
303
|
+
const outdir = path3.dirname(outfile);
|
|
304
|
+
await fsp.mkdir(outdir, { recursive: true });
|
|
305
|
+
await fsp.writeFile(outfile, fileContent, "utf-8");
|
|
306
|
+
console.log(`Done generating types to ${outfile}`);
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
export {
|
|
310
|
+
actorClassByType,
|
|
311
|
+
createLocalMigrator,
|
|
312
|
+
getPackageJsonInfo,
|
|
313
|
+
validateConfig,
|
|
314
|
+
loadConfig,
|
|
315
|
+
generateTypes
|
|
316
|
+
};
|
|
317
|
+
//# sourceMappingURL=chunk-UUDYXYJW.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/migrate/localMigrator.ts","../src/cli/commands/config.ts","../src/utils/package.ts","../src/types/index.ts","../src/cli/commands/codegen.ts","../src/constants.ts"],"sourcesContent":["import path from 'node:path';\nimport dotenv from 'dotenv';\nimport { glob } from 'glob';\nimport { ApifyClient } from 'apify-client';\n\nimport type { Migration } from './types.js';\n\nexport const createLocalMigrator = ({\n migrationsDir,\n extension = '.js',\n delimeter = '_',\n}: {\n migrationsDir: string;\n /** Extension glob */\n extension: string;\n /** Delimeter between version and rest of file name */\n delimeter: string;\n}) => {\n const findLocalMigrationFileByVersion = async (version: string): Promise<string> => {\n // Find files like \"v1_bla_bla_bla.js\" (by default)\n const files = await glob(path.join(migrationsDir, `${version}${delimeter}*${extension}`));\n if (!files.length) {\n throw Error(`No migration file matched version \"${version}\"`);\n }\n if (files.length > 1) {\n throw Error(\n `Ambiguous migration version. Version \"${version}\" matched multiple migration files`\n );\n }\n return files[0];\n };\n\n const setup = () => {\n dotenv.config();\n const client = new ApifyClient({\n token: process.env.APIFY_TOKEN,\n });\n return { client };\n };\n\n const migrate = async (version: string) => {\n const migFile = await findLocalMigrationFileByVersion(version);\n const { client } = setup();\n const { migrate } = ((await import(migFile)) as { default: Migration }).default;\n await migrate(client);\n };\n\n const unmigrate = async (version: string) => {\n const migFile = await findLocalMigrationFileByVersion(version);\n const { client } = setup();\n const { unmigrate } = ((await import(migFile)) as { default: Migration }).default;\n await unmigrate(client);\n };\n\n return {\n migrate,\n unmigrate,\n };\n};\n","import { cosmiconfig } from 'cosmiconfig';\nimport Joi from 'joi';\n\nimport { getPackageJsonInfo } from '../../utils/package.js';\nimport type {\n CrawleeOneConfig,\n CrawleeOneConfigSchema,\n CrawleeOneConfigSchemaCrawler,\n} from '../../types/config.js';\nimport { CRAWLER_TYPE } from '../../types/index.js';\n\n/** Pattern for a valid JS variable */\nconst varNamePattern = /^[a-z_][a-z0-9_]*$/i;\n\nconst configSchemaCrawlerJoiSchema = Joi.object({\n type: Joi.string().allow(...CRAWLER_TYPE).required(),\n routes: Joi.array().items(Joi.string().required()).required(),\n} satisfies Record<keyof CrawleeOneConfigSchemaCrawler, Joi.Schema>)\n .required().unknown(false); // prettier-ignore\n\nconst configSchemaJoiSchema = Joi.object({\n crawlers: Joi.object()\n .pattern(Joi.string().pattern(varNamePattern), configSchemaCrawlerJoiSchema)\n .required().unknown(false),\n} satisfies Record<keyof CrawleeOneConfigSchema, Joi.Schema>)\n .required().unknown(false); // prettier-ignore\n\nconst configJoiSchema = Joi.object({\n version: Joi.number().allow(1).required(),\n schema: configSchemaJoiSchema,\n} satisfies Record<keyof CrawleeOneConfig, Joi.Schema>)\n .required().unknown(false); // prettier-ignore\n\n/**\n * Validate given CrawleeOne config.\n *\n * Config can be passed directly, or you can specify the path to the config file.\n * For the latter, the config will be loaded using {@link loadConfig}.\n */\nexport const validateConfig = (config: unknown | string) => {\n Joi.assert(config, configJoiSchema);\n};\n\n/**\n * Load CrawleeOne config file. Config will be searched for using CosmicConfig.\n *\n * Optionally, you can supply path to the config file.\n *\n * Learn more: https://github.com/cosmiconfig/cosmiconfig\n */\nexport const loadConfig = async (configFilePath?: string) => {\n const pkgJson = getPackageJsonInfo(import.meta.url, ['name']);\n\n // See https://github.com/cosmiconfig/cosmiconfig#usage-for-tooling-developers\n const explorer = cosmiconfig(pkgJson.name);\n const result = configFilePath ? await explorer.load(configFilePath) : await explorer.search();\n const config = (result?.config ?? null) as CrawleeOneConfig | null;\n return config;\n};\n","import { readFileSync } from 'node:fs';\nimport { fileURLToPath } from 'node:url';\nimport path from 'node:path';\n\n/**\n * Finds the closest package.json by walking up from the given\n * `import.meta.url` and returns only the requested fields.\n */\nexport const getPackageJsonInfo = <TFields extends string = string>(\n importMetaUrl: string,\n fields: TFields[]\n): Record<TFields, any> => {\n let dir = path.dirname(fileURLToPath(importMetaUrl));\n while (true) {\n try {\n const pkg = JSON.parse(readFileSync(path.join(dir, 'package.json'), 'utf-8'));\n const result = {} as Record<TFields, any>;\n for (const field of fields) result[field] = pkg[field];\n return result;\n } catch {\n const parent = path.dirname(dir);\n if (parent === dir) throw new Error('package.json not found');\n dir = parent;\n }\n }\n};\n","import type {\n AdaptivePlaywrightCrawler,\n AdaptivePlaywrightCrawlerOptions,\n BasicCrawler,\n CrawlingContext,\n BasicCrawlerOptions,\n BasicCrawlingContext,\n HttpCrawler,\n HttpCrawlingContext,\n HttpCrawlerOptions,\n InternalHttpCrawlingContext,\n JSDOMCrawler,\n JSDOMCrawlingContext,\n JSDOMCrawlerOptions,\n CheerioCrawler,\n CheerioCrawlingContext,\n CheerioCrawlerOptions,\n PlaywrightCrawler,\n PlaywrightCrawlingContext,\n PlaywrightCrawlerOptions,\n PuppeteerCrawler,\n PuppeteerCrawlerOptions,\n PuppeteerCrawlingContext,\n} from 'crawlee';\n\nimport type { ArrVal } from '../utils/types.js';\n\n/** Available Crawlee crawler types. */\nexport const CRAWLER_TYPE = [\n 'basic',\n 'http',\n 'jsdom',\n 'cheerio',\n 'playwright',\n 'adaptive-playwright',\n 'puppeteer',\n] as const;\nexport type CrawlerType = ArrVal<typeof CRAWLER_TYPE>;\n\n/**\n * Type utility that retrieves types related to specific Crawlee crawlers.\n *\n * E.g. `CrawleeMeta<'jsdom'>` will return types for JSDOM crawler:\n *\n * ```ts\n * {\n * crawler: JSDOMCrawler,\n * context: JSDOMCrawlingContext<TData>,\n * options: JSDOMCrawlerOptions<TData>\n * }\n * ```\n *\n * Which can then be used like so:\n * ```ts\n * type MyType = CrawleeMeta<'jsdom'>['context'];\n * ```\n * */\nexport type CrawlerMeta<\n T extends CrawlerType,\n Ctx extends CrawlingContext = CrawlingContext,\n TData extends Record<string, any> = Record<string, any>,\n> = T extends 'http'\n ? Ctx extends InternalHttpCrawlingContext\n ? { crawler: HttpCrawler<Ctx>, context: HttpCrawlingContext<TData>, options: HttpCrawlerOptions<Ctx> } // prettier-ignore\n : never\n : T extends 'jsdom'\n ? { crawler: JSDOMCrawler, context: JSDOMCrawlingContext<TData>, options: JSDOMCrawlerOptions<TData> } // prettier-ignore\n : T extends 'cheerio'\n ? { crawler: CheerioCrawler, context: CheerioCrawlingContext<TData>, options: CheerioCrawlerOptions<TData> } // prettier-ignore\n : T extends 'playwright'\n ? { crawler: PlaywrightCrawler, context: PlaywrightCrawlingContext<TData>, options: PlaywrightCrawlerOptions } // prettier-ignore\n : T extends 'adaptive-playwright'\n ? { crawler: AdaptivePlaywrightCrawler, context: PlaywrightCrawlingContext<TData>, options: AdaptivePlaywrightCrawlerOptions } // prettier-ignore\n : T extends 'puppeteer'\n ? { crawler: PuppeteerCrawler, context: PuppeteerCrawlingContext<TData>, options: PuppeteerCrawlerOptions } // prettier-ignore\n : T extends 'basic'\n ? Ctx extends CrawlingContext\n ? { crawler: BasicCrawler<Ctx>, context: BasicCrawlingContext<TData>, options: BasicCrawlerOptions<Ctx> } // prettier-ignore\n : never\n : never;\n\n/** URL string or object passed to Crawler.run */\nexport type CrawlerUrl = import('@crawlee/core').Source | string;\n","/* eslint-disable @typescript-eslint/no-unused-vars */\nimport fsp from 'fs/promises';\nimport path from 'node:path';\n\nimport type { CrawleeOneConfig, CrawleeOneConfigSchema } from '../../types/config.js';\nimport { crawlingContextNameByType } from '../../constants.js';\nimport { loadConfig, validateConfig } from './config.js';\n// NOTE: We intentionally import these to know when their names change\nimport type { AllActorInputs } from '../../lib/input.js';\nimport type { CrawleeOneActorInst, CrawleeOneActorRouterCtx } from '../../lib/actor/types.js';\nimport type {\n CrawleeOneRoute,\n CrawleeOneRouteHandler,\n CrawleeOneRouteMatcher,\n CrawleeOneRouteMatcherFn,\n CrawleeOneRouteWrapper,\n} from '../../lib/router/types.js';\nimport type { MaybePromise } from '../../utils/types.js';\nimport type { CrawleeOneArgs, crawleeOne } from '../../api.js';\n\nconst makeUnion = (items: string[]) => items.map((s) => `\"${s}\"`).join(` | `);\nconst makeEnum = (items: string[]) =>\n '{\\n' + items.map((s) => ` '${s}' = '${s}'`).join(`,\\n`) + '\\n}';\n\nconst formatters = {\n type: (name: string, value: string, typeArgs?: string[]) => {\n const typeArgsStr = typeArgs?.length ? `<${typeArgs.join(', ')}>` : '';\n return `export type ${name}${typeArgsStr} = ${value};`;\n },\n typeFunc: (name: string, value: string, typeArgs?: string[]) => {\n const typeArgsStr = typeArgs?.length ? `<${typeArgs.join(', ')}>` : '';\n return `export type ${name} = ${typeArgsStr}${value};`;\n },\n func: (name: string, value: string, typeArgs?: string[]) => {\n const typeArgsStr = typeArgs?.length ? `<${typeArgs.join(', ')}>` : '';\n return `export const ${name} = ${typeArgsStr}${value};`;\n },\n // enum `CrawlerName`LabelEnum { \"detailPage\" = \"detailPage\", \"otherLabel\" = \"otherLabel\", ... };\n enum: (name: string, value: string, typeArgs?: string[]) => {\n return `export enum ${name} ${value}`;\n },\n} satisfies Record<string, (name: string, value: string, args?: string[]) => string>;\n\nconst parseTypesFromSchema = (schema: CrawleeOneConfigSchema) => {\n /** Remember which types we've already generated */\n const definitions: Record<string, string> = {};\n\n /** Remember what values need to be imported and from where */\n const imports: Record<string, Set<{ name: string; typeOnly?: boolean }>> = {};\n\n const addImports = <T extends string>(\n pkg: string,\n newEntries: T[],\n options?: { typeOnly?: boolean }\n ) => {\n const { typeOnly } = options ?? {};\n const entries = (imports[pkg] = imports[pkg] || new Set());\n newEntries.forEach((name) => entries.add({ name, typeOnly }));\n // Return the entries as variables, so we can define them in a single\n // place but still reference them in code.\n return newEntries.reduce<{ [Key in T]: Key }>((agg, key) => {\n agg[key] = key;\n return agg;\n }, {} as any);\n };\n\n const define = (\n key: string,\n value: string | (() => string),\n options?: { typeArgs?: string[]; kind?: keyof typeof formatters }\n ) => {\n const kind = options?.kind ?? 'type';\n const typeArgs = options?.typeArgs ?? [];\n if (!definitions[key]) {\n const resolvedVal = typeof value === 'function' ? value() : value;\n const formatter = formatters[kind];\n const valFormatted = formatter(key, resolvedVal, typeArgs);\n definitions[key] = valFormatted;\n }\n // Return the key as variable, so we can reference it\n return key;\n };\n\n // 1. Define imports\n const {\n AllActorInputs: actorInput,\n CrawleeOneActorRouterCtx: actorRouterCtx,\n CrawleeOneActorInst: actorCtx,\n CrawleeOneRoute: routeType,\n CrawleeOneRouteHandler: routeHandler,\n CrawleeOneRouteWrapper: routeWrapper,\n CrawleeOneRouteMatcher: routeMatcher,\n CrawleeOneRouteMatcherFn: routeMatcherFn,\n CrawleeOneIO: ioType,\n CrawleeOneTelemetry: telemType,\n CrawleeOneCtx: ctxType,\n CrawleeOneArgs: argsType,\n crawleeOne: crawleeOneFn,\n } = addImports('crawlee-one', [\n 'AllActorInputs',\n 'CrawleeOneActorRouterCtx',\n 'CrawleeOneActorInst',\n 'CrawleeOneRoute',\n 'CrawleeOneRouteHandler',\n 'CrawleeOneRouteWrapper',\n 'CrawleeOneRouteMatcher',\n 'CrawleeOneRouteMatcherFn',\n 'CrawleeOneIO',\n 'CrawleeOneTelemetry',\n 'CrawleeOneCtx',\n 'CrawleeOneArgs',\n 'crawleeOne',\n ]);\n addImports('crawlee', Object.values(crawlingContextNameByType), { typeOnly: true });\n\n // 2. Define utils\n const maybeP = define('MaybePromise', 'T | Promise<T>', { typeArgs: ['T'] });\n\n Object.entries(schema.crawlers).forEach(([crawlerName, crawler]) => {\n const crawlerType = crawler.type;\n\n // 2. Get `CrawlingContext`, e.g. 'cheerio' => `CheerioCrawlingContext`;\n const crawlingContextTypeName = crawlingContextNameByType[crawlerType];\n\n // 3. Generate type for route labels\n // type `CrawlerName`Label = \"detailPage\" | \"otherLabel\" | ...;\n const labelKey = define(`${crawlerName}Label`, () => makeUnion(crawler.routes));\n\n // enum `CrawlerName`LabelEnum { \"detailPage\" = \"detailPage\", \"otherLabel\" = \"otherLabel\", ... };\n const labelEnumKey = define(`${crawlerName}LabelEnum`, () => makeEnum(crawler.routes), {\n kind: 'enum',\n });\n\n const ctxTypeArgs = [\n `TInput extends Record<string, any> = ${actorInput}`,\n `TIO extends ${ioType} = ${ioType}`,\n `Telem extends ${telemType}<any, any> = ${telemType}<any, any>`,\n ];\n\n // 4. Create CrawleeOne context\n // type `CrawlerName`Ctx = <TIO, Telem>CrawleeOneCtx<CheerioCrawlingContext, `CrawlerName`Label, AllActorInputs, TIO, Telem>\n const ctxKey = define(\n `${crawlerName}Ctx`,\n `${ctxType}<${crawlingContextTypeName}, ${labelKey}, TInput, TIO, Telem>`,\n { typeArgs: ctxTypeArgs }\n );\n\n // 5. Create CrawleeOne instance\n // const customCrawler = <TIO, Telem>(args: CrawleeOneArgs<TType, T>) => crawleeOne(args);\n const crawlerKey = define(\n `${crawlerName}Crawler`,\n `(args: Omit<${argsType}<\"${crawlerType}\", ${ctxKey}<TInput, TIO, Telem>>, 'type'>) => ${crawleeOneFn}<\"${crawlerType}\", ${ctxKey}<TInput, TIO, Telem>>({ ...args, type: \"${crawlerType}\"});`,\n { kind: 'func', typeArgs: ctxTypeArgs }\n );\n\n // 6. Get actor router context (`CrawleeOneActorRouterCtx`)\n // NOTE: We use `AllActorInput` for the Actor input, because this type definition\n // will be used by developers.\n const routerCtxKey = define(\n `${crawlerName}RouterContext`,\n `${actorRouterCtx}<${ctxKey}<TInput, TIO, Telem>>`,\n { typeArgs: ctxTypeArgs }\n );\n\n // 7. Get actor context (`CrawleeOneActorInst`)\n // NOTE: We use `AllActorInput` for the Actor input, because this type definition\n // will be used by developers.\n const actorCtxKey = define(\n `${crawlerName}ActorCtx`,\n `${actorCtx}<${ctxKey}<TInput, TIO, Telem>>`,\n { typeArgs: ctxTypeArgs }\n );\n\n // 8. Create Route types\n // E.g. `type `crawlerName`Route = CrawleeOneRout<`CrawlerName`Ctx>`\n const routeKey = define(\n `${crawlerName}Route`,\n `${routeType}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,\n {\n typeArgs: ctxTypeArgs,\n }\n );\n\n // E.g. `type `crawlerName`RouteHandler = CrawleeOneRouteHandler<`CrawlerName`Ctx, CrawlerName`ActorRouterCtx>`\n const routeHandlerValue = `${routeHandler}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`;\n const routeHandlerKey = define(`${crawlerName}RouteHandler`, routeHandlerValue, {\n typeArgs: ctxTypeArgs,\n });\n\n // E.g. `type `crawlerName`RouteWrapper = CrawleeOneRouteWrapper<`CrawlerName`Ctx, CrawlerName`ActorRouterCtx>`\n const routeWrapperKey = define(\n `${crawlerName}RouteWrapper`,\n `${routeWrapper}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,\n { typeArgs: ctxTypeArgs }\n );\n\n // E.g. `type `crawlerName`Matcher = CrawleeOneRouteMatcher<`CrawlerName`Ctx, CrawlerName`ActorRouterCtx>`\n const routeMatcherKey = define(\n `${crawlerName}RouteMatcher`,\n `${routeMatcher}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,\n { typeArgs: ctxTypeArgs }\n );\n\n // E.g. `type `crawlerName`Matcher = CrawleeOneRouteMatcher<`CrawlerName`Ctx, CrawlerName`ActorRouterCtx>`\n const routeMatcherFnKey = define(\n `${crawlerName}RouteMatcherFn`,\n `${routeMatcherFn}<${ctxKey}<TInput, TIO, Telem>, ${routerCtxKey}<TInput, TIO, Telem>>`,\n { typeArgs: ctxTypeArgs }\n );\n\n // 9. Create Crawler hooks\n\n // NOTE: Type for before/after handler is the same as for handlers\n // E.g. `type `CrawlerName`OnBeforeHandler = CrawleeOneRouteHandler<CheerioCrawlingContext, ProfesiaRouterContext>`\n // E.g. `type `CrawlerName`OnAfterHandler = CrawleeOneRouteHandler<CheerioCrawlingContext, ProfesiaRouterContext>`\n const onBeforeHandlerKey = define(`${crawlerName}OnBeforeHandler`, routeHandlerValue, {\n typeArgs: ctxTypeArgs,\n });\n const onAfterHandlerKey = define(`${crawlerName}OnAfterHandler`, routeHandlerValue, {\n typeArgs: ctxTypeArgs,\n });\n\n // type `CrawlerName`OnReady = <TIO, Telem>(actor: CrawleeOneActorInst<`CrawlerName`Label, AllActorInputs, TIO, Telem, `type`CrawlingContext>) => MaybePromise<void>;\n const onReadyKey = define(\n `${crawlerName}OnReady`,\n `(actor: ${actorCtxKey}<TInput, TIO, Telem>) => ${maybeP}<void>;`,\n { kind: 'typeFunc', typeArgs: ctxTypeArgs }\n );\n });\n\n const finalImports = Object.entries(imports).reduce<Record<string, string>>(\n (agg, [pkg, entriesSet]) => {\n const entries = [...entriesSet.values()];\n const formattedEntries = entries.map((e) => e.name).join(', ');\n const typeStr = entries.every((e) => e.typeOnly) ? 'type ' : '';\n agg[pkg] = `import ${typeStr}{ ${formattedEntries} } from \"${pkg}\"`;\n return agg;\n },\n {}\n );\n\n return { imports: finalImports, definitions };\n};\n\n/**\n * Generate types for CrawleeOne given a config.\n *\n * Config can be passed directly, or as the path to the config file.\n * If the config is omitted, it is automatically searched for using CosmicConfig.\n */\nexport const generateTypes = async (outfile: string, configOrPath?: CrawleeOneConfig | string) => {\n const config =\n !configOrPath || typeof configOrPath === 'string'\n ? await loadConfig(configOrPath)\n : configOrPath;\n validateConfig(config);\n\n const { imports, definitions } = await parseTypesFromSchema(config!.schema);\n const fileContent =\n Object.values(imports).join('\\n') + '\\n\\n\\n' + Object.values(definitions).join('\\n\\n');\n\n const outdir = path.dirname(outfile);\n await fsp.mkdir(outdir, { recursive: true });\n await fsp.writeFile(outfile, fileContent, 'utf-8');\n\n console.log(`Done generating types to ${outfile}`);\n};\n","import {\n AdaptivePlaywrightCrawler,\n BasicCrawler,\n type BasicCrawlingContext,\n CheerioCrawler,\n type CheerioCrawlingContext,\n HttpCrawler,\n type HttpCrawlingContext,\n JSDOMCrawler,\n type JSDOMCrawlingContext,\n PlaywrightCrawler,\n type PlaywrightCrawlingContext,\n PuppeteerCrawler,\n type PuppeteerCrawlingContext,\n} from 'crawlee';\n\nimport type { CrawlerType } from './types/index.js';\n\nexport const actorClassByType = {\n basic: BasicCrawler,\n http: HttpCrawler,\n cheerio: CheerioCrawler,\n jsdom: JSDOMCrawler,\n playwright: PlaywrightCrawler,\n 'adaptive-playwright': AdaptivePlaywrightCrawler,\n puppeteer: PuppeteerCrawler,\n} as const satisfies Record<CrawlerType, { new (options: Record<string, any>): any }>;\n\ntype CrawlingContextName = keyof {\n BasicCrawlingContext: BasicCrawlingContext;\n HttpCrawlingContext: HttpCrawlingContext;\n CheerioCrawlingContext: CheerioCrawlingContext;\n JSDOMCrawlingContext: JSDOMCrawlingContext;\n PlaywrightCrawlingContext: PlaywrightCrawlingContext;\n PuppeteerCrawlingContext: PuppeteerCrawlingContext;\n};\n\nexport const crawlingContextNameByType = {\n basic: 'BasicCrawlingContext',\n http: 'HttpCrawlingContext',\n cheerio: 'CheerioCrawlingContext',\n jsdom: 'JSDOMCrawlingContext',\n playwright: 'PlaywrightCrawlingContext',\n 'adaptive-playwright': 'PlaywrightCrawlingContext',\n puppeteer: 'PuppeteerCrawlingContext',\n} satisfies Record<CrawlerType, CrawlingContextName>;\n"],"mappings":";AAAA,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,YAAY;AACrB,SAAS,mBAAmB;AAIrB,IAAM,sBAAsB,CAAC;AAAA,EAClC;AAAA,EACA,YAAY;AAAA,EACZ,YAAY;AACd,MAMM;AACJ,QAAM,kCAAkC,OAAO,YAAqC;AAElF,UAAM,QAAQ,MAAM,KAAK,KAAK,KAAK,eAAe,GAAG,OAAO,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC;AACxF,QAAI,CAAC,MAAM,QAAQ;AACjB,YAAM,MAAM,sCAAsC,OAAO,GAAG;AAAA,IAC9D;AACA,QAAI,MAAM,SAAS,GAAG;AACpB,YAAM;AAAA,QACJ,yCAAyC,OAAO;AAAA,MAClD;AAAA,IACF;AACA,WAAO,MAAM,CAAC;AAAA,EAChB;AAEA,QAAM,QAAQ,MAAM;AAClB,WAAO,OAAO;AACd,UAAM,SAAS,IAAI,YAAY;AAAA,MAC7B,OAAO,QAAQ,IAAI;AAAA,IACrB,CAAC;AACD,WAAO,EAAE,OAAO;AAAA,EAClB;AAEA,QAAM,UAAU,OAAO,YAAoB;AACzC,UAAM,UAAU,MAAM,gCAAgC,OAAO;AAC7D,UAAM,EAAE,OAAO,IAAI,MAAM;AACzB,UAAM,EAAE,SAAAA,SAAQ,KAAM,MAAM,OAAO,UAAqC;AACxE,UAAMA,SAAQ,MAAM;AAAA,EACtB;AAEA,QAAM,YAAY,OAAO,YAAoB;AAC3C,UAAM,UAAU,MAAM,gCAAgC,OAAO;AAC7D,UAAM,EAAE,OAAO,IAAI,MAAM;AACzB,UAAM,EAAE,WAAAC,WAAU,KAAM,MAAM,OAAO,UAAqC;AAC1E,UAAMA,WAAU,MAAM;AAAA,EACxB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;;;AC1DA,SAAS,mBAAmB;AAC5B,OAAO,SAAS;;;ACDhB,SAAS,oBAAoB;AAC7B,SAAS,qBAAqB;AAC9B,OAAOC,WAAU;AAMV,IAAM,qBAAqB,CAChC,eACA,WACyB;AACzB,MAAI,MAAMA,MAAK,QAAQ,cAAc,aAAa,CAAC;AACnD,SAAO,MAAM;AACX,QAAI;AACF,YAAM,MAAM,KAAK,MAAM,aAAaA,MAAK,KAAK,KAAK,cAAc,GAAG,OAAO,CAAC;AAC5E,YAAM,SAAS,CAAC;AAChB,iBAAW,SAAS,OAAQ,QAAO,KAAK,IAAI,IAAI,KAAK;AACrD,aAAO;AAAA,IACT,QAAQ;AACN,YAAM,SAASA,MAAK,QAAQ,GAAG;AAC/B,UAAI,WAAW,IAAK,OAAM,IAAI,MAAM,wBAAwB;AAC5D,YAAM;AAAA,IACR;AAAA,EACF;AACF;;;ACGO,IAAM,eAAe;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;;;AFxBA,IAAM,iBAAiB;AAEvB,IAAM,+BAA+B,IAAI,OAAO;AAAA,EAC9C,MAAM,IAAI,OAAO,EAAE,MAAM,GAAG,YAAY,EAAE,SAAS;AAAA,EACnD,QAAQ,IAAI,MAAM,EAAE,MAAM,IAAI,OAAO,EAAE,SAAS,CAAC,EAAE,SAAS;AAC9D,CAAmE,EAChE,SAAS,EAAE,QAAQ,KAAK;AAE3B,IAAM,wBAAwB,IAAI,OAAO;AAAA,EACvC,UAAU,IAAI,OAAO,EAClB,QAAQ,IAAI,OAAO,EAAE,QAAQ,cAAc,GAAG,4BAA4B,EAC1E,SAAS,EAAE,QAAQ,KAAK;AAC7B,CAA4D,EACzD,SAAS,EAAE,QAAQ,KAAK;AAE3B,IAAM,kBAAkB,IAAI,OAAO;AAAA,EACjC,SAAS,IAAI,OAAO,EAAE,MAAM,CAAC,EAAE,SAAS;AAAA,EACxC,QAAQ;AACV,CAAsD,EACnD,SAAS,EAAE,QAAQ,KAAK;AAQpB,IAAM,iBAAiB,CAAC,WAA6B;AAC1D,MAAI,OAAO,QAAQ,eAAe;AACpC;AASO,IAAM,aAAa,OAAO,mBAA4B;AAC3D,QAAM,UAAU,mBAAmB,YAAY,KAAK,CAAC,MAAM,CAAC;AAG5D,QAAM,WAAW,YAAY,QAAQ,IAAI;AACzC,QAAM,SAAS,iBAAiB,MAAM,SAAS,KAAK,cAAc,IAAI,MAAM,SAAS,OAAO;AAC5F,QAAM,SAAU,QAAQ,UAAU;AAClC,SAAO;AACT;;;AGzDA,OAAO,SAAS;AAChB,OAAOC,WAAU;;;ACFjB;AAAA,EACE;AAAA,EACA;AAAA,EAEA;AAAA,EAEA;AAAA,EAEA;AAAA,EAEA;AAAA,EAEA;AAAA,OAEK;AAIA,IAAM,mBAAmB;AAAA,EAC9B,OAAO;AAAA,EACP,MAAM;AAAA,EACN,SAAS;AAAA,EACT,OAAO;AAAA,EACP,YAAY;AAAA,EACZ,uBAAuB;AAAA,EACvB,WAAW;AACb;AAWO,IAAM,4BAA4B;AAAA,EACvC,OAAO;AAAA,EACP,MAAM;AAAA,EACN,SAAS;AAAA,EACT,OAAO;AAAA,EACP,YAAY;AAAA,EACZ,uBAAuB;AAAA,EACvB,WAAW;AACb;;;ADzBA,IAAM,YAAY,CAAC,UAAoB,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,KAAK;AAC5E,IAAM,WAAW,CAAC,UAChB,QAAQ,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK;AAAA,CAAK,IAAI;AAE9D,IAAM,aAAa;AAAA,EACjB,MAAM,CAAC,MAAc,OAAe,aAAwB;AAC1D,UAAM,cAAc,UAAU,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,MAAM;AACpE,WAAO,eAAe,IAAI,GAAG,WAAW,MAAM,KAAK;AAAA,EACrD;AAAA,EACA,UAAU,CAAC,MAAc,OAAe,aAAwB;AAC9D,UAAM,cAAc,UAAU,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,MAAM;AACpE,WAAO,eAAe,IAAI,MAAM,WAAW,GAAG,KAAK;AAAA,EACrD;AAAA,EACA,MAAM,CAAC,MAAc,OAAe,aAAwB;AAC1D,UAAM,cAAc,UAAU,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,MAAM;AACpE,WAAO,gBAAgB,IAAI,MAAM,WAAW,GAAG,KAAK;AAAA,EACtD;AAAA;AAAA,EAEA,MAAM,CAAC,MAAc,OAAe,aAAwB;AAC1D,WAAO,eAAe,IAAI,IAAI,KAAK;AAAA,EACrC;AACF;AAEA,IAAM,uBAAuB,CAAC,WAAmC;AAE/D,QAAM,cAAsC,CAAC;AAG7C,QAAM,UAAqE,CAAC;AAE5E,QAAM,aAAa,CACjB,KACA,YACA,YACG;AACH,UAAM,EAAE,SAAS,IAAI,WAAW,CAAC;AACjC,UAAM,UAAW,QAAQ,GAAG,IAAI,QAAQ,GAAG,KAAK,oBAAI,IAAI;AACxD,eAAW,QAAQ,CAAC,SAAS,QAAQ,IAAI,EAAE,MAAM,SAAS,CAAC,CAAC;AAG5D,WAAO,WAAW,OAA4B,CAAC,KAAK,QAAQ;AAC1D,UAAI,GAAG,IAAI;AACX,aAAO;AAAA,IACT,GAAG,CAAC,CAAQ;AAAA,EACd;AAEA,QAAM,SAAS,CACb,KACA,OACA,YACG;AACH,UAAM,OAAO,SAAS,QAAQ;AAC9B,UAAM,WAAW,SAAS,YAAY,CAAC;AACvC,QAAI,CAAC,YAAY,GAAG,GAAG;AACrB,YAAM,cAAc,OAAO,UAAU,aAAa,MAAM,IAAI;AAC5D,YAAM,YAAY,WAAW,IAAI;AACjC,YAAM,eAAe,UAAU,KAAK,aAAa,QAAQ;AACzD,kBAAY,GAAG,IAAI;AAAA,IACrB;AAEA,WAAO;AAAA,EACT;AAGA,QAAM;AAAA,IACJ,gBAAgB;AAAA,IAChB,0BAA0B;AAAA,IAC1B,qBAAqB;AAAA,IACrB,iBAAiB;AAAA,IACjB,wBAAwB;AAAA,IACxB,wBAAwB;AAAA,IACxB,wBAAwB;AAAA,IACxB,0BAA0B;AAAA,IAC1B,cAAc;AAAA,IACd,qBAAqB;AAAA,IACrB,eAAe;AAAA,IACf,gBAAgB;AAAA,IAChB,YAAY;AAAA,EACd,IAAI,WAAW,eAAe;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,aAAW,WAAW,OAAO,OAAO,yBAAyB,GAAG,EAAE,UAAU,KAAK,CAAC;AAGlF,QAAM,SAAS,OAAO,gBAAgB,kBAAkB,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC;AAE3E,SAAO,QAAQ,OAAO,QAAQ,EAAE,QAAQ,CAAC,CAAC,aAAa,OAAO,MAAM;AAClE,UAAM,cAAc,QAAQ;AAG5B,UAAM,0BAA0B,0BAA0B,WAAW;AAIrE,UAAM,WAAW,OAAO,GAAG,WAAW,SAAS,MAAM,UAAU,QAAQ,MAAM,CAAC;AAG9E,UAAM,eAAe,OAAO,GAAG,WAAW,aAAa,MAAM,SAAS,QAAQ,MAAM,GAAG;AAAA,MACrF,MAAM;AAAA,IACR,CAAC;AAED,UAAM,cAAc;AAAA,MAClB,wCAAwC,UAAU;AAAA,MAClD,eAAe,MAAM,MAAM,MAAM;AAAA,MACjC,iBAAiB,SAAS,gBAAgB,SAAS;AAAA,IACrD;AAIA,UAAM,SAAS;AAAA,MACb,GAAG,WAAW;AAAA,MACd,GAAG,OAAO,IAAI,uBAAuB,KAAK,QAAQ;AAAA,MAClD,EAAE,UAAU,YAAY;AAAA,IAC1B;AAIA,UAAM,aAAa;AAAA,MACjB,GAAG,WAAW;AAAA,MACd,eAAe,QAAQ,KAAK,WAAW,MAAM,MAAM,sCAAsC,YAAY,KAAK,WAAW,MAAM,MAAM,2CAA2C,WAAW;AAAA,MACvL,EAAE,MAAM,QAAQ,UAAU,YAAY;AAAA,IACxC;AAKA,UAAM,eAAe;AAAA,MACnB,GAAG,WAAW;AAAA,MACd,GAAG,cAAc,IAAI,MAAM;AAAA,MAC3B,EAAE,UAAU,YAAY;AAAA,IAC1B;AAKA,UAAM,cAAc;AAAA,MAClB,GAAG,WAAW;AAAA,MACd,GAAG,QAAQ,IAAI,MAAM;AAAA,MACrB,EAAE,UAAU,YAAY;AAAA,IAC1B;AAIA,UAAM,WAAW;AAAA,MACf,GAAG,WAAW;AAAA,MACd,GAAG,SAAS,IAAI,MAAM,yBAAyB,YAAY;AAAA,MAC3D;AAAA,QACE,UAAU;AAAA,MACZ;AAAA,IACF;AAGA,UAAM,oBAAoB,GAAG,YAAY,IAAI,MAAM,yBAAyB,YAAY;AACxF,UAAM,kBAAkB,OAAO,GAAG,WAAW,gBAAgB,mBAAmB;AAAA,MAC9E,UAAU;AAAA,IACZ,CAAC;AAGD,UAAM,kBAAkB;AAAA,MACtB,GAAG,WAAW;AAAA,MACd,GAAG,YAAY,IAAI,MAAM,yBAAyB,YAAY;AAAA,MAC9D,EAAE,UAAU,YAAY;AAAA,IAC1B;AAGA,UAAM,kBAAkB;AAAA,MACtB,GAAG,WAAW;AAAA,MACd,GAAG,YAAY,IAAI,MAAM,yBAAyB,YAAY;AAAA,MAC9D,EAAE,UAAU,YAAY;AAAA,IAC1B;AAGA,UAAM,oBAAoB;AAAA,MACxB,GAAG,WAAW;AAAA,MACd,GAAG,cAAc,IAAI,MAAM,yBAAyB,YAAY;AAAA,MAChE,EAAE,UAAU,YAAY;AAAA,IAC1B;AAOA,UAAM,qBAAqB,OAAO,GAAG,WAAW,mBAAmB,mBAAmB;AAAA,MACpF,UAAU;AAAA,IACZ,CAAC;AACD,UAAM,oBAAoB,OAAO,GAAG,WAAW,kBAAkB,mBAAmB;AAAA,MAClF,UAAU;AAAA,IACZ,CAAC;AAGD,UAAM,aAAa;AAAA,MACjB,GAAG,WAAW;AAAA,MACd,WAAW,WAAW,4BAA4B,MAAM;AAAA,MACxD,EAAE,MAAM,YAAY,UAAU,YAAY;AAAA,IAC5C;AAAA,EACF,CAAC;AAED,QAAM,eAAe,OAAO,QAAQ,OAAO,EAAE;AAAA,IAC3C,CAAC,KAAK,CAAC,KAAK,UAAU,MAAM;AAC1B,YAAM,UAAU,CAAC,GAAG,WAAW,OAAO,CAAC;AACvC,YAAM,mBAAmB,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,IAAI;AAC7D,YAAM,UAAU,QAAQ,MAAM,CAAC,MAAM,EAAE,QAAQ,IAAI,UAAU;AAC7D,UAAI,GAAG,IAAI,UAAU,OAAO,KAAK,gBAAgB,YAAY,GAAG;AAChE,aAAO;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EACH;AAEA,SAAO,EAAE,SAAS,cAAc,YAAY;AAC9C;AAQO,IAAM,gBAAgB,OAAO,SAAiB,iBAA6C;AAChG,QAAM,SACJ,CAAC,gBAAgB,OAAO,iBAAiB,WACrC,MAAM,WAAW,YAAY,IAC7B;AACN,iBAAe,MAAM;AAErB,QAAM,EAAE,SAAS,YAAY,IAAI,MAAM,qBAAqB,OAAQ,MAAM;AAC1E,QAAM,cACJ,OAAO,OAAO,OAAO,EAAE,KAAK,IAAI,IAAI,WAAW,OAAO,OAAO,WAAW,EAAE,KAAK,MAAM;AAEvF,QAAM,SAASC,MAAK,QAAQ,OAAO;AACnC,QAAM,IAAI,MAAM,QAAQ,EAAE,WAAW,KAAK,CAAC;AAC3C,QAAM,IAAI,UAAU,SAAS,aAAa,OAAO;AAEjD,UAAQ,IAAI,4BAA4B,OAAO,EAAE;AACnD;","names":["migrate","unmigrate","path","path","path"]}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// src/lib/telemetry/sentry.ts
|
|
2
|
+
import * as Sentry from "@sentry/node";
|
|
3
|
+
var setupSentry = async (sentryOptions) => {
|
|
4
|
+
const enabled = sentryOptions?.enabled != null && sentryOptions.enabled;
|
|
5
|
+
if (!enabled) return;
|
|
6
|
+
if (!sentryOptions?.serverName) throw Error('Sentry setup is missing "serverName" property.');
|
|
7
|
+
Sentry.init(sentryOptions);
|
|
8
|
+
};
|
|
9
|
+
var createSentryTelemetry = (sentryOptions) => {
|
|
10
|
+
return {
|
|
11
|
+
setup: async (_actor) => {
|
|
12
|
+
await setupSentry(sentryOptions);
|
|
13
|
+
},
|
|
14
|
+
onSendErrorToTelemetry: (error, report, _options, _ctx) => {
|
|
15
|
+
Sentry.captureException(error, { extra: report });
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export {
|
|
21
|
+
createSentryTelemetry
|
|
22
|
+
};
|
|
23
|
+
//# sourceMappingURL=chunk-V524BXEG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/telemetry/sentry.ts"],"sourcesContent":["import * as Sentry from '@sentry/node';\n\nimport type { CrawleeOneTelemetry } from './types.js';\nimport type { CrawleeOneCtx } from '../actor/types.js';\n\n/**\n * Sentry configuration common to all crawlers.\n *\n * By default, sentry is enabled only on the server.\n * In Apify, whis is when `process.env.APIFY_IS_AT_HOME` is true.\n */\nconst setupSentry = async (sentryOptions?: Sentry.NodeOptions) => {\n // As default, enable sentry only on Apify server\n const enabled = sentryOptions?.enabled != null && sentryOptions.enabled;\n\n if (!enabled) return;\n\n // We use this field for identification in UI, so it's required.\n if (!sentryOptions?.serverName) throw Error('Sentry setup is missing \"serverName\" property.'); // prettier-ignore\n\n Sentry.init(sentryOptions);\n};\n\nexport const createSentryTelemetry = <T extends CrawleeOneTelemetry<CrawleeOneCtx>>(\n sentryOptions?: Sentry.NodeOptions\n) => {\n return {\n setup: async (_actor) => {\n await setupSentry(sentryOptions);\n },\n onSendErrorToTelemetry: (error, report, _options, _ctx) => {\n Sentry.captureException(error, { extra: report as any });\n },\n } as T;\n};\n"],"mappings":";AAAA,YAAY,YAAY;AAWxB,IAAM,cAAc,OAAO,kBAAuC;AAEhE,QAAM,UAAU,eAAe,WAAW,QAAQ,cAAc;AAEhE,MAAI,CAAC,QAAS;AAGd,MAAI,CAAC,eAAe,WAAY,OAAM,MAAM,gDAAgD;AAE5F,EAAO,YAAK,aAAa;AAC3B;AAEO,IAAM,wBAAwB,CACnC,kBACG;AACH,SAAO;AAAA,IACL,OAAO,OAAO,WAAW;AACvB,YAAM,YAAY,aAAa;AAAA,IACjC;AAAA,IACA,wBAAwB,CAAC,OAAO,QAAQ,UAAU,SAAS;AACzD,MAAO,wBAAiB,OAAO,EAAE,OAAO,OAAc,CAAC;AAAA,IACzD;AAAA,EACF;AACF;","names":[]}
|