crawlee-one 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +81 -0
  2. package/dist/cjs/cli/cli.d.ts +1 -0
  3. package/dist/cjs/cli/cli.js +61 -0
  4. package/dist/cjs/cli/cli.js.map +1 -0
  5. package/dist/cjs/cli/index.d.ts +2 -0
  6. package/dist/cjs/cli/index.js +6 -0
  7. package/dist/cjs/cli/index.js.map +1 -0
  8. package/dist/cjs/index.d.ts +24 -0
  9. package/dist/cjs/index.js +43 -0
  10. package/dist/cjs/index.js.map +1 -0
  11. package/dist/cjs/lib/actions/dom.d.ts +102 -0
  12. package/dist/cjs/lib/actions/dom.js +743 -0
  13. package/dist/cjs/lib/actions/dom.js.map +1 -0
  14. package/dist/cjs/lib/actions/domUtils.d.ts +42 -0
  15. package/dist/cjs/lib/actions/domUtils.js +126 -0
  16. package/dist/cjs/lib/actions/domUtils.js.map +1 -0
  17. package/dist/cjs/lib/actions/page.d.ts +69 -0
  18. package/dist/cjs/lib/actions/page.js +205 -0
  19. package/dist/cjs/lib/actions/page.js.map +1 -0
  20. package/dist/cjs/lib/actions/scrapeListing.d.ts +78 -0
  21. package/dist/cjs/lib/actions/scrapeListing.js +242 -0
  22. package/dist/cjs/lib/actions/scrapeListing.js.map +1 -0
  23. package/dist/cjs/lib/actor/actor.d.ts +90 -0
  24. package/dist/cjs/lib/actor/actor.js +306 -0
  25. package/dist/cjs/lib/actor/actor.js.map +1 -0
  26. package/dist/cjs/lib/actor/types.d.ts +162 -0
  27. package/dist/cjs/lib/actor/types.js +3 -0
  28. package/dist/cjs/lib/actor/types.js.map +1 -0
  29. package/dist/cjs/lib/actor.d.ts +189 -0
  30. package/dist/cjs/lib/actor.js +225 -0
  31. package/dist/cjs/lib/actor.js.map +1 -0
  32. package/dist/cjs/lib/actorSpec.d.ts +20 -0
  33. package/dist/cjs/lib/actorSpec.js +3 -0
  34. package/dist/cjs/lib/actorSpec.js.map +1 -0
  35. package/dist/cjs/lib/config.d.ts +561 -0
  36. package/dist/cjs/lib/config.js +707 -0
  37. package/dist/cjs/lib/config.js.map +1 -0
  38. package/dist/cjs/lib/dataset/maxCount.d.ts +30 -0
  39. package/dist/cjs/lib/dataset/maxCount.js +55 -0
  40. package/dist/cjs/lib/dataset/maxCount.js.map +1 -0
  41. package/dist/cjs/lib/dataset/pushData.d.ts +123 -0
  42. package/dist/cjs/lib/dataset/pushData.js +182 -0
  43. package/dist/cjs/lib/dataset/pushData.js.map +1 -0
  44. package/dist/cjs/lib/dataset.d.ts +98 -0
  45. package/dist/cjs/lib/dataset.js +122 -0
  46. package/dist/cjs/lib/dataset.js.map +1 -0
  47. package/dist/cjs/lib/dom.d.ts +78 -0
  48. package/dist/cjs/lib/dom.js +243 -0
  49. package/dist/cjs/lib/dom.js.map +1 -0
  50. package/dist/cjs/lib/error/errorHandler.d.ts +112 -0
  51. package/dist/cjs/lib/error/errorHandler.js +164 -0
  52. package/dist/cjs/lib/error/errorHandler.js.map +1 -0
  53. package/dist/cjs/lib/error/sentry.d.ts +11 -0
  54. package/dist/cjs/lib/error/sentry.js +60 -0
  55. package/dist/cjs/lib/error/sentry.js.map +1 -0
  56. package/dist/cjs/lib/integrations/apify.d.ts +67 -0
  57. package/dist/cjs/lib/integrations/apify.js +106 -0
  58. package/dist/cjs/lib/integrations/apify.js.map +1 -0
  59. package/dist/cjs/lib/integrations/types.d.ts +274 -0
  60. package/dist/cjs/lib/integrations/types.js +3 -0
  61. package/dist/cjs/lib/integrations/types.js.map +1 -0
  62. package/dist/cjs/lib/io/dataset.d.ts +67 -0
  63. package/dist/cjs/lib/io/dataset.js +86 -0
  64. package/dist/cjs/lib/io/dataset.js.map +1 -0
  65. package/dist/cjs/lib/io/maxCount.d.ts +30 -0
  66. package/dist/cjs/lib/io/maxCount.js +55 -0
  67. package/dist/cjs/lib/io/maxCount.js.map +1 -0
  68. package/dist/cjs/lib/io/pushData.d.ts +124 -0
  69. package/dist/cjs/lib/io/pushData.js +193 -0
  70. package/dist/cjs/lib/io/pushData.js.map +1 -0
  71. package/dist/cjs/lib/io/pushRequests.d.ts +38 -0
  72. package/dist/cjs/lib/io/pushRequests.js +63 -0
  73. package/dist/cjs/lib/io/pushRequests.js.map +1 -0
  74. package/dist/cjs/lib/io/requestQueue.d.ts +28 -0
  75. package/dist/cjs/lib/io/requestQueue.js +40 -0
  76. package/dist/cjs/lib/io/requestQueue.js.map +1 -0
  77. package/dist/cjs/lib/log.d.ts +38 -0
  78. package/dist/cjs/lib/log.js +54 -0
  79. package/dist/cjs/lib/log.js.map +1 -0
  80. package/dist/cjs/lib/migrate/localMigrator.d.ts +10 -0
  81. package/dist/cjs/lib/migrate/localMigrator.js +57 -0
  82. package/dist/cjs/lib/migrate/localMigrator.js.map +1 -0
  83. package/dist/cjs/lib/migrate/localState.d.ts +7 -0
  84. package/dist/cjs/lib/migrate/localState.js +43 -0
  85. package/dist/cjs/lib/migrate/localState.js.map +1 -0
  86. package/dist/cjs/lib/migrate/types.d.ts +6 -0
  87. package/dist/cjs/lib/migrate/types.js +3 -0
  88. package/dist/cjs/lib/migrate/types.js.map +1 -0
  89. package/dist/cjs/lib/readme/readme.d.ts +65 -0
  90. package/dist/cjs/lib/readme/readme.js +534 -0
  91. package/dist/cjs/lib/readme/readme.js.map +1 -0
  92. package/dist/cjs/lib/readme/types.d.ts +260 -0
  93. package/dist/cjs/lib/readme/types.js +54 -0
  94. package/dist/cjs/lib/readme/types.js.map +1 -0
  95. package/dist/cjs/lib/router.d.ts +132 -0
  96. package/dist/cjs/lib/router.js +165 -0
  97. package/dist/cjs/lib/router.js.map +1 -0
  98. package/dist/cjs/lib/scraper/scrapeListing.d.ts +78 -0
  99. package/dist/cjs/lib/scraper/scrapeListing.js +242 -0
  100. package/dist/cjs/lib/scraper/scrapeListing.js.map +1 -0
  101. package/dist/cjs/lib/test/actor.d.ts +21 -0
  102. package/dist/cjs/lib/test/actor.js +56 -0
  103. package/dist/cjs/lib/test/actor.js.map +1 -0
  104. package/dist/cjs/lib/test/mockApifyClient.d.ts +32 -0
  105. package/dist/cjs/lib/test/mockApifyClient.js +176 -0
  106. package/dist/cjs/lib/test/mockApifyClient.js.map +1 -0
  107. package/dist/cjs/types.d.ts +31 -0
  108. package/dist/cjs/types.js +3 -0
  109. package/dist/cjs/types.js.map +1 -0
  110. package/dist/cjs/utils/async.d.ts +19 -0
  111. package/dist/cjs/utils/async.js +74 -0
  112. package/dist/cjs/utils/async.js.map +1 -0
  113. package/dist/cjs/utils/error.d.ts +1 -0
  114. package/dist/cjs/utils/error.js +10 -0
  115. package/dist/cjs/utils/error.js.map +1 -0
  116. package/dist/cjs/utils/format.d.ts +9 -0
  117. package/dist/cjs/utils/format.js +19 -0
  118. package/dist/cjs/utils/format.js.map +1 -0
  119. package/dist/cjs/utils/package.d.ts +15 -0
  120. package/dist/cjs/utils/package.js +25 -0
  121. package/dist/cjs/utils/package.js.map +1 -0
  122. package/dist/cjs/utils/types.d.ts +6 -0
  123. package/dist/cjs/utils/types.js +9 -0
  124. package/dist/cjs/utils/types.js.map +1 -0
  125. package/dist/cjs/utils/url.d.ts +9 -0
  126. package/dist/cjs/utils/url.js +32 -0
  127. package/dist/cjs/utils/url.js.map +1 -0
  128. package/dist/cjs/utils/valueMonitor.d.ts +31 -0
  129. package/dist/cjs/utils/valueMonitor.js +91 -0
  130. package/dist/cjs/utils/valueMonitor.js.map +1 -0
  131. package/package.json +85 -0
@@ -0,0 +1,54 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.logLevelHandlerWrapper = exports.logLevelToCrawlee = exports.LOG_LEVEL = void 0;
4
+ const crawlee_1 = require("crawlee");
5
+ exports.LOG_LEVEL = ['debug', 'info', 'warn', 'error', 'off']; // prettier-ignore
6
+ /** Map log levels of `crawlee-one` to log levels of `crawlee` */
7
+ exports.logLevelToCrawlee = {
8
+ off: crawlee_1.LogLevel.OFF,
9
+ debug: crawlee_1.LogLevel.DEBUG,
10
+ info: crawlee_1.LogLevel.INFO,
11
+ warn: crawlee_1.LogLevel.WARNING,
12
+ error: crawlee_1.LogLevel.ERROR,
13
+ };
14
+ /**
15
+ * Wrapper for Crawlee route handler that configures log level.
16
+ *
17
+ *
18
+ * Usage with Crawlee's `RouterHandler.addDefaultHandler`
19
+ * ```ts
20
+ * const wrappedHandler = logLevelHandlerWrapper('debug')(handler)
21
+ * await router.addDefaultHandler<Ctx>(wrappedHandler);
22
+ * ```
23
+ *
24
+ * Usage with Crawlee's `RouterHandler.addHandler`
25
+ * ```ts
26
+ * const wrappedHandler = logLevelHandlerWrapper('error')(handler)
27
+ * await router.addHandler<Ctx>(wrappedHandler);
28
+ * ```
29
+ *
30
+ * Usage with `createCrawleeOne`
31
+ * ```ts
32
+ * const actor = await createCrawleeOne<CheerioCrawlingContext>({
33
+ * validateInput,
34
+ * router: createCheerioRouter(),
35
+ * routes,
36
+ * routeHandlers: ({ input }) => createHandlers(input!),
37
+ * routerWrappers: ({ input }) => [
38
+ * logLevelHandlerWrapper<CheerioCrawlingContext<any, any>>(input?.logLevel ?? 'info'),
39
+ * ],
40
+ * createCrawler: ({ router, input }) => createCrawler({ router, input, crawlerConfig }),
41
+ * });
42
+ * ```
43
+ */
44
+ const logLevelHandlerWrapper = (logLevel) => {
45
+ return (handler) => {
46
+ return (ctx, ...args) => {
47
+ ctx.log.info(`Setting log level to ${logLevel}`);
48
+ ctx.log.setLevel(exports.logLevelToCrawlee[logLevel]);
49
+ return handler(ctx, ...args);
50
+ };
51
+ };
52
+ };
53
+ exports.logLevelHandlerWrapper = logLevelHandlerWrapper;
54
+ //# sourceMappingURL=log.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"log.js","sourceRoot":"","sources":["../../../src/lib/log.ts"],"names":[],"mappings":";;;AAAA,qCAA4E;AAK/D,QAAA,SAAS,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAU,CAAC,CAAC,kBAAkB;AAG/F,iEAAiE;AACpD,QAAA,iBAAiB,GAAsC;IAClE,GAAG,EAAE,kBAAe,CAAC,GAAG;IACxB,KAAK,EAAE,kBAAe,CAAC,KAAK;IAC5B,IAAI,EAAE,kBAAe,CAAC,IAAI;IAC1B,IAAI,EAAE,kBAAe,CAAC,OAAO;IAC7B,KAAK,EAAE,kBAAe,CAAC,KAAK;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACI,MAAM,sBAAsB,GAAG,CAIpC,QAAkB,EACkB,EAAE;IACtC,OAAO,CAAC,OAAO,EAAE,EAAE;QACjB,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE;YACtB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;YACjD,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,yBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9C,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC/B,CAAC,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC,CAAC;AAbW,QAAA,sBAAsB,0BAajC","sourcesContent":["import { type CrawlingContext, LogLevel as CrawleeLogLevel } from 'crawlee';\n\nimport type { ArrVal } from '../utils/types';\nimport type { CrawlerRouterWrapper } from './router';\n\nexport const LOG_LEVEL = ['debug', 'info', 'warn', 'error', 'off'] as const; // prettier-ignore\nexport type LogLevel = ArrVal<typeof LOG_LEVEL>;\n\n/** Map log levels of `crawlee-one` to log levels of `crawlee` */\nexport const logLevelToCrawlee: Record<LogLevel, CrawleeLogLevel> = {\n off: CrawleeLogLevel.OFF,\n debug: CrawleeLogLevel.DEBUG,\n info: CrawleeLogLevel.INFO,\n warn: CrawleeLogLevel.WARNING,\n error: CrawleeLogLevel.ERROR,\n};\n\n/**\n * Wrapper for Crawlee route handler that configures log level.\n *\n *\n * Usage with Crawlee's `RouterHandler.addDefaultHandler`\n * ```ts\n * const wrappedHandler = logLevelHandlerWrapper('debug')(handler)\n * await router.addDefaultHandler<Ctx>(wrappedHandler);\n * ```\n *\n * Usage with Crawlee's `RouterHandler.addHandler`\n * ```ts\n * const wrappedHandler = logLevelHandlerWrapper('error')(handler)\n * await router.addHandler<Ctx>(wrappedHandler);\n * ```\n *\n * Usage with `createCrawleeOne`\n * ```ts\n * const actor = await createCrawleeOne<CheerioCrawlingContext>({\n * validateInput,\n * router: createCheerioRouter(),\n * routes,\n * routeHandlers: ({ input }) => createHandlers(input!),\n * routerWrappers: ({ input }) => [\n * logLevelHandlerWrapper<CheerioCrawlingContext<any, any>>(input?.logLevel ?? 'info'),\n * ],\n * createCrawler: ({ router, input }) => createCrawler({ router, input, crawlerConfig }),\n * });\n * ```\n */\nexport const logLevelHandlerWrapper = <\n T extends CrawlingContext,\n RouterCtx extends Record<string, any> = Record<string, any>\n>(\n logLevel: LogLevel\n): CrawlerRouterWrapper<T, RouterCtx> => {\n return (handler) => {\n return (ctx, ...args) => {\n ctx.log.info(`Setting log level to ${logLevel}`);\n ctx.log.setLevel(logLevelToCrawlee[logLevel]);\n return handler(ctx, ...args);\n };\n };\n};\n"]}
@@ -0,0 +1,10 @@
1
+ export declare const createLocalMigrator: ({ migrationsDir, extension, delimeter, }: {
2
+ migrationsDir: string;
3
+ /** Extension glob */
4
+ extension: string;
5
+ /** Delimeter between version and rest of file name */
6
+ delimeter: string;
7
+ }) => {
8
+ migrate: (version: string) => Promise<void>;
9
+ unmigrate: (version: string) => Promise<void>;
10
+ };
@@ -0,0 +1,57 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.createLocalMigrator = void 0;
16
+ const path_1 = __importDefault(require("path"));
17
+ const dotenv_1 = __importDefault(require("dotenv"));
18
+ const glob_1 = require("glob");
19
+ const apify_client_1 = require("apify-client");
20
+ const createLocalMigrator = ({ migrationsDir, extension = '.js', delimeter = '_', }) => {
21
+ const findLocalMigrationFileByVersion = (version) => __awaiter(void 0, void 0, void 0, function* () {
22
+ // Find files like "v1_bla_bla_bla.js" (by default)
23
+ const files = yield (0, glob_1.glob)(path_1.default.join(migrationsDir, `${version}${delimeter}*${extension}`));
24
+ if (!files.length) {
25
+ throw Error(`No migration file matched version "${version}"`);
26
+ }
27
+ if (files.length > 1) {
28
+ throw Error(`Ambiguous migration version. Version "${version}" matched multiple migration files`);
29
+ }
30
+ return files[0];
31
+ });
32
+ const setup = () => {
33
+ dotenv_1.default.config();
34
+ const client = new apify_client_1.ApifyClient({
35
+ token: process.env.APIFY_TOKEN,
36
+ });
37
+ return { client };
38
+ };
39
+ const migrate = (version) => __awaiter(void 0, void 0, void 0, function* () {
40
+ const migFile = yield findLocalMigrationFileByVersion(version);
41
+ const { client } = setup();
42
+ const { migrate } = require(migFile).default;
43
+ yield migrate(client);
44
+ });
45
+ const unmigrate = (version) => __awaiter(void 0, void 0, void 0, function* () {
46
+ const migFile = yield findLocalMigrationFileByVersion(version);
47
+ const { client } = setup();
48
+ const { unmigrate } = require(migFile).default;
49
+ yield unmigrate(client);
50
+ });
51
+ return {
52
+ migrate,
53
+ unmigrate,
54
+ };
55
+ };
56
+ exports.createLocalMigrator = createLocalMigrator;
57
+ //# sourceMappingURL=localMigrator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"localMigrator.js","sourceRoot":"","sources":["../../../../src/lib/migrate/localMigrator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,gDAAwB;AACxB,oDAA4B;AAC5B,+BAA4B;AAC5B,+CAA2C;AAIpC,MAAM,mBAAmB,GAAG,CAAC,EAClC,aAAa,EACb,SAAS,GAAG,KAAK,EACjB,SAAS,GAAG,GAAG,GAOhB,EAAE,EAAE;IACH,MAAM,+BAA+B,GAAG,CAAO,OAAe,EAAmB,EAAE;QACjF,mDAAmD;QACnD,MAAM,KAAK,GAAG,MAAM,IAAA,WAAI,EAAC,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,OAAO,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YACjB,MAAM,KAAK,CAAC,sCAAsC,OAAO,GAAG,CAAC,CAAC;SAC/D;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YACpB,MAAM,KAAK,CACT,yCAAyC,OAAO,oCAAoC,CACrF,CAAC;SACH;QACD,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAA,CAAC;IAEF,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,gBAAM,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,IAAI,0BAAW,CAAC;YAC7B,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW;SAC/B,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAO,OAAe,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,MAAM,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAoB,CAAC;QAC1D,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,MAAM,SAAS,GAAG,CAAO,OAAe,EAAE,EAAE;QAC1C,MAAM,OAAO,GAAG,MAAM,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAoB,CAAC;QAC5D,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAA,CAAC;IAEF,OAAO;QACL,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAnDW,QAAA,mBAAmB,uBAmD9B","sourcesContent":["import path from 'path';\nimport dotenv from 'dotenv';\nimport { glob } from 'glob';\nimport { ApifyClient } from 'apify-client';\n\nimport type { Migration } from './types';\n\nexport const createLocalMigrator = ({\n migrationsDir,\n extension = '.js',\n delimeter = '_',\n}: {\n migrationsDir: string;\n /** Extension glob */\n extension: string;\n /** Delimeter between version and rest of file name */\n delimeter: string;\n}) => {\n const findLocalMigrationFileByVersion = async (version: string): Promise<string> => {\n // Find files like \"v1_bla_bla_bla.js\" (by default)\n const files = await glob(path.join(migrationsDir, `${version}${delimeter}*${extension}`));\n if (!files.length) {\n throw Error(`No migration file matched version \"${version}\"`);\n }\n if (files.length > 1) {\n throw Error(\n `Ambiguous migration version. Version \"${version}\" matched multiple migration files`\n );\n }\n return files[0];\n };\n\n const setup = () => {\n dotenv.config();\n const client = new ApifyClient({\n token: process.env.APIFY_TOKEN,\n });\n return { client };\n };\n\n const migrate = async (version: string) => {\n const migFile = await findLocalMigrationFileByVersion(version);\n const { client } = setup();\n const { migrate } = require(migFile).default as Migration;\n await migrate(client);\n };\n\n const unmigrate = async (version: string) => {\n const migFile = await findLocalMigrationFileByVersion(version);\n const { client } = setup();\n const { unmigrate } = require(migFile).default as Migration;\n await unmigrate(client);\n };\n\n return {\n migrate,\n unmigrate,\n };\n};\n"]}
@@ -0,0 +1,7 @@
1
+ import type { Actor, ActorClient } from 'apify-client';
2
+ export declare const createLocalMigrationState: ({ stateDir }: {
3
+ stateDir: string;
4
+ }) => {
5
+ saveState: (migrationFilename: string, actor: ActorClient) => Promise<void>;
6
+ loadState: (migrationFilename: string) => Promise<Actor>;
7
+ };
@@ -0,0 +1,43 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.createLocalMigrationState = void 0;
16
+ const promises_1 = __importDefault(require("fs/promises"));
17
+ const path_1 = __importDefault(require("path"));
18
+ const createLocalMigrationState = ({ stateDir }) => {
19
+ const genStateFilepath = (migrationFilename) => {
20
+ const filenameNoExt = path_1.default.parse(migrationFilename).name;
21
+ const stateFilename = `${filenameNoExt}.json`;
22
+ const stateFilepath = path_1.default.join(stateDir, stateFilename);
23
+ return stateFilepath;
24
+ };
25
+ const saveState = (migrationFilename, actor) => __awaiter(void 0, void 0, void 0, function* () {
26
+ const stateFilepath = genStateFilepath(migrationFilename);
27
+ const actorData = JSON.stringify(yield actor.get());
28
+ yield promises_1.default.mkdir(path_1.default.dirname(stateFilepath), { recursive: true });
29
+ yield promises_1.default.writeFile(stateFilepath, actorData, 'utf-8');
30
+ });
31
+ const loadState = (migrationFilename) => __awaiter(void 0, void 0, void 0, function* () {
32
+ const stateFilepath = genStateFilepath(migrationFilename);
33
+ const fileContent = yield promises_1.default.readFile(stateFilepath, 'utf-8');
34
+ const actorData = JSON.parse(fileContent);
35
+ return actorData;
36
+ });
37
+ return {
38
+ saveState,
39
+ loadState,
40
+ };
41
+ };
42
+ exports.createLocalMigrationState = createLocalMigrationState;
43
+ //# sourceMappingURL=localState.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"localState.js","sourceRoot":"","sources":["../../../../src/lib/migrate/localState.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2DAA8B;AAC9B,gDAAwB;AAGjB,MAAM,yBAAyB,GAAG,CAAC,EAAE,QAAQ,EAAwB,EAAE,EAAE;IAC9E,MAAM,gBAAgB,GAAG,CAAC,iBAAyB,EAAE,EAAE;QACrD,MAAM,aAAa,GAAG,cAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC;QACzD,MAAM,aAAa,GAAG,GAAG,aAAa,OAAO,CAAC;QAC9C,MAAM,aAAa,GAAG,cAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QACzD,OAAO,aAAa,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAO,iBAAyB,EAAE,KAAkB,EAAiB,EAAE;QACvF,MAAM,aAAa,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QAEpD,MAAM,kBAAG,CAAC,KAAK,CAAC,cAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,MAAM,kBAAG,CAAC,SAAS,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC,CAAA,CAAC;IAEF,MAAM,SAAS,GAAG,CAAO,iBAAyB,EAAkB,EAAE;QACpE,MAAM,aAAa,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,MAAM,kBAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAU,CAAC;QACnD,OAAO,SAAS,CAAC;IACnB,CAAC,CAAA,CAAC;IAEF,OAAO;QACL,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA3BW,QAAA,yBAAyB,6BA2BpC","sourcesContent":["import fsp from 'fs/promises';\nimport path from 'path';\nimport type { Actor, ActorClient } from 'apify-client';\n\nexport const createLocalMigrationState = ({ stateDir }: { stateDir: string }) => {\n const genStateFilepath = (migrationFilename: string) => {\n const filenameNoExt = path.parse(migrationFilename).name;\n const stateFilename = `${filenameNoExt}.json`;\n const stateFilepath = path.join(stateDir, stateFilename);\n return stateFilepath;\n };\n\n const saveState = async (migrationFilename: string, actor: ActorClient): Promise<void> => {\n const stateFilepath = genStateFilepath(migrationFilename);\n const actorData = JSON.stringify(await actor.get());\n\n await fsp.mkdir(path.dirname(stateFilepath), { recursive: true });\n await fsp.writeFile(stateFilepath, actorData, 'utf-8');\n };\n\n const loadState = async (migrationFilename: string): Promise<Actor> => {\n const stateFilepath = genStateFilepath(migrationFilename);\n const fileContent = await fsp.readFile(stateFilepath, 'utf-8');\n const actorData = JSON.parse(fileContent) as Actor;\n return actorData;\n };\n\n return {\n saveState,\n loadState,\n };\n};\n"]}
@@ -0,0 +1,6 @@
1
+ import type { ApifyClient } from 'apify-client';
2
+ import type { MaybePromise } from '../../utils/types';
3
+ export interface Migration {
4
+ migrate: (client: ApifyClient) => MaybePromise<void>;
5
+ unmigrate: (client: ApifyClient) => MaybePromise<void>;
6
+ }
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/lib/migrate/types.ts"],"names":[],"mappings":"","sourcesContent":["import type { ApifyClient } from 'apify-client';\n\nimport type { MaybePromise } from '../../utils/types';\n\nexport interface Migration {\n migrate: (client: ApifyClient) => MaybePromise<void>;\n unmigrate: (client: ApifyClient) => MaybePromise<void>;\n}\n"]}
@@ -0,0 +1,65 @@
1
+ import { CrawleeOneReadmeTemplates, ReadmeFeature, ReadmeFeatureType } from './types';
2
+ import type { CrawleeOneScraperActorSpec } from '../actorSpec';
3
+ export interface CrawleeOneReadmeTemplatesOverrides extends Omit<CrawleeOneReadmeTemplates, 'features'> {
4
+ features?: Partial<Record<ReadmeFeatureType, Partial<ReadmeFeature>>>;
5
+ }
6
+ /** Define the texts in features sections that are, by default, common across all actors */
7
+ export declare const defaultFeatureTexts: CrawleeOneReadmeTemplates['features'];
8
+ /**
9
+ * Render a README.md file from a common template for a given Apify crawler.
10
+ *
11
+ * See https://docs.apify.com/academy/get-most-of-actors/actor-readme
12
+ *
13
+ * The templates are rendered using ETA (https://eta.js.org/)
14
+ *
15
+ * Each template has access to `it` global variable. `it` has these props:
16
+ *
17
+ * - `it.fn` - The functions passed to this function + more (see below)
18
+ * - `it.t` - The templates object passed to this function
19
+ * - `it.a` - The actorSpec object passed to this function
20
+ *
21
+ * Example:
22
+ * ```eta
23
+ * ActorId: <%~ it.a.platform.actorId %>
24
+ * ```
25
+ *
26
+ * Following functions are available by default:
27
+ * - `it.fn.enumerate`
28
+ * - `it.fn.perfStat`
29
+ * - `it.fn.millify`
30
+ * - `it.fn.capitalize`
31
+ * - `it.fn.stringify`
32
+ * - `it.fn.email`
33
+ * - `it.fn.includesPersonalData`
34
+ * - `it.fn.collectFilters`
35
+ * - `it.fn.collectModes`
36
+ * - `it.fn.collectEmails`
37
+ *
38
+ * See their definitions for details
39
+ */
40
+ export declare const renderApifyReadme: (input: {
41
+ /** Filepath (relative to CWD) where the generated README should be written. */
42
+ filepath: string;
43
+ /**
44
+ * Info about a particular actor.
45
+ *
46
+ * Inside the template during rendering, this object
47
+ * can be accessed as `<%~ it.a.platform.actorId %>`
48
+ */
49
+ actorSpec: CrawleeOneScraperActorSpec;
50
+ /**
51
+ * Custom eta template strings that plug into different
52
+ * parts of the README template.
53
+ *
54
+ * Inside the template during rendering, these templates
55
+ * can be accessed as `<%~ it.t.someTemplate %>`
56
+ */
57
+ templates: CrawleeOneReadmeTemplatesOverrides;
58
+ /**
59
+ * Functions to be made available in the template.
60
+ *
61
+ * Inside the template during rendering, these functions
62
+ * can be accessed as `<%~ it.fn.funcName() %>`
63
+ */
64
+ fn?: Record<string, (...args: any[]) => any> | undefined;
65
+ }) => Promise<void>;