crawlee-one 1.0.0

package/dist/cjs/lib/readme/readme.js

@@ -0,0 +1,534 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderApifyReadme = exports.defaultFeatureTexts = void 0;
+const eta_1 = __importDefault(require("eta"));
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const millify_1 = __importDefault(require("millify"));
+const lodash_1 = require("lodash");
+const types_1 = require("./types");
+/**
+ * Turn
+ *
+ * `[1, 2, 3, 4]`
+ *
+ * into
+ *
+ * `[{ item: 1, separator: ', '}, { item: 2, separator: ', '}, { item: 3, separator: ' or '}, { item: 4, separator: '' }]`
+ *
+ * So that the list can be rendered as:
+ *
+ * `1, 2, 3 or 4`
+ */
+const addListSeparators = (arr) => arr.map((item, index) => ({
+    item,
+    separator: index === arr.length - 1 ? '' : index === arr.length - 2 ? ' or ' : ', ',
+}));
+/**
+ * Render
+ *
+ * `[1, 2, 3, 4]`
+ *
+ * as
+ *
+ * `1, 2, 3 or 4`
+ */
+const renderList = (arr) => addListSeparators(arr)
+    .map((d) => `${d.item}${d.separator}`)
+    .join(''); // prettier-ignore
+/** Given time in seconds, get the hours, minutes and seconds */
+const timeDeltaFromSec = (timeInSec) => {
+    const secInMin = 60;
+    const secInHour = 60 * 60;
+    let remainingTimeInSec = timeInSec;
+    const hours = Math.floor(remainingTimeInSec / secInHour);
+    remainingTimeInSec = remainingTimeInSec - hours * secInHour;
+    const minutes = Math.floor(remainingTimeInSec / secInMin);
+    const seconds = remainingTimeInSec - minutes * secInMin;
+    return { hours, minutes, seconds };
+};
+/** Format time delta as "6h 2m 23s" or "2m 23s" */
+const renderTimeDelta = ({ hours, minutes, seconds }) => {
+    return [hours ? `${hours}h` : '', minutes ? `${minutes}m` : '', hours ? '' : `${seconds}s`].join(' '); // prettier-ignore
+};
+/** Render perf stat as "$0.016 in 0h 2m 23s" */
+const renderPerfStat = (perf) => {
+    const timeText = renderTimeDelta(timeDeltaFromSec(perf.timeSec));
+    return `$${(0, lodash_1.round)(perf.costUsd, 3)} in ${timeText}`;
+};
+/** Render email safely as name[dot]surname[at]domain[dot]com */
+const renderEmail = (email) => email.replace(/\./g, '[dot]').replace(/\@/g, '[at]');
+const renderJson = (s, options = {}) => JSON.stringify(s, null, 2)
+    .split('\n')
+    .map((s, index) => {
+    const shouldApplyPadding = index >= (options.paddingStartLine || 0);
+    const paddingLeftRepeats = shouldApplyPadding ? options.paddingLeft || 0 : 0;
+    return `${' '.repeat(paddingLeftRepeats)}${s}`;
+})
+    .join('\n');
+const includesPersonalData = (it) => it.a.datasets.some((d) => d.privacy.personalDataFields.length);
+const collectFilters = (it) => (0, lodash_1.uniqBy)(it.a.datasets.flatMap((d) => d.filters), m => m); // prettier-ignore
+const collectModes = (it) => (0, lodash_1.uniqBy)(it.a.datasets.flatMap((d) => d.modes), (m) => m.name); // prettier-ignore
+const collectEmails = (it) => (0, lodash_1.uniqBy)(it.a.authors.flatMap((a) => a.email), e => e); // prettier-ignore
+// TODO
+/** Define the texts in features sections that are, by default, common across all actors */
+exports.defaultFeatureTexts = {
+    datasets: {
+        supported: (it) => it.a.datasets.length > 1,
+        title: '<%~ it.a.datasets.length %> kinds of datasets',
+        mainText: '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.',
+    },
+    modes: {
+        supported: (it) => it.fn.collectModes(it).length > 1,
+        title: `<%~ it.fn.enumerate(it.fn.collectModes(it).map((m) => it.fn.capitalize(m.name))) %> modes`,
+        mainText: `- Scraping can be ` +
+            `<%~ it.fn.enumerate(it.fn.collectModes(it).map((m) => m.name.toLowerCase() + ' (' + m.shortDesc + ')')) %>` +
+            `.`,
+    },
+    filters: {
+        supported: (it) => it.a.datasets.some((d) => d.filters.length),
+        data: {
+            maxEntriesSupported: true,
+        },
+        title: 'Filter support',
+        mainText: `- Filter the results by <%~ it.fn.enumerate(it.fn.collectFilters(it)) %>.\n` +
+            `<% if (it.t.features.filters.data.maxEntriesSupported) { %>\n` +
+            `  - Limit the number of results.\n` +
+            `<% } %>`,
+    },
+    noBrowser: {
+        supported: (it) => it.a.datasets.some((d) => !d.features.usesBrowser),
+        title: 'Blazing fast',
+        mainText: `- The actor doesn't use a browser, which means it's fast and cheap.`,
+    },
+    proxy: {
+        supported: (it) => it.a.datasets.some((d) => d.features.proxySupport),
+        title: 'Proxy support',
+        mainText: `- You can use Apify's proxy, or your own, via Input.`,
+    },
+    integratedETL: {
+        supported: (it) => it.a.datasets.some((d) => d.features.integratedETL),
+        title: 'Integrated data filtering and transformation',
+        mainText: `- Filter and modify scraped entries out of the box from within Apify UI, without needing other tools.`,
+    },
+    integratedCache: {
+        supported: (it) => it.a.datasets.some((d) => d.features.integratedCache),
+        title: 'Integrated cache',
+        mainText: `- You can use cache together with custom filtering to e.g. save only NEW entries to the dataset. Save time and reduce cost.\n` +
+            `  - Cache automatically stores which entries were already scraped. Cache can persist between different scraper runs.`,
+    },
+    crawlerConfig: {
+        supported: (it) => it.a.datasets.some((d) => d.features.configurable),
+        title: 'Custom crawler configuration',
+        mainText: `- For advanced needs, you can pass Crawler configuration via Input.`,
+    },
+    tests: {
+        supported: (it) => it.a.datasets.some((d) => d.features.regularlyTested),
+        title: 'Tested daily for high reliability',
+        mainText: `- The actor is regularly tested end-to-end to minimize the risk of a broken integration.`,
+    },
+    privacy: {
+        supported: (it) => it.a.datasets.some((d) => d.features.privacyCompliance),
+        title: 'Privacy-compliant (GDPR)',
+        mainText: `- By default, personal data is redacted to avoid privacy issues. You can opt-in to include un-censored data.`,
+    },
+    metamorph: {
+        supported: () => true,
+        title: 'Pass scraped dataset to other actors',
+        mainText: `- Automatically trigger another actor when this one is done to process the scraped dataset.\n` +
+            `  - Metamorphing means that the dataset and key-value store is passed to another actor.\n` +
+            `  - Actor metamorph can be configure via actor input. No need to define custom actors just for that.`,
+    },
+    errorMonitoring: {
+        supported: (it) => it.a.datasets.some((d) => d.features.errorMonitoring),
+        data: {
+            hasSentry: true,
+        },
+        title: 'Error monitoring',
+        mainText: `- Errors from your runs are captured and surfaced in the \`REPORTING\` dataset. (See Storage > Dataset > Select dropdown).\n` +
+            `<% if (it.t.features.errorMonitoring.data.hasSentry) { %>\n` +
+            `  - Errors are also automatically reported to [Sentry](https://sentry.io/).\n` +
+            `<% } %>`,
+    },
+};
+const H = types_1.README_HOOK_ENUM;
+/** The template for rendering README for crawler */
+const readmeTemplate = `
+<%~ it.a.actor.title %>
+===============================
+
+<%~ it.a.actor.shortDesc %>
+
+## What is <%~ it.a.actor.title %> and how it works?
+
+<%~ include("hook.${H.introAfterBegin}", it) %>
+
+With <%~ it.a.actor.title %>, you can extract:
+
+<%- it.a.datasets.forEach((dataset) => { %>
+- [<%~ dataset.shortDesc %>]( <%~ dataset.url %> )
+<%- }) %>
+
+<%-~ include("hook.${H.introAfterDatasets}", it) %>
+
+See the [outputs section](#outputs) for a detailed description.
+
+The data can be downloaded in JSON, JSONL, XML, CSV, Excel, or HTML formats.
+
+<%~ include("hook.${H.introBeforeEnd}", it) %>
+
+## Features
+
+<%~ include("hook.${H.featuresAfterBegin}", it) %>
+
+This actor is a robust production-grade solution suitable for businesses and those that need reliability.
+
+<%- Object.entries(it.t.features).forEach(([featName, feat]) => { %>
+<%- if (feat.supported(it)) { %>
+- **<%~ include("feat." + featName + ".title", it) %>**
+<% if (feat.afterBegin) { -%>
+  <%~ include("feat." + featName + ".afterBegin", it) %>
+<%- } -%>
+  
+  <%-~ include("feat." + featName + ".mainText", it) -%>
+
+<% if (feat.beforeEnd) { -%>
+  <%~ include("feat." + featName + ".beforeEnd", it) %>
+<%- } -%>
+<%- } %>
+<%- }) %>
+
+<%~ include("hook.${H.featuresBeforeEnd}", it) %>
+
+## How can you use the data scraped from <%~ it.a.websites[0].name %>? (Examples)
+
+<%~ include("hook.${H.useCases}", it) %>
+
+## How to use <%~ it.a.actor.title %>
+
+<%~ include("hook.${H.usageAfterBegin}", it) %>
+
+1. Create a free Apify account using your email
+2. Open <%~ it.a.actor.title %>
+3. In Input, select the dataset to scrape, and filters to apply.
+4. Click "Start" and wait for the data to be extracted.
+5. Download your data in JSON, JSONL, XML, CSV, Excel, or HTML format.
+
+For details and examples for all input fields, please visit the [Input tab](<%~ it.a.actor.publicUrl %>/input-schema).
+
+<%~ include("hook.${H.usageBeforeEnd}", it) %>
+
+## How much does it cost to scrape <%~ it.a.websites[0].name %>?
+
+<%~ include("hook.${H.costAfterBegin}", it) %>
+
+<%- it.a.datasets.filter((d) => d.perfStats && d.perfStats.length).forEach((dataset) => { %>
+### <%~ it.fn.capitalize(dataset.name) %>
+
+<table>
+  <thead>
+    <tr>
+      <td></td>
+      <%- it.t.perfTables[dataset.perfTable].cols.forEach((col) => { -%>
+      <td><strong>
+        <%~ include("perfTable." + dataset.perfTable + ".col." + col.colId, { ...it, dataset }) %>
+      </strong></td>
+      <%- }) -%>
+    </tr>
+  </thead>
+
+  <tbody>
+    <%- it.t.perfTables[dataset.perfTable].rows.forEach((row) => { -%>
+    <tr>
+      <td>
+        <%~ include("perfTable." + dataset.perfTable + ".row." + row.rowId, { ...it, dataset }) %>
+      </td>
+      <%- it.t.perfTables[dataset.perfTable].cols.forEach((col) => { -%>
+      <td>
+        <%~ it.fn.perfStat(
+          dataset.perfStats.find(d => d.rowId === row.rowId && d.colId === col.colId)
+        ) %>
+      </td>
+      <%- }) -%>
+    </tr>
+    <%- }) -%>
+  </tbody>
+</table>
+
+<%- }) %>
+
+<br/>
+
+<%~ include("hook.${H.costAfterPerfTables}", it) %>
+
+Remember that with the [Apify Free plan](https://apify.com/pricing), you have $5 free usage per month.
+
+<%~ include("hook.${H.costBeforeEnd}", it) %>
+
+## Input options
+
+<%~ include("hook.${H.inputAfterBegin}", it) %>
+
+For details and examples for all input fields, please visit the [Input tab](<%~ it.a.actor.publicUrl %>/input-schema).
+
+<%~ include("hook.${H.inputBeforeEnd}", it) %>
+
+### Filter options
+
+<%~ include("hook.${H.filterAfterBegin}", it) %>
+
+You can run <%~ it.a.actor.title %> as is, with the default options, to get a sample of the 
+<%~ it.a.datasets.find(d => d.isDefault).name %> entries
+<%_ if (it.fn.collectModes(it).find(m => m.isDefault)) { _%>
+<%_~ ' (' + it.fn.collectModes(it).find(m => m.isDefault).name %> mode)
+<%_ } _%>.
+
+<%- if (it.fn.collectFilters(it).length) { %>
+Otherwise, you can filter by:
+
+<%- it.fn.collectFilters(it).forEach((filter) => { %>
+  - <%~ it.fn.capitalize(filter) %>
+<%- }) %>
+<%- } %>
+
+<%~ include("hook.${H.filterBeforeEnd}", it) %>
+
+### Limit options
+
+<%~ include("hook.${H.limitAfterBegin}", it) %>
+
+To limit how many results you get, set \`<%~ it.t.input.maxCount %>\` to desired amount.
+
+<%~ include("hook.${H.limitBeforeEnd}", it) %>
+
+### Input examples
+
+<%~ include("hook.${H.inputExampleAfterBegin}", it) %>
+
+<% it.t.exampleInputs.forEach((example, index) => { %>
+#### Example <%~ index + 1 %>: <%~ example.title %>
+
+\`\`\`json
+{
+<%- Object.entries(example.inputData).forEach(([key, value]) => { -%>
+<%- if (example.inputDataComments && example.inputDataComments[key]) { %>
+  // <%~ example.inputDataComments[key] %>
+<%- } %>
+  "<%~ key %>": <% _%><%~ it.fn.stringify(value, { paddingLeft: 2, paddingStartLine: 1 }) %>,
+<%- }) %>
+}
+\`\`\`
+
+<% }) %>
+
+<%~ include("hook.${H.inputExampleBeforeEnd}", it) %>
+
+## Outputs
+
+<%~ include("hook.${H.outputAfterBegin}", it) %>
+
+Once the actor is done, you can see the overview of results in the Output tab.
+
+To export the data, head over to the Storage tab.
+
+![<%~ it.a.actor.title %> dataset overview](<%~ it.a.actor.datasetOverviewImgUrl %>)
+
+<%~ include("hook.${H.outputBeforeEnd}", it) %>
+
+## Sample output from <%~ it.a.actor.title %>
+
+<%~ include("hook.${H.outputExampleAfterBegin}", it) %>
+
+<%- it.a.datasets.forEach((dataset) => { %>
+### <%~ it.fn.capitalize(dataset.name) %> output
+
+\`\`\`json
+{
+<%- Object.entries(dataset.output.exampleEntry).forEach(([key, value]) => { -%>
+<%- if (dataset.output.exampleEntryComments && dataset.output.exampleEntryComments[key]) { %>
+  // <%~ dataset.output.exampleEntryComments[key] %>
+<%- } %>
+  "<%~ key %>": <% _%><%~ it.fn.stringify(value, { paddingLeft: 2, paddingStartLine: 1 }) %>,
+<%- }) %>
+}
+\`\`\`
+
+<%- }) %>
+
+<%~ include("hook.${H.outputExampleBeforeEnd}", it) %>
+
+## How to integrate <%~ it.a.actor.title %> with other services, APIs or Actors
+
+<%~ include("hook.${H.integrationAfterBegin}", it) %>
+
+You can connect the actor with many of the
+[integrations on the Apify platform](https://apify.com/integrations).
+You can integrate with Make, Zapier, Slack, Airbyte, GitHub, Google Sheets, Google Drive,
+[and more](https://docs.apify.com/integrations).
+Or you can use
+[webhooks](https://docs.apify.com/integrations/webhooks)
+to carry out an action whenever an event occurs, e.g. get a notification whenever
+Instagram API Scraper successfully finishes a run.
+
+<%~ include("hook.${H.integrationBeforeEnd}", it) %>
+
+## Use <%~ it.a.actor.title %> with Apify API
+
+<%~ include("hook.${H.apifyAfterBegin}", it) %>
+
+The Apify API gives you programmatic access to the Apify platform.
+The API is organized around RESTful HTTP endpoints that enable you to manage,
+schedule and run Apify actors. The API also lets you access any datasets,
+monitor actor performance, fetch results, create and update versions, and more.
+
+To access the API using Node.js, use the \`apify-client\` NPM package.
+To access the API using Python, use the \`apify-client\` PyPI package.
+
+Check out the [Apify API reference](https://docs.apify.com/api/v2) docs
+for full details or click on the
+[API tab](<%~ it.a.actor.publicUrl %>/api)
+for code examples.
+
+<%~ include("hook.${H.apifyBeforeEnd}", it) %>
+
+## Is it legal to scrape <%~ it.a.websites[0].name %>?
+
+<%~ include("hook.${H.legalityAfterBegin}", it) %>
+
+It is legal to scrape publicly available data such as product descriptions,
+prices, or ratings. Read Apify's blog post on
+[the legality of web scraping](https://blog.apify.com/is-web-scraping-legal/)
+to learn more.
+
+<%- if (it.fn.includesPersonalData(it)) { %>
+However, the following datasets include personal data:
+
+<%- it.a.datasets.filter((d) => d.privacy.personalDataFields.length).forEach((dataset) => { %>
+- <%~ it.fn.capitalize(dataset.name) %> dataset includes info about <%~ it.fn.enumerate(dataset.privacy.personalDataSubjects) %>.
+  - Fields: <%~ dataset.privacy.personalDataFields.join(', ') %>
+<% if (dataset.privacy.isPersonalDataRedacted) { -%>
+  - By default, this personal data is redacted, and in such case, it's safe to scrape the data.
+<%- } %>
+<%- }) %>
+
+To get the unredacted data, toggle on the "<%~ it.t.input.privacyName %>" actor input.
+
+> **Warning:** Including personal data is done at your own risk. It is your
+responsibility to make sure you have obtained consent or have a legal basis
+for using the data.
+>
+> By using this actor, you agree not to hold the author of this actor liable for privacy
+or data-related issues that may arise during its use.
+
+Redacted fields may show a message like this instead of the actual value:
+
+\`\`\`txt
+<Redacted property "email". To include the actual value, toggle ON the Actor input option "<%~ it.t.input.privacyName %>">
+\`\`\`
+
+<%- } %>
+
+<%~ include("hook.${H.legalityBeforeEnd}", it) %>
+
+## Who can I contact for issues with <%~ it.a.websites[0].name %> actor?
+
+<%~ include("hook.${H.contactAfterBegin}", it) %>
+
+To report issues and find help,
+<%- if (it.a.platform.socials.discord) { %>
+head over to the
+[Discord community](<%~ it.a.platform.socials.discord %>)
+<%- } %>
+<%_ if (it.a.platform.socials.discord && it.fn.collectEmails(it).length) { %>, or <% } _%>
+email me at <%~ it.fn.email(it.fn.collectEmails(it)[0]) %>
+
+<%~ include("hook.${H.contactBeforeEnd}", it) %>
+`;
+/**
+ * Render a README.md file from a common template for a given Apify crawler.
+ *
+ * See https://docs.apify.com/academy/get-most-of-actors/actor-readme
+ *
+ * The templates are rendered using ETA (https://eta.js.org/)
+ *
+ * Each template has access to `it` global variable. `it` has these props:
+ *
+ * - `it.fn` - The functions passed to this function + more (see below)
+ * - `it.t` - The templates object passed to this function
+ * - `it.a` - The actorSpec object passed to this function
+ *
+ * Example:
+ * ```eta
+ * ActorId: <%~ it.a.platform.actorId %>
+ * ```
+ *
+ * Following functions are available by default:
+ * - `it.fn.enumerate`
+ * - `it.fn.perfStat`
+ * - `it.fn.millify`
+ * - `it.fn.capitalize`
+ * - `it.fn.stringify`
+ * - `it.fn.email`
+ * - `it.fn.includesPersonalData`
+ * - `it.fn.collectFilters`
+ * - `it.fn.collectModes`
+ * - `it.fn.collectEmails`
+ *
+ * See their definitions for details
+ */
+const renderApifyReadme = (input) => __awaiter(void 0, void 0, void 0, function* () {
+    // Assign the default values to a clone
+    const templates = (0, lodash_1.cloneDeep)(input.templates);
+    templates.features = templates.features || {};
+    Object.entries(exports.defaultFeatureTexts).forEach(([key, feat]) => {
+        templates.features[key] = (0, lodash_1.defaults)(templates.features[key] || {}, feat);
+    });
+    // Define templates for 'include(...)'s for template hooks
+    types_1.README_HOOK.forEach((key) => {
+        var _a;
+        const template = (_a = templates.hooks) === null || _a === void 0 ? void 0 : _a[key];
+        eta_1.default.templates.define(`hook.${key}`, eta_1.default.compile(template || ''));
+    });
+    // Define templates for 'include(...)'s for feature hooks
+    Object.entries(templates.features).forEach(([key, feat]) => {
+        const { title, mainText, afterBegin, beforeEnd } = feat;
+        eta_1.default.templates.define(`feat.${key}.title`, eta_1.default.compile(title));
+        eta_1.default.templates.define(`feat.${key}.mainText`, eta_1.default.compile(mainText));
+        eta_1.default.templates.define(`feat.${key}.afterBegin`, eta_1.default.compile(afterBegin !== null && afterBegin !== void 0 ? afterBegin : ''));
+        eta_1.default.templates.define(`feat.${key}.beforeEnd`, eta_1.default.compile(beforeEnd !== null && beforeEnd !== void 0 ? beforeEnd : ''));
+    });
+    // Define templates for 'include(...)'s for perf table hooks
+    Object.entries(templates.perfTables || {}).forEach(([key, perfTable]) => {
+        perfTable.rows.forEach((row) => eta_1.default.templates.define(`perfTable.${key}.row.${row.rowId}`, eta_1.default.compile(row.template))); // prettier-ignore
+        perfTable.cols.forEach((col) => eta_1.default.templates.define(`perfTable.${key}.col.${col.colId}`, eta_1.default.compile(col.template))); // prettier-ignore
+    });
+    const fn = Object.assign({ enumerate: renderList, perfStat: renderPerfStat, millify: millify_1.default,
+        capitalize: lodash_1.capitalize, stringify: renderJson, email: renderEmail, includesPersonalData,
+        collectFilters,
+        collectModes,
+        collectEmails }, input.fn);
+    const data = { fn, t: templates, a: input.actorSpec };
+    const readmeContent = eta_1.default.render(readmeTemplate, data, {
+        strict: true,
+        rmWhitespace: false,
+        autoTrim: false,
+        context: {},
+    });
+    const readmePath = path_1.default.resolve(process.cwd(), input.filepath);
+    yield promises_1.default.mkdir(path_1.default.dirname(readmePath), { recursive: true });
+    yield promises_1.default.writeFile(readmePath, readmeContent, 'utf-8');
+});
+exports.renderApifyReadme = renderApifyReadme;
+//# sourceMappingURL=readme.js.map

package/dist/cjs/lib/readme/readme.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readme.js","sourceRoot":"","sources":["../../../../src/lib/readme/readme.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,8CAAsB;AACtB,2DAA8B;AAC9B,gDAAwB;AACxB,sDAA8B;AAC9B,mCAAwE;AAGxE,mCAOiB;AAQjB;;;;;;;;;;;;GAYG;AACH,MAAM,iBAAiB,GAAG,CAAI,GAAQ,EAAE,EAAE,CACxC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IACxB,IAAI;IACJ,SAAS,EAAE,KAAK,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;CACpF,CAAC,CAAC,CAAC;AAEN;;;;;;;;GAQG;AACH,MAAM,UAAU,GAAG,CAAC,GAAU,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC;KACtD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC;KACrC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB;AAQ/B,gEAAgE;AAChE,MAAM,gBAAgB,GAAG,CAAC,SAAiB,EAAa,EAAE;IACxD,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAC1B,IAAI,kBAAkB,GAAG,SAAS,CAAC;IAEnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAAC;IACzD,kBAAkB,GAAG,kBAAkB,GAAG,KAAK,GAAG,SAAS,CAAC;IAE5D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,QAAQ,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,kBAAkB,GAAG,OAAO,GAAG,QAAQ,CAAC;IACxD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC,CAAC;AAEF,mDAAmD;AACnD,MAAM,eAAe,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAa,EAAE,EAAE;IACjE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;AAC3H,CAAC,CAAC;AAEF,gDAAgD;AAChD,MAAM,cAAc,GAAG,CAAC,IAAqB,EAAE,EAAE;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,OAAO,IAAI,IAAA,cAAK,EAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,QAAQ,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,gEAAgE;AAChE,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAE5F,MAAM,UAAU,GAAG,CAAC,CAAS,EAAE,UAA+D,EAAE,EAAE,EAAE,CAClG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;KACvB,KAAK,CAAC,IAAI,CAAC;KACX,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;IAChB,MAAM,kBAAkB,GAAG,KAAK,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACpE,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;AACjD,CAAC,CAAC;KACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAEhB,MAAM,oBAAoB,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACpG,MAAM,cAAc,GAAG,CAAC,EAAO,EAAE,EAAE,CAAC,IAAA,eAAM,EAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;AAC/G,MAAM,YAAY,GAAG,CAAC,EAAO,EAAE,EAAE,CAAC,IAAA,eAAM,EAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB;AACvH,MAAM,aAAa,GAAG,CAAC,EAAO,EAAE,EAAE,CAAC,IAAA,eAAM,EAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;AAE3G,OAAO;AACP,2FAA2F;AAC9E,QAAA,mBAAmB,GAA0C;IACxE,QAAQ,EAAE;QACR,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAC3C,KAAK,EAAE,+CAA+C;QACtD,QAAQ,EAAE,+EAA+E;KAC1F;IACD,KAAK,EAAE;QACL,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;QACpD,KAAK,EAAE,2FAA2F;QAClG,QAAQ,EACN,oBAAoB;YACpB,4GAA4G;YAC5G,GAAG;KACN;IACD,OAAO,EAAE;QACP,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAC9D,IAAI,EAAE;YACJ,mBAAmB,EAAE,IAAI;SAC1B;QACD,KAAK,EAAE,gBAAgB;QACvB,QAAQ,EACN,6EAA6E;YAC7E,+DAA+D;YAC/D,oCAAoC;YACpC,SAAS;KACZ;IACD,SAAS,EAAE;QACT,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;QACrE,KAAK,EAAE,cAAc;QACrB,QAAQ,EAAE,qEAAqE;KAChF;IACD,KAAK,EAAE;QACL,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;QACrE,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,sDAAsD;KACjE;IACD,aAAa,EAAE;QACb,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;QACtE,KAAK,EAAE,8CAA8C;QACrD,QAAQ,EAAE,uGAAuG;KAClH;IACD,eAAe,EAAE;QACf,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;QACxE,KAAK,EAAE,kBAAkB;QACzB,QAAQ,EACN,+HAA+H;YAC/H,sHAAsH;KACzH;IACD,aAAa,EAAE;QACb,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;QACrE,KAAK,EAAE,8BAA8B;QACrC,QAAQ,EAAE,qEAAqE;KAChF;IACD,KAAK,EAAE;QACL,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;QACxE,KAAK,EAAE,mCAAmC;QAC1C,QAAQ,EAAE,0FAA0F;KACrG;IACD,OAAO,EAAE;QACP,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC1E,KAAK,EAAE,0BAA0B;QACjC,QAAQ,EAAE,8GAA8G;KACzH;IACD,SAAS,EAAE;QACT,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;QACrB,KAAK,EAAE,sCAAsC;QAC7C,QAAQ,EACN,+FAA+F;YAC/F,2FAA2F;YAC3F,sGAAsG;KACzG;IACD,eAAe,EAAE;QACf,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;QACxE,IAAI,EAAE;YACJ,SAAS,EAAE,IAAI;SAChB;QACD,KAAK,EAAE,kBAAkB;QACzB,QAAQ,EACN,8HAA8H;YAC9H,6DAA6D;YAC7D,+EAA+E;YAC/E,SAAS;KACZ;CACF,CAAC;AAEF,MAAM,CAAC,GAAG,wBAAgB,CAAC;AAE3B,oDAAoD;AACpD,MAAM,cAAc,GAAG;;;;;;;;oBAQH,CAAC,CAAC,eAAe;;;;;;;;qBAQhB,CAAC,CAAC,kBAAkB;;;;;;oBAMrB,CAAC,CAAC,cAAc;;;;oBAIhB,CAAC,CAAC,kBAAkB;;;;;;;;;;;;;;;;;;;oBAmBpB,CAAC,CAAC,iBAAiB;;;;oBAInB,CAAC,CAAC,QAAQ;;;;oBAIV,CAAC,CAAC,eAAe;;;;;;;;;;oBAUjB,CAAC,CAAC,cAAc;;;;oBAIhB,CAAC,CAAC,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAuChB,CAAC,CAAC,mBAAmB;;;;oBAIrB,CAAC,CAAC,aAAa;;;;oBAIf,CAAC,CAAC,eAAe;;;;oBAIjB,CAAC,CAAC,cAAc;;;;oBAIhB,CAAC,CAAC,gBAAgB;;;;;;;;;;;;;;;;oBAgBlB,CAAC,CAAC,eAAe;;;;oBAIjB,CAAC,CAAC,eAAe;;;;oBAIjB,CAAC,CAAC,cAAc;;;;oBAIhB,CAAC,CAAC,sBAAsB;;;;;;;;;;;;;;;;;;oBAkBxB,CAAC,CAAC,qBAAqB;;;;oBAIvB,CAAC,CAAC,gBAAgB;;;;;;;;oBAQlB,CAAC,CAAC,eAAe;;;;oBAIjB,CAAC,CAAC,uBAAuB;;;;;;;;;;;;;;;;;;oBAkBzB,CAAC,CAAC,sBAAsB;;;;oBAIxB,CAAC,CAAC,qBAAqB;;;;;;;;;;;oBAWvB,CAAC,CAAC,oBAAoB;;;;oBAItB,CAAC,CAAC,eAAe;;;;;;;;;;;;;;;oBAejB,CAAC,CAAC,cAAc;;;;oBAIhB,CAAC,CAAC,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAmCpB,CAAC,CAAC,iBAAiB;;;;oBAInB,CAAC,CAAC,iBAAiB;;;;;;;;;;oBAUnB,CAAC,CAAC,gBAAgB;CACrC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACI,MAAM,iBAAiB,GAAG,CAAO,KAyBvC,EAAE,EAAE;IACH,uCAAuC;IACvC,MAAM,SAAS,GAAG,IAAA,kBAAS,EAAC,KAAK,CAAC,SAAS,CAA8B,CAAC;IAC1E,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC9C,MAAM,CAAC,OAAO,CAAC,2BAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;QAC1D,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAA,iBAAQ,EAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,0DAA0D;IAC1D,mBAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;;QAC1B,MAAM,QAAQ,GAAG,MAAA,SAAS,CAAC,KAAK,0CAAG,GAAG,CAAC,CAAC;QACxC,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IACH,yDAAyD;IACzD,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;QACzD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACxD,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,EAAE,aAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,GAAG,WAAW,EAAE,aAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,GAAG,aAAa,EAAE,aAAG,CAAC,OAAO,CAAC,UAAU,aAAV,UAAU,cAAV,UAAU,GAAI,EAAE,CAAC,CAAC,CAAC;QAC9E,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,GAAG,YAAY,EAAE,aAAG,CAAC,OAAO,CAAC,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IACH,4DAA4D;IAC5D,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE;QACtE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,GAAG,CAAC,KAAK,EAAE,EAAE,aAAG,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;QACzI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAG,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,GAAG,CAAC,KAAK,EAAE,EAAE,aAAG,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;IAC3I,CAAC,CAAC,CAAC;IAEH,MAAM,EAAE,mBACN,SAAS,EAAE,UAAU,EACrB,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAP,iBAAO;QACP,UAAU,EAAV,mBAAU,EACV,SAAS,EAAE,UAAU,EACrB,KAAK,EAAE,WAAW,EAClB,oBAAoB;QACpB,cAAc;QACd,YAAY;QACZ,aAAa,IACV,KAAK,CAAC,EAAE,CACZ,CAAC;IAEF,MAAM,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,CAAC,SAAS,EAA0B,CAAC;IAC9E,MAAM,aAAa,GAAG,aAAG,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,EAAE;QACrD,MAAM,EAAE,IAAI;QACZ,YAAY,EAAE,KAAK;QACnB,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE,EAAE;KACZ,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,kBAAG,CAAC,KAAK,CAAC,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,kBAAG,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC,CAAA,CAAC;AA7EW,QAAA,iBAAiB,qBA6E5B","sourcesContent":["import Eta from 'eta';\nimport fsp from 'fs/promises';\nimport path from 'path';\nimport millify from 'millify';\nimport { capitalize, cloneDeep, defaults, round, uniqBy } from 'lodash';\nimport type { DatasetPerfStat } from 'actor-spec';\n\nimport {\n  CrawleeOneReadmeTemplates,\n  README_HOOK,\n  README_HOOK_ENUM,\n  ReadmeFeature,\n  ReadmeFeatureType,\n  RenderContext,\n} from './types';\nimport type { CrawleeOneScraperActorSpec } from '../actorSpec';\n\nexport interface CrawleeOneReadmeTemplatesOverrides\n  extends Omit<CrawleeOneReadmeTemplates, 'features'> {\n  features?: Partial<Record<ReadmeFeatureType, Partial<ReadmeFeature>>>;\n}\n\n/**\n * Turn\n *\n * `[1, 2, 3, 4]`\n *\n * into\n *\n * `[{ item: 1, separator: ', '}, { item: 2, separator: ', '}, { item: 3, separator: ' or '}, { item: 4, separator: '' }]`\n *\n * So that the list can be rendered as:\n *\n * `1, 2, 3 or 4`\n */\nconst addListSeparators = <T>(arr: T[]) =>\n  arr.map((item, index) => ({\n    item,\n    separator: index === arr.length - 1 ? '' : index === arr.length - 2 ? ' or ' : ', ',\n  }));\n\n/**\n * Render\n *\n * `[1, 2, 3, 4]`\n *\n * as\n *\n * `1, 2, 3 or 4`\n */\nconst renderList = (arr: any[]) => addListSeparators(arr)\n  .map((d) => `${d.item}${d.separator}`)\n  .join(''); // prettier-ignore\n\ninterface TimeDelta {\n  hours: number;\n  minutes: number;\n  seconds: number;\n}\n\n/** Given time in seconds, get the hours, minutes and seconds */\nconst timeDeltaFromSec = (timeInSec: number): TimeDelta => {\n  const secInMin = 60;\n  const secInHour = 60 * 60;\n  let remainingTimeInSec = timeInSec;\n\n  const hours = Math.floor(remainingTimeInSec / secInHour);\n  remainingTimeInSec = remainingTimeInSec - hours * secInHour;\n\n  const minutes = Math.floor(remainingTimeInSec / secInMin);\n  const seconds = remainingTimeInSec - minutes * secInMin;\n  return { hours, minutes, seconds };\n};\n\n/** Format time delta as \"6h 2m 23s\" or \"2m 23s\" */\nconst renderTimeDelta = ({ hours, minutes, seconds }: TimeDelta) => {\n  return [hours ? `${hours}h` : '', minutes ? `${minutes}m` : '', hours ? '' : `${seconds}s`].join(' '); // prettier-ignore\n};\n\n/** Render perf stat as \"$0.016 in 0h 2m 23s\" */\nconst renderPerfStat = (perf: DatasetPerfStat) => {\n  const timeText = renderTimeDelta(timeDeltaFromSec(perf.timeSec));\n  return `$${round(perf.costUsd, 3)} in ${timeText}`;\n};\n\n/** Render email safely as name[dot]surname[at]domain[dot]com */\nconst renderEmail = (email: string) => email.replace(/\\./g, '[dot]').replace(/\\@/g, '[at]');\n\nconst renderJson = (s: string, options: { paddingLeft?: number; paddingStartLine?: number } = {}) =>\n  JSON.stringify(s, null, 2)\n    .split('\\n')\n    .map((s, index) => {\n      const shouldApplyPadding = index >= (options.paddingStartLine || 0);\n      const paddingLeftRepeats = shouldApplyPadding ? options.paddingLeft || 0 : 0;\n      return `${' '.repeat(paddingLeftRepeats)}${s}`;\n    })\n    .join('\\n');\n\nconst includesPersonalData = (it) => it.a.datasets.some((d) => d.privacy.personalDataFields.length);\nconst collectFilters = (it: any) => uniqBy(it.a.datasets.flatMap((d) => d.filters), m => m); // prettier-ignore\nconst collectModes = (it: any) => uniqBy(it.a.datasets.flatMap((d) => d.modes), (m: any) => m.name); // prettier-ignore\nconst collectEmails = (it: any) => uniqBy(it.a.authors.flatMap((a) => a.email), e => e); // prettier-ignore\n\n// TODO\n/** Define the texts in features sections that are, by default, common across all actors */\nexport const defaultFeatureTexts: CrawleeOneReadmeTemplates['features'] = {\n  datasets: {\n    supported: (it) => it.a.datasets.length > 1,\n    title: '<%~ it.a.datasets.length %> kinds of datasets',\n    mainText: '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.',\n  },\n  modes: {\n    supported: (it) => it.fn.collectModes(it).length > 1,\n    title: `<%~ it.fn.enumerate(it.fn.collectModes(it).map((m) => it.fn.capitalize(m.name))) %> modes`,\n    mainText:\n      `- Scraping can be ` +\n      `<%~ it.fn.enumerate(it.fn.collectModes(it).map((m) => m.name.toLowerCase() + ' (' + m.shortDesc + ')')) %>` +\n      `.`,\n  },\n  filters: {\n    supported: (it) => it.a.datasets.some((d) => d.filters.length),\n    data: {\n      maxEntriesSupported: true,\n    },\n    title: 'Filter support',\n    mainText:\n      `- Filter the results by <%~ it.fn.enumerate(it.fn.collectFilters(it)) %>.\\n` +\n      `<% if (it.t.features.filters.data.maxEntriesSupported) { %>\\n` +\n      `  - Limit the number of results.\\n` +\n      `<% } %>`,\n  },\n  noBrowser: {\n    supported: (it) => it.a.datasets.some((d) => !d.features.usesBrowser),\n    title: 'Blazing fast',\n    mainText: `- The actor doesn't use a browser, which means it's fast and cheap.`,\n  },\n  proxy: {\n    supported: (it) => it.a.datasets.some((d) => d.features.proxySupport),\n    title: 'Proxy support',\n    mainText: `- You can use Apify's proxy, or your own, via Input.`,\n  },\n  integratedETL: {\n    supported: (it) => it.a.datasets.some((d) => d.features.integratedETL),\n    title: 'Integrated data filtering and transformation',\n    mainText: `- Filter and modify scraped entries out of the box from within Apify UI, without needing other tools.`,\n  },\n  integratedCache: {\n    supported: (it) => it.a.datasets.some((d) => d.features.integratedCache),\n    title: 'Integrated cache',\n    mainText:\n      `- You can use cache together with custom filtering to e.g. save only NEW entries to the dataset. Save time and reduce cost.\\n` +\n      `  - Cache automatically stores which entries were already scraped. Cache can persist between different scraper runs.`,\n  },\n  crawlerConfig: {\n    supported: (it) => it.a.datasets.some((d) => d.features.configurable),\n    title: 'Custom crawler configuration',\n    mainText: `- For advanced needs, you can pass Crawler configuration via Input.`,\n  },\n  tests: {\n    supported: (it) => it.a.datasets.some((d) => d.features.regularlyTested),\n    title: 'Tested daily for high reliability',\n    mainText: `- The actor is regularly tested end-to-end to minimize the risk of a broken integration.`,\n  },\n  privacy: {\n    supported: (it) => it.a.datasets.some((d) => d.features.privacyCompliance),\n    title: 'Privacy-compliant (GDPR)',\n    mainText: `- By default, personal data is redacted to avoid privacy issues. You can opt-in to include un-censored data.`,\n  },\n  metamorph: {\n    supported: () => true,\n    title: 'Pass scraped dataset to other actors',\n    mainText:\n      `- Automatically trigger another actor when this one is done to process the scraped dataset.\\n` +\n      `  - Metamorphing means that the dataset and key-value store is passed to another actor.\\n` +\n      `  - Actor metamorph can be configure via actor input. No need to define custom actors just for that.`,\n  },\n  errorMonitoring: {\n    supported: (it) => it.a.datasets.some((d) => d.features.errorMonitoring),\n    data: {\n      hasSentry: true,\n    },\n    title: 'Error monitoring',\n    mainText:\n      `- Errors from your runs are captured and surfaced in the \\`REPORTING\\` dataset. (See Storage > Dataset > Select dropdown).\\n` +\n      `<% if (it.t.features.errorMonitoring.data.hasSentry) { %>\\n` +\n      `  - Errors are also automatically reported to [Sentry](https://sentry.io/).\\n` +\n      `<% } %>`,\n  },\n};\n\nconst H = README_HOOK_ENUM;\n\n/** The template for rendering README for crawler */\nconst readmeTemplate = `\n<%~ it.a.actor.title %>\n===============================\n\n<%~ it.a.actor.shortDesc %>\n\n## What is <%~ it.a.actor.title %> and how it works?\n\n<%~ include(\"hook.${H.introAfterBegin}\", it) %>\n\nWith <%~ it.a.actor.title %>, you can extract:\n\n<%- it.a.datasets.forEach((dataset) => { %>\n- [<%~ dataset.shortDesc %>]( <%~ dataset.url %> )\n<%- }) %>\n\n<%-~ include(\"hook.${H.introAfterDatasets}\", it) %>\n\nSee the [outputs section](#outputs) for a detailed description.\n\nThe data can be downloaded in JSON, JSONL, XML, CSV, Excel, or HTML formats.\n\n<%~ include(\"hook.${H.introBeforeEnd}\", it) %>\n\n## Features\n\n<%~ include(\"hook.${H.featuresAfterBegin}\", it) %>\n\nThis actor is a robust production-grade solution suitable for businesses and those that need reliability.\n\n<%- Object.entries(it.t.features).forEach(([featName, feat]) => { %>\n<%- if (feat.supported(it)) { %>\n- **<%~ include(\"feat.\" + featName + \".title\", it) %>**\n<% if (feat.afterBegin) { -%>\n  <%~ include(\"feat.\" + featName + \".afterBegin\", it) %>\n<%- } -%>\n  \n  <%-~ include(\"feat.\" + featName + \".mainText\", it) -%>\n\n<% if (feat.beforeEnd) { -%>\n  <%~ include(\"feat.\" + featName + \".beforeEnd\", it) %>\n<%- } -%>\n<%- } %>\n<%- }) %>\n\n<%~ include(\"hook.${H.featuresBeforeEnd}\", it) %>\n\n## How can you use the data scraped from <%~ it.a.websites[0].name %>? (Examples)\n\n<%~ include(\"hook.${H.useCases}\", it) %>\n\n## How to use <%~ it.a.actor.title %>\n\n<%~ include(\"hook.${H.usageAfterBegin}\", it) %>\n\n1. Create a free Apify account using your email\n2. Open <%~ it.a.actor.title %>\n3. In Input, select the dataset to scrape, and filters to apply.\n4. Click \"Start\" and wait for the data to be extracted.\n5. Download your data in JSON, JSONL, XML, CSV, Excel, or HTML format.\n\nFor details and examples for all input fields, please visit the [Input tab](<%~ it.a.actor.publicUrl %>/input-schema).\n\n<%~ include(\"hook.${H.usageBeforeEnd}\", it) %>\n\n## How much does it cost to scrape <%~ it.a.websites[0].name %>?\n\n<%~ include(\"hook.${H.costAfterBegin}\", it) %>\n\n<%- it.a.datasets.filter((d) => d.perfStats && d.perfStats.length).forEach((dataset) => { %>\n### <%~ it.fn.capitalize(dataset.name) %>\n\n<table>\n  <thead>\n    <tr>\n      <td></td>\n      <%- it.t.perfTables[dataset.perfTable].cols.forEach((col) => { -%>\n      <td><strong>\n        <%~ include(\"perfTable.\" + dataset.perfTable + \".col.\" + col.colId, { ...it, dataset }) %>\n      </strong></td>\n      <%- }) -%>\n    </tr>\n  </thead>\n\n  <tbody>\n    <%- it.t.perfTables[dataset.perfTable].rows.forEach((row) => { -%>\n    <tr>\n      <td>\n        <%~ include(\"perfTable.\" + dataset.perfTable + \".row.\" + row.rowId, { ...it, dataset }) %>\n      </td>\n      <%- it.t.perfTables[dataset.perfTable].cols.forEach((col) => { -%>\n      <td>\n        <%~ it.fn.perfStat(\n          dataset.perfStats.find(d => d.rowId === row.rowId && d.colId === col.colId)\n        ) %>\n      </td>\n      <%- }) -%>\n    </tr>\n    <%- }) -%>\n  </tbody>\n</table>\n\n<%- }) %>\n\n<br/>\n\n<%~ include(\"hook.${H.costAfterPerfTables}\", it) %>\n\nRemember that with the [Apify Free plan](https://apify.com/pricing), you have $5 free usage per month.\n\n<%~ include(\"hook.${H.costBeforeEnd}\", it) %>\n\n## Input options\n\n<%~ include(\"hook.${H.inputAfterBegin}\", it) %>\n\nFor details and examples for all input fields, please visit the [Input tab](<%~ it.a.actor.publicUrl %>/input-schema).\n\n<%~ include(\"hook.${H.inputBeforeEnd}\", it) %>\n\n### Filter options\n\n<%~ include(\"hook.${H.filterAfterBegin}\", it) %>\n\nYou can run <%~ it.a.actor.title %> as is, with the default options, to get a sample of the \n<%~ it.a.datasets.find(d => d.isDefault).name %> entries\n<%_ if (it.fn.collectModes(it).find(m => m.isDefault)) { _%>\n<%_~ ' (' + it.fn.collectModes(it).find(m => m.isDefault).name %> mode)\n<%_ } _%>.\n\n<%- if (it.fn.collectFilters(it).length) { %>\nOtherwise, you can filter by:\n\n<%- it.fn.collectFilters(it).forEach((filter) => { %>\n  - <%~ it.fn.capitalize(filter) %>\n<%- }) %>\n<%- } %>\n\n<%~ include(\"hook.${H.filterBeforeEnd}\", it) %>\n\n### Limit options\n\n<%~ include(\"hook.${H.limitAfterBegin}\", it) %>\n\nTo limit how many results you get, set \\`<%~ it.t.input.maxCount %>\\` to desired amount.\n\n<%~ include(\"hook.${H.limitBeforeEnd}\", it) %>\n\n### Input examples\n\n<%~ include(\"hook.${H.inputExampleAfterBegin}\", it) %>\n\n<% it.t.exampleInputs.forEach((example, index) => { %>\n#### Example <%~ index + 1 %>: <%~ example.title %>\n\n\\`\\`\\`json\n{\n<%- Object.entries(example.inputData).forEach(([key, value]) => { -%>\n<%- if (example.inputDataComments && example.inputDataComments[key]) { %>\n  // <%~ example.inputDataComments[key] %>\n<%- } %>\n  \"<%~ key %>\": <% _%><%~ it.fn.stringify(value, { paddingLeft: 2, paddingStartLine: 1 }) %>,\n<%- }) %>\n}\n\\`\\`\\`\n\n<% }) %>\n\n<%~ include(\"hook.${H.inputExampleBeforeEnd}\", it) %>\n\n## Outputs\n\n<%~ include(\"hook.${H.outputAfterBegin}\", it) %>\n\nOnce the actor is done, you can see the overview of results in the Output tab.\n\nTo export the data, head over to the Storage tab.\n\n![<%~ it.a.actor.title %> dataset overview](<%~ it.a.actor.datasetOverviewImgUrl %>)\n\n<%~ include(\"hook.${H.outputBeforeEnd}\", it) %>\n\n## Sample output from <%~ it.a.actor.title %>\n\n<%~ include(\"hook.${H.outputExampleAfterBegin}\", it) %>\n\n<%- it.a.datasets.forEach((dataset) => { %>\n### <%~ it.fn.capitalize(dataset.name) %> output\n\n\\`\\`\\`json\n{\n<%- Object.entries(dataset.output.exampleEntry).forEach(([key, value]) => { -%>\n<%- if (dataset.output.exampleEntryComments && dataset.output.exampleEntryComments[key]) { %>\n  // <%~ dataset.output.exampleEntryComments[key] %>\n<%- } %>\n  \"<%~ key %>\": <% _%><%~ it.fn.stringify(value, { paddingLeft: 2, paddingStartLine: 1 }) %>,\n<%- }) %>\n}\n\\`\\`\\`\n\n<%- }) %>\n\n<%~ include(\"hook.${H.outputExampleBeforeEnd}\", it) %>\n\n## How to integrate <%~ it.a.actor.title %> with other services, APIs or Actors\n\n<%~ include(\"hook.${H.integrationAfterBegin}\", it) %>\n\nYou can connect the actor with many of the\n[integrations on the Apify platform](https://apify.com/integrations).\nYou can integrate with Make, Zapier, Slack, Airbyte, GitHub, Google Sheets, Google Drive,\n[and more](https://docs.apify.com/integrations).\nOr you can use\n[webhooks](https://docs.apify.com/integrations/webhooks)\nto carry out an action whenever an event occurs, e.g. get a notification whenever\nInstagram API Scraper successfully finishes a run.\n\n<%~ include(\"hook.${H.integrationBeforeEnd}\", it) %>\n\n## Use <%~ it.a.actor.title %> with Apify API\n\n<%~ include(\"hook.${H.apifyAfterBegin}\", it) %>\n\nThe Apify API gives you programmatic access to the Apify platform.\nThe API is organized around RESTful HTTP endpoints that enable you to manage,\nschedule and run Apify actors. The API also lets you access any datasets,\nmonitor actor performance, fetch results, create and update versions, and more.\n\nTo access the API using Node.js, use the \\`apify-client\\` NPM package.\nTo access the API using Python, use the \\`apify-client\\` PyPI package.\n\nCheck out the [Apify API reference](https://docs.apify.com/api/v2) docs\nfor full details or click on the\n[API tab](<%~ it.a.actor.publicUrl %>/api)\nfor code examples.\n\n<%~ include(\"hook.${H.apifyBeforeEnd}\", it) %>\n\n## Is it legal to scrape <%~ it.a.websites[0].name %>?\n\n<%~ include(\"hook.${H.legalityAfterBegin}\", it) %>\n\nIt is legal to scrape publicly available data such as product descriptions,\nprices, or ratings. Read Apify's blog post on\n[the legality of web scraping](https://blog.apify.com/is-web-scraping-legal/)\nto learn more.\n\n<%- if (it.fn.includesPersonalData(it)) { %>\nHowever, the following datasets include personal data:\n\n<%- it.a.datasets.filter((d) => d.privacy.personalDataFields.length).forEach((dataset) => { %>\n- <%~ it.fn.capitalize(dataset.name) %> dataset includes info about <%~ it.fn.enumerate(dataset.privacy.personalDataSubjects) %>.\n  - Fields: <%~ dataset.privacy.personalDataFields.join(', ') %>\n<% if (dataset.privacy.isPersonalDataRedacted) { -%>\n  - By default, this personal data is redacted, and in such case, it's safe to scrape the data.\n<%- } %>\n<%- }) %>\n\nTo get the unredacted data, toggle on the \"<%~ it.t.input.privacyName %>\" actor input.\n\n> **Warning:** Including personal data is done at your own risk. It is your\nresponsibility to make sure you have obtained consent or have a legal basis\nfor using the data.\n>\n> By using this actor, you agree not to hold the author of this actor liable for privacy\nor data-related issues that may arise during its use.\n\nRedacted fields may show a message like this instead of the actual value:\n\n\\`\\`\\`txt\n<Redacted property \"email\". To include the actual value, toggle ON the Actor input option \"<%~ it.t.input.privacyName %>\">\n\\`\\`\\`\n\n<%- } %>\n\n<%~ include(\"hook.${H.legalityBeforeEnd}\", it) %>\n\n## Who can I contact for issues with <%~ it.a.websites[0].name %> actor?\n\n<%~ include(\"hook.${H.contactAfterBegin}\", it) %>\n\nTo report issues and find help,\n<%- if (it.a.platform.socials.discord) { %>\nhead over to the\n[Discord community](<%~ it.a.platform.socials.discord %>)\n<%- } %>\n<%_ if (it.a.platform.socials.discord && it.fn.collectEmails(it).length) { %>, or <% } _%>\nemail me at <%~ it.fn.email(it.fn.collectEmails(it)[0]) %>\n\n<%~ include(\"hook.${H.contactBeforeEnd}\", it) %>\n`;\n\n/**\n * Render a README.md file from a common template for a given Apify crawler.\n *\n * See https://docs.apify.com/academy/get-most-of-actors/actor-readme\n *\n * The templates are rendered using ETA (https://eta.js.org/)\n *\n * Each template has access to `it` global variable. `it` has these props:\n *\n * - `it.fn` - The functions passed to this function + more (see below)\n * - `it.t` - The templates object passed to this function\n * - `it.a` - The actorSpec object passed to this function\n *\n * Example:\n * ```eta\n * ActorId: <%~ it.a.platform.actorId %>\n * ```\n *\n * Following functions are available by default:\n * - `it.fn.enumerate`\n * - `it.fn.perfStat`\n * - `it.fn.millify`\n * - `it.fn.capitalize`\n * - `it.fn.stringify`\n * - `it.fn.email`\n * - `it.fn.includesPersonalData`\n * - `it.fn.collectFilters`\n * - `it.fn.collectModes`\n * - `it.fn.collectEmails`\n *\n * See their definitions for details\n */\nexport const renderApifyReadme = async (input: {\n  /** Filepath (relative to CWD) where the generated README should be written. */\n  filepath: string;\n  /**\n   * Info about a particular actor.\n   *\n   * Inside the template during rendering, this object\n   * can be accessed as `<%~ it.a.platform.actorId %>`\n   */\n  actorSpec: CrawleeOneScraperActorSpec;\n  /**\n   * Custom eta template strings that plug into different\n   * parts of the README template.\n   *\n   * Inside the template during rendering, these templates\n   * can be accessed as `<%~ it.t.someTemplate %>`\n   */\n  templates: CrawleeOneReadmeTemplatesOverrides;\n  /**\n   * Functions to be made available in the template.\n   *\n   * Inside the template during rendering, these functions\n   * can be accessed as `<%~ it.fn.funcName() %>`\n   */\n  fn?: Record<string, (...args: any[]) => any>;\n}) => {\n  // Assign the default values to a clone\n  const templates = cloneDeep(input.templates) as CrawleeOneReadmeTemplates;\n  templates.features = templates.features || {};\n  Object.entries(defaultFeatureTexts).forEach(([key, feat]) => {\n    templates.features[key] = defaults(templates.features[key] || {}, feat);\n  });\n\n  // Define templates for 'include(...)'s for template hooks\n  README_HOOK.forEach((key) => {\n    const template = templates.hooks?.[key];\n    Eta.templates.define(`hook.${key}`, Eta.compile(template || ''));\n  });\n  // Define templates for 'include(...)'s for feature hooks\n  Object.entries(templates.features).forEach(([key, feat]) => {\n    const { title, mainText, afterBegin, beforeEnd } = feat;\n    Eta.templates.define(`feat.${key}.title`, Eta.compile(title));\n    Eta.templates.define(`feat.${key}.mainText`, Eta.compile(mainText));\n    Eta.templates.define(`feat.${key}.afterBegin`, Eta.compile(afterBegin ?? ''));\n    Eta.templates.define(`feat.${key}.beforeEnd`, Eta.compile(beforeEnd ?? ''));\n  });\n  // Define templates for 'include(...)'s for perf table hooks\n  Object.entries(templates.perfTables || {}).forEach(([key, perfTable]) => {\n    perfTable.rows.forEach((row) => Eta.templates.define(`perfTable.${key}.row.${row.rowId}`, Eta.compile(row.template))); // prettier-ignore\n    perfTable.cols.forEach((col) => Eta.templates.define(`perfTable.${key}.col.${col.colId}`, Eta.compile(col.template))); // prettier-ignore\n  });\n\n  const fn = {\n    enumerate: renderList,\n    perfStat: renderPerfStat,\n    millify,\n    capitalize,\n    stringify: renderJson,\n    email: renderEmail,\n    includesPersonalData,\n    collectFilters,\n    collectModes,\n    collectEmails,\n    ...input.fn,\n  };\n\n  const data = { fn, t: templates, a: input.actorSpec } satisfies RenderContext;\n  const readmeContent = Eta.render(readmeTemplate, data, {\n    strict: true,\n    rmWhitespace: false,\n    autoTrim: false,\n    context: {},\n  });\n\n  const readmePath = path.resolve(process.cwd(), input.filepath);\n  await fsp.mkdir(path.dirname(readmePath), { recursive: true });\n  await fsp.writeFile(readmePath, readmeContent, 'utf-8');\n};\n"]}