@crawlee/templates 3.0.0-beta.0

package/templates/cheerio-ts/README.md

@@ -0,0 +1,33 @@
+# Crawlee + CheerioCrawler + TypeScript project
+
+This template is a production ready boilerplate for developing with `CheerioCrawler`. Use this to bootstrap your projects using the most up-to-date code.
+
+If you're looking for examples or want to learn more visit:
+
+- [Tutorial](https://sdk.apify.com/docs/guides/getting-started#cheeriocrawler-aka-jquery-crawler)
+- [Documentation](https://sdk.apify.com/docs/api/cheerio-crawler)
+- [Examples](https://sdk.apify.com/docs/examples/cheerio-crawler)
+
+## Documentation reference
+
+- [Apify SDK](https://sdk.apify.com/)
+- [Apify Actor documentation](https://docs.apify.com/actor)
+- [Apify CLI](https://docs.apify.com/cli)
+
+## Writing a README
+
+See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
+
+### Table of contents
+
+If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
+
+<!-- toc start -->
+- Introduction
+- Use Cases
+    - Case 1
+    - Case 2
+- Input
+- Output
+- Miscellaneous
+ <!-- toc end -->

package/templates/cheerio-ts/package.json

@@ -0,0 +1,23 @@
+{
+    "name": "crawlee-cheerio-ts",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "next"
+    },
+    "devDependencies": {
+        "@apify/tsconfig": "^0.1.0",
+        "ts-node": "^10.8.0",
+        "typescript": "^4.7.4"
+    },
+    "scripts": {
+        "start": "npm run start:dev",
+        "start:prod": "node dist/main.js",
+        "start:dev": "ts-node-esm -T src/main.ts",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/cheerio-ts/src/main.ts

@@ -0,0 +1,28 @@
+// For more information, see https://crawlee.dev/
+import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';
+import { router } from './routes.js';
+
+interface InputSchema {
+    startUrls: string[];
+    debug?: boolean;
+}
+
+const { startUrls = [], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
+
+log.setLevel(log.LEVELS.DEBUG);
+if (debug) {
+    log.setLevel(log.LEVELS.DEBUG);
+}
+
+const crawler = new CheerioCrawler({
+    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+    // Be nice to the websites. Remove to unleash full power.
+    maxConcurrency: 50,
+    requestHandler: router,
+});
+
+await crawler.addRequests(startUrls);
+
+log.info('Starting the crawl.');
+await crawler.run();
+log.info('Crawl finished.');

package/templates/cheerio-ts/src/routes.ts

@@ -0,0 +1,21 @@
+import { createCheerioRouter, Dataset } from 'crawlee';
+
+export const router = createCheerioRouter();
+
+router.addDefaultHandler(async ({ enqueueLinks, log }) => {
+    log.info(`Handle Start URLs`);
+    await enqueueLinks({
+        globs: ['https://apify.com/*'],
+        label: 'DETAIL',
+    });
+});
+
+router.addHandler('LIST', async ({ log }) => {
+    log.info(`Handle pagination`);
+});
+
+router.addHandler('DETAIL', async ({ request, $, log }) => {
+    const title = $('title').text();
+    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
+    await Dataset.pushData({ url: request.loadedUrl });
+});

package/templates/cheerio-ts/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "@apify/tsconfig",
+    "compilerOptions": {
+        "module": "ES2022",
+        "target": "ES2022",
+        "outDir": "dist",
+        "noUnusedLocals": false,
+        "lib": ["DOM"]
+    },
+    "include": [
+        "./src/**/*"
+    ]
+}

package/templates/playwright-js/Dockerfile

@@ -0,0 +1,13 @@
+FROM apify/actor-node-playwright-chrome:16
+
+COPY . ./
+
+# install only prod deps
+RUN npm --quiet set progress=false \
+    && npm install --only=prod --no-optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --only=prod --no-optional --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version

package/templates/playwright-js/README.md

@@ -0,0 +1,32 @@
+# Crawlee + PlaywrightCrawler + JavaScript project
+
+This template is a production ready boilerplate for developing with `PlaywrightCrawler`. Use this to bootstrap your projects using the most up-to-date code.
+
+If you're looking for examples or want to learn more visit:
+
+- [Documentation](https://sdk.apify.com/docs/api/playwright-crawler)
+- [Examples](https://sdk.apify.com/docs/examples/playwright-crawler)
+
+## Documentation reference
+
+- [Apify SDK](https://sdk.apify.com/)
+- [Apify Actor documentation](https://docs.apify.com/actor)
+- [Apify CLI](https://docs.apify.com/cli)
+
+## Writing a README
+
+See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
+
+### Table of contents
+
+If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
+
+<!-- toc start -->
+- Introduction
+- Use Cases
+    - Case 1
+    - Case 2
+- Input
+- Output
+- Miscellaneous
+ <!-- toc end -->

package/templates/playwright-js/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "crawlee-playwright-js",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "next",
+        "playwright": "*"
+    },
+    "scripts": {
+        "start": "node src/main.js",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/playwright-js/src/main.js

@@ -0,0 +1,22 @@
+// For more information, see https://crawlee.dev/
+import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
+import { router } from './routes.js';
+
+const { startUrls = [], debug } = await KeyValueStore.getInput() ?? {};
+
+if (debug) {
+    log.setLevel(log.LEVELS.DEBUG);
+}
+
+const crawler = new PlaywrightCrawler({
+    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+    // Be nice to the websites. Remove to unleash full power.
+    maxConcurrency: 50,
+    requestHandler: router,
+});
+
+await crawler.addRequests(startUrls);
+
+log.info('Starting the crawl.');
+await crawler.run();
+log.info('Crawl finished.');

package/templates/playwright-js/src/routes.js

@@ -0,0 +1,21 @@
+import { Dataset, createPlaywrightRouter } from 'crawlee';
+
+export const router = createPlaywrightRouter();
+
+router.addDefaultHandler(async ({ enqueueLinks, log }) => {
+    log.info(`Handle Start URLs`);
+    await enqueueLinks({
+        globs: ['https://apify.com/*'],
+        label: 'DETAIL',
+    });
+});
+
+router.addHandler('handleList', async ({ log }) => {
+    log.info(`Handle pagination`);
+});
+
+router.addHandler('handleDetail', async ({ request, page, log }) => {
+    const title = await page.title();
+    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
+    await Dataset.pushData({ url: request.loadedUrl });
+});

package/templates/playwright-ts/Dockerfile

@@ -0,0 +1,27 @@
+# using multistage build, as we need dev deps to build the TS source code
+FROM apify/actor-node:16 AS builder
+
+# copy all files, install all dependencies (including dev deps) and build the project
+COPY . ./
+RUN npm install --include=dev \
+    && npm run build
+
+# create final image
+FROM apify/actor-node-playwright-chrome:16
+# copy only necessary files
+COPY --from=builder /usr/src/app/package.json ./
+COPY --from=builder /usr/src/app/README.md ./
+COPY --from=builder /usr/src/app/dist ./dist
+
+# install only prod deps
+RUN npm --quiet set progress=false \
+    && npm install --only=prod --no-optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --only=prod --no-optional --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+# run compiled code
+CMD npm run start:prod

package/templates/playwright-ts/README.md

@@ -0,0 +1,32 @@
+# Crawlee + PlaywrightCrawler + TypeScript project
+
+This template is a production ready boilerplate for developing with `PlaywrightCrawler`. Use this to bootstrap your projects using the most up-to-date code.
+
+If you're looking for examples or want to learn more visit:
+
+- [Documentation](https://sdk.apify.com/docs/api/playwright-crawler)
+- [Examples](https://sdk.apify.com/docs/examples/playwright-crawler)
+
+## Documentation reference
+
+- [Apify SDK](https://sdk.apify.com/)
+- [Apify Actor documentation](https://docs.apify.com/actor)
+- [Apify CLI](https://docs.apify.com/cli)
+
+## Writing a README
+
+See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
+
+### Table of contents
+
+If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
+
+<!-- toc start -->
+- Introduction
+- Use Cases
+    - Case 1
+    - Case 2
+- Input
+- Output
+- Miscellaneous
+ <!-- toc end -->

package/templates/playwright-ts/package.json

@@ -0,0 +1,24 @@
+{
+    "name": "crawlee-playwright-ts",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "next",
+        "playwright": "*"
+    },
+    "devDependencies": {
+        "@apify/tsconfig": "^0.1.0",
+        "ts-node": "^10.8.0",
+        "typescript": "^4.7.4"
+    },
+    "scripts": {
+        "start": "npm run start:dev",
+        "start:prod": "node dist/main.js",
+        "start:dev": "ts-node-esm -T src/main.ts",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/playwright-ts/src/main.ts

@@ -0,0 +1,27 @@
+// For more information, see https://crawlee.dev/
+import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
+import { router } from './routes.js';
+
+interface InputSchema {
+    startUrls: string[];
+    debug?: boolean;
+}
+
+const { startUrls = [], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
+
+if (debug) {
+    log.setLevel(log.LEVELS.DEBUG);
+}
+
+const crawler = new PlaywrightCrawler({
+    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+    // Be nice to the websites. Remove to unleash full power.
+    maxConcurrency: 50,
+    requestHandler: router,
+});
+
+await crawler.addRequests(startUrls);
+
+log.info('Starting the crawl.');
+await crawler.run();
+log.info('Crawl finished.');

package/templates/playwright-ts/src/routes.ts

@@ -0,0 +1,21 @@
+import { Dataset, createPlaywrightRouter } from 'crawlee';
+
+export const router = createPlaywrightRouter();
+
+router.addDefaultHandler(async ({ enqueueLinks, log }) => {
+    log.info(`Handle Start URLs`);
+    await enqueueLinks({
+        globs: ['https://apify.com/*'],
+        label: 'DETAIL',
+    });
+});
+
+router.addHandler('handleList', async ({ log }) => {
+    log.info(`Handle pagination`);
+});
+
+router.addHandler('handleDetail', async ({ request, page, log }) => {
+    const title = await page.title();
+    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
+    await Dataset.pushData({ url: request.loadedUrl });
+});

package/templates/playwright-ts/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "@apify/tsconfig",
+    "compilerOptions": {
+        "module": "ES2022",
+        "target": "ES2022",
+        "outDir": "dist",
+        "noUnusedLocals": false,
+        "lib": ["DOM"]
+    },
+    "include": [
+        "./src/**/*"
+    ]
+}

package/templates/puppeteer-js/Dockerfile

@@ -0,0 +1,13 @@
+FROM apify/actor-node-puppeteer-chrome:16
+
+COPY . ./
+
+# install only prod deps
+RUN npm --quiet set progress=false \
+    && npm install --only=prod --no-optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --only=prod --no-optional --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version

package/templates/puppeteer-js/README.md

@@ -0,0 +1,32 @@
+# Crawlee + PuppeteerCrawler + JavaScript project
+
+This template is a production ready boilerplate for developing with `PuppeteerCrawler`. Use this to bootstrap your projects using the most up-to-date code.
+
+If you're looking for examples or want to learn more visit:
+
+- [Documentation](https://sdk.apify.com/docs/api/puppeteer-crawler)
+- [Examples](https://sdk.apify.com/docs/examples/puppeteer-crawler)
+
+## Documentation reference
+
+- [Apify SDK](https://sdk.apify.com/)
+- [Apify Actor documentation](https://docs.apify.com/actor)
+- [Apify CLI](https://docs.apify.com/cli)
+
+## Writing a README
+
+See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
+
+### Table of contents
+
+If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
+
+<!-- toc start -->
+- Introduction
+- Use Cases
+    - Case 1
+    - Case 2
+- Input
+- Output
+- Miscellaneous
+ <!-- toc end -->

package/templates/puppeteer-js/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "crawlee-puppeteer-js",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "next",
+        "puppeteer": "*"
+    },
+    "scripts": {
+        "start": "node src/main.js",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/puppeteer-js/src/main.js

@@ -0,0 +1,22 @@
+// For more information, see https://crawlee.dev/
+import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
+import { router } from './routes.js';
+
+const { startUrls = [], debug } = await KeyValueStore.getInput() ?? {};
+
+if (debug) {
+    log.setLevel(log.LEVELS.DEBUG);
+}
+
+const crawler = new PuppeteerCrawler({
+    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+    // Be nice to the websites. Remove to unleash full power.
+    maxConcurrency: 50,
+    requestHandler: router,
+});
+
+await crawler.addRequests(startUrls);
+
+log.info('Starting the crawl.');
+await crawler.run();
+log.info('Crawl finished.');

package/templates/puppeteer-js/src/routes.js

@@ -0,0 +1,21 @@
+import { Dataset, createPuppeteerRouter } from 'crawlee';
+
+export const router = createPuppeteerRouter();
+
+router.addDefaultHandler(async ({ enqueueLinks, log }) => {
+    log.info(`Handle Start URLs`);
+    await enqueueLinks({
+        globs: ['https://apify.com/*'],
+        label: 'DETAIL',
+    });
+});
+
+router.addHandler('handleList', async ({ log }) => {
+    log.info(`Handle pagination`);
+});
+
+router.addHandler('handleDetail', async ({ request, page, log }) => {
+    const title = await page.title();
+    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
+    await Dataset.pushData({ url: request.loadedUrl });
+});

package/templates/puppeteer-ts/Dockerfile

@@ -0,0 +1,27 @@
+# using multistage build, as we need dev deps to build the TS source code
+FROM apify/actor-node:16 AS builder
+
+# copy all files, install all dependencies (including dev deps) and build the project
+COPY . ./
+RUN npm install --include=dev \
+    && npm run build
+
+# create final image
+FROM apify/actor-node-puppeteer-chrome:16
+# copy only necessary files
+COPY --from=builder /usr/src/app/package.json ./
+COPY --from=builder /usr/src/app/README.md ./
+COPY --from=builder /usr/src/app/dist ./dist
+
+# install only prod deps
+RUN npm --quiet set progress=false \
+    && npm install --only=prod --no-optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --only=prod --no-optional --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+# run compiled code
+CMD npm run start:prod

package/templates/puppeteer-ts/README.md

@@ -0,0 +1,32 @@
+# Crawlee + PuppeteerCrawler + TypeScript project
+
+This template is a production ready boilerplate for developing with `PuppeteerCrawler`. Use this to bootstrap your projects using the most up-to-date code.
+
+If you're looking for examples or want to learn more visit:
+
+- [Documentation](https://sdk.apify.com/docs/api/puppeteer-crawler)
+- [Examples](https://sdk.apify.com/docs/examples/puppeteer-crawler)
+
+## Documentation reference
+
+- [Apify SDK](https://sdk.apify.com/)
+- [Apify Actor documentation](https://docs.apify.com/actor)
+- [Apify CLI](https://docs.apify.com/cli)
+
+## Writing a README
+
+See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
+
+### Table of contents
+
+If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
+
+<!-- toc start -->
+- Introduction
+- Use Cases
+    - Case 1
+    - Case 2
+- Input
+- Output
+- Miscellaneous
+ <!-- toc end -->

package/templates/puppeteer-ts/package.json

@@ -0,0 +1,24 @@
+{
+    "name": "crawlee-puppeteer-ts",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "next",
+        "puppeteer": "*"
+    },
+    "devDependencies": {
+        "@apify/tsconfig": "^0.1.0",
+        "ts-node": "^10.8.0",
+        "typescript": "^4.7.4"
+    },
+    "scripts": {
+        "start": "npm run start:dev",
+        "start:prod": "node dist/main.js",
+        "start:dev": "ts-node-esm -T src/main.ts",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/puppeteer-ts/src/main.ts

@@ -0,0 +1,27 @@
+// For more information, see https://crawlee.dev/
+import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
+import { router } from './routes.js';
+
+interface InputSchema {
+    startUrls: string[];
+    debug?: boolean;
+}
+
+const { startUrls = [], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
+
+if (debug) {
+    log.setLevel(log.LEVELS.DEBUG);
+}
+
+const crawler = new PuppeteerCrawler({
+    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+    // Be nice to the websites. Remove to unleash full power.
+    maxConcurrency: 50,
+    requestHandler: router,
+});
+
+await crawler.addRequests(startUrls);
+
+log.info('Starting the crawl.');
+await crawler.run();
+log.info('Crawl finished.');

package/templates/puppeteer-ts/src/routes.ts

@@ -0,0 +1,21 @@
+import { Dataset, createPuppeteerRouter } from 'crawlee';
+
+export const router = createPuppeteerRouter();
+
+router.addDefaultHandler(async ({ enqueueLinks, log }) => {
+    log.info(`Handle Start URLs`);
+    await enqueueLinks({
+        globs: ['https://apify.com/*'],
+        label: 'DETAIL',
+    });
+});
+
+router.addHandler('handleList', async ({ log }) => {
+    log.info(`Handle pagination`);
+});
+
+router.addHandler('handleDetail', async ({ request, page, log }) => {
+    const title = await page.title();
+    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
+    await Dataset.pushData({ url: request.loadedUrl });
+});

package/templates/puppeteer-ts/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "@apify/tsconfig",
+    "compilerOptions": {
+        "module": "ES2022",
+        "target": "ES2022",
+        "outDir": "dist",
+        "noUnusedLocals": false,
+        "lib": ["DOM"]
+    },
+    "include": [
+        "./src/**/*"
+    ]
+}