@crawlee/templates 3.0.5-beta.9 → 3.1.0

package/manifest.json

@@ -1,36 +0,0 @@
-{
-    "templates": [
-        {
-            "name": "getting-started-ts",
-            "description": "Getting started example [TypeScript]"
-        },
-        {
-            "name": "getting-started-js",
-            "description": "Getting started example [JavaScript]"
-        },
-        {
-            "name": "cheerio-ts",
-            "description": "CheerioCrawler template project [TypeScript]"
-        },
-        {
-            "name": "playwright-ts",
-            "description": "PlaywrightCrawler template project [TypeScript]"
-        },
-        {
-            "name": "puppeteer-ts",
-            "description": "PuppeteerCrawler template project [TypeScript]"
-        },
-        {
-            "name": "cheerio-js",
-            "description": "CheerioCrawler template project [JavaScript]"
-        },
-        {
-            "name": "playwright-js",
-            "description": "PlaywrightCrawler template project [JavaScript]"
-        },
-        {
-            "name": "puppeteer-js",
-            "description": "PuppeteerCrawler template project [JavaScript]"
-        }
-    ]
-}

package/templates/cheerio-js/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/cheerio-js/Dockerfile

@@ -1,29 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node:16
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY . ./
-
-
-# Run the image.
-CMD npm start --silent

package/templates/cheerio-js/README.md

@@ -1,27 +0,0 @@
-# Crawlee + CheerioCrawler + JavaScript project
-
-This template is a production ready boilerplate for developing with `CheerioCrawler`. Use this to bootstrap your projects using the most up-to-date code.
-
-If you're looking for examples or want to learn more visit:
-
-- [Tutorial](https://crawlee.dev/docs/guides/cheerio-crawler-guide)
-- [Documentation](https://crawlee.dev/api/cheerio-crawler/class/CheerioCrawler)
-- [Examples](https://crawlee.dev/docs/examples/cheerio-crawler)
-
-## Writing a README
-
-See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
-
-### Table of contents
-
-If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
-
-<!-- toc start -->
-- Introduction
-- Use Cases
-    - Case 1
-    - Case 2
-- Input
-- Output
-- Miscellaneous
- <!-- toc end -->

package/templates/cheerio-js/package.json

@@ -1,15 +0,0 @@
-{
-    "name": "crawlee-cheerio-js",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0"
-    },
-    "scripts": {
-        "start": "node src/main.js",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/cheerio-js/src/main.js

@@ -1,12 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
-import { router } from './routes.js';
-
-const startUrls = ['https://crawlee.dev'];
-
-const crawler = new CheerioCrawler({
-    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    requestHandler: router,
-});
-
-await crawler.run(startUrls);

package/templates/cheerio-js/src/routes.js

@@ -1,21 +0,0 @@
-import { Dataset, createCheerioRouter } from 'crawlee';
-
-export const router = createCheerioRouter();
-
-router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`enqueueing new URLs`);
-    await enqueueLinks({
-        globs: ['https://crawlee.dev/**'],
-        label: 'detail',
-    });
-});
-
-router.addHandler('detail', async ({ request, $, log }) => {
-    const title = $('title').text();
-    log.info(`${title}`, { url: request.loadedUrl });
-
-    await Dataset.pushData({
-        url: request.loadedUrl,
-        title,
-    });
-});

package/templates/cheerio-ts/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/cheerio-ts/Dockerfile

@@ -1,50 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node:16 AS builder
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY package*.json ./
-
-# Install all dependencies. Don't audit to speed up the installation.
-RUN npm install --include=dev --audit=false
-
-# Next, copy the source files using the user set
-# in the base image.
-COPY . ./
-
-# Install all dependencies and build the project.
-# Don't audit to speed up the installation.
-RUN npm run build
-
-# Create final image
-FROM apify/actor-node:16
-
-# Copy only built JS files from builder image
-COPY --from=builder /usr/src/app/dist ./dist
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY . ./
-
-
-# Run the image.
-CMD npm run start:prod --silent

package/templates/cheerio-ts/README.md

@@ -1,27 +0,0 @@
-# Crawlee + CheerioCrawler + TypeScript project
-
-This template is a production ready boilerplate for developing with `CheerioCrawler`. Use this to bootstrap your projects using the most up-to-date code.
-
-If you're looking for examples or want to learn more visit:
-
-- [Tutorial](https://crawlee.dev/docs/guides/cheerio-crawler-guide)
-- [Documentation](https://crawlee.dev/api/cheerio-crawler/class/CheerioCrawler)
-- [Examples](https://crawlee.dev/docs/examples/cheerio-crawler)
-
-## Writing a README
-
-See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
-
-### Table of contents
-
-If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
-
-<!-- toc start -->
-- Introduction
-- Use Cases
-    - Case 1
-    - Case 2
-- Input
-- Output
-- Miscellaneous
- <!-- toc end -->

package/templates/cheerio-ts/package.json

@@ -1,23 +0,0 @@
-{
-    "name": "crawlee-cheerio-ts",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0"
-    },
-    "devDependencies": {
-        "@apify/tsconfig": "^0.1.0",
-        "ts-node": "^10.8.0",
-        "typescript": "^4.7.4"
-    },
-    "scripts": {
-        "start": "npm run start:dev",
-        "start:prod": "node dist/main.js",
-        "start:dev": "ts-node-esm -T src/main.ts",
-        "build": "tsc",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/cheerio-ts/src/main.ts

@@ -1,12 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
-import { router } from './routes.js';
-
-const startUrls = ['https://crawlee.dev'];
-
-const crawler = new CheerioCrawler({
-    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    requestHandler: router,
-});
-
-await crawler.run(startUrls);

package/templates/cheerio-ts/src/routes.ts

@@ -1,21 +0,0 @@
-import { Dataset, createCheerioRouter } from 'crawlee';
-
-export const router = createCheerioRouter();
-
-router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`enqueueing new URLs`);
-    await enqueueLinks({
-        globs: ['https://crawlee.dev/**'],
-        label: 'detail',
-    });
-});
-
-router.addHandler('detail', async ({ request, $, log }) => {
-    const title = $('title').text();
-    log.info(`${title}`, { url: request.loadedUrl });
-
-    await Dataset.pushData({
-        url: request.loadedUrl,
-        title,
-    });
-});

package/templates/cheerio-ts/tsconfig.json

@@ -1,13 +0,0 @@
-{
-    "extends": "@apify/tsconfig",
-    "compilerOptions": {
-        "module": "ES2022",
-        "target": "ES2022",
-        "outDir": "dist",
-        "noUnusedLocals": false,
-        "lib": ["DOM"]
-    },
-    "include": [
-        "./src/**/*"
-    ]
-}

package/templates/getting-started-js/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/getting-started-js/Dockerfile

@@ -1,29 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY --chown=myuser . ./
-
-
-# Run the image.
-CMD npm start --silent

package/templates/getting-started-js/README.md

@@ -1,9 +0,0 @@
-# Getting started with Crawlee
-
-This example uses `PlaywrightCrawler` to recursively crawl https://crawlee.dev using the browser automation library [Playwright](https://playwright.dev).
-
-You can find more examples and documentation at the following links:
-
-- [Step-by-step tutorial](https://crawlee.dev/docs/introduction) for Crawlee
-- `PlaywrightCrawler` [API documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
-- Other [examples](https://crawlee.dev/docs/examples/playwright-crawler)

package/templates/getting-started-js/package.json

@@ -1,16 +0,0 @@
-{
-    "name": "crawlee-getting-started-js",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0",
-        "playwright": "*"
-    },
-    "scripts": {
-        "start": "node src/main.js",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/getting-started-js/src/main.js

@@ -1,24 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { PlaywrightCrawler, Dataset } from 'crawlee';
-
-// PlaywrightCrawler crawls the web using a headless
-// browser controlled by the Playwright library.
-const crawler = new PlaywrightCrawler({
-    // Use the requestHandler to process each of the crawled pages.
-    async requestHandler({ request, page, enqueueLinks, log }) {
-        const title = await page.title();
-        log.info(`Title of ${request.loadedUrl} is '${title}'`);
-
-        // Save results as JSON to ./storage/datasets/default
-        await Dataset.pushData({ title, url: request.loadedUrl });
-
-        // Extract links from the current page
-        // and add them to the crawling queue.
-        await enqueueLinks();
-    },
-    // Uncomment this option to see the browser window.
-    // headless: false,
-});
-
-// Add first URL to the queue and start the crawl.
-await crawler.run(['https://crawlee.dev']);

package/templates/getting-started-ts/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/getting-started-ts/Dockerfile

@@ -1,51 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16 AS builder
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install all dependencies. Don't audit to speed up the installation.
-RUN npm install --include=dev --audit=false
-
-# Next, copy the source files using the user set
-# in the base image.
-COPY --chown=myuser . ./
-
-# Install all dependencies and build the project.
-# Don't audit to speed up the installation.
-RUN npm run build
-
-# Create final image
-FROM apify/actor-node-playwright-chrome:16
-
-# Copy only built JS files from builder image
-COPY --from=builder --chown=myuser /home/myuser/dist ./dist
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY --chown=myuser . ./
-
-
-# Run the image. If you know you won't need headful browsers,
-# you can remove the XVFB start script for a micro perf gain.
-CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

package/templates/getting-started-ts/README.md

@@ -1,9 +0,0 @@
-# Getting started with Crawlee
-
-This example uses `PlaywrightCrawler` to recursively crawl https://crawlee.dev using the browser automation library [Playwright](https://playwright.dev).
-
-You can find more examples and documentation at the following links:
-
-- [Step-by-step tutorial](https://crawlee.dev/docs/introduction) for Crawlee
-- `PlaywrightCrawler` [API documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
-- Other [examples](https://crawlee.dev/docs/examples/playwright-crawler)

package/templates/getting-started-ts/package.json

@@ -1,24 +0,0 @@
-{
-    "name": "crawlee-getting-started-ts",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0",
-        "playwright": "*"
-    },
-    "devDependencies": {
-        "@apify/tsconfig": "^0.1.0",
-        "ts-node": "^10.8.0",
-        "typescript": "^4.7.4"
-    },
-    "scripts": {
-        "start": "npm run start:dev",
-        "start:prod": "node dist/main.js",
-        "start:dev": "ts-node-esm -T src/main.ts",
-        "build": "tsc",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/getting-started-ts/src/main.ts

@@ -1,24 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { PlaywrightCrawler, Dataset } from 'crawlee';
-
-// PlaywrightCrawler crawls the web using a headless
-// browser controlled by the Playwright library.
-const crawler = new PlaywrightCrawler({
-    // Use the requestHandler to process each of the crawled pages.
-    async requestHandler({ request, page, enqueueLinks, log }) {
-        const title = await page.title();
-        log.info(`Title of ${request.loadedUrl} is '${title}'`);
-
-        // Save results as JSON to ./storage/datasets/default
-        await Dataset.pushData({ title, url: request.loadedUrl });
-
-        // Extract links from the current page
-        // and add them to the crawling queue.
-        await enqueueLinks();
-    },
-    // Uncomment this option to see the browser window.
-    // headless: false,
-});
-
-// Add first URL to the queue and start the crawl.
-await crawler.run(['https://crawlee.dev']);

package/templates/getting-started-ts/tsconfig.json

@@ -1,13 +0,0 @@
-{
-    "extends": "@apify/tsconfig",
-    "compilerOptions": {
-        "module": "ES2022",
-        "target": "ES2022",
-        "outDir": "dist",
-        "noUnusedLocals": false,
-        "lib": ["DOM"]
-    },
-    "include": [
-        "./src/**/*"
-    ]
-}

package/templates/playwright-js/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/playwright-js/Dockerfile

@@ -1,29 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY --chown=myuser . ./
-
-
-# Run the image.
-CMD npm start --silent

package/templates/playwright-js/README.md

@@ -1,8 +0,0 @@
-# Crawlee + PlaywrightCrawler + JavaScript project
-
-This template is a production ready boilerplate for developing with `PlaywrightCrawler`. Use this to bootstrap your projects using the most up-to-date code.
-
-If you're looking for examples or want to learn more visit:
-
-- [Documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
-- [Examples](https://crawlee.dev/docs/examples/playwright-crawler)

package/templates/playwright-js/package.json

@@ -1,16 +0,0 @@
-{
-    "name": "crawlee-playwright-js",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0",
-        "playwright": "*"
-    },
-    "scripts": {
-        "start": "node src/main.js",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/playwright-js/src/main.js

@@ -1,12 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
-import { router } from './routes.js';
-
-const startUrls = ['https://crawlee.dev'];
-
-const crawler = new PlaywrightCrawler({
-    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    requestHandler: router,
-});
-
-await crawler.run(startUrls);