@crawlee/templates 3.0.4-beta.5 → 3.0.4-beta.8

package/manifest.json

@@ -1,28 +1,36 @@
 {
     "templates": [
+        {
+            "name": "getting-started-ts",
+            "description": "Getting started example [TypeScript]"
+        },
+        {
+            "name": "getting-started-js",
+            "description": "Getting started example [JavaScript]"
+        },
         {
             "name": "cheerio-ts",
-            "description": "Crawlee cheerio template [TypeScript]"
+            "description": "CheerioCrawler template project [TypeScript]"
         },
         {
             "name": "playwright-ts",
-            "description": "Crawlee playwright template [TypeScript]"
+            "description": "PlaywrightCrawler template project [TypeScript]"
         },
         {
             "name": "puppeteer-ts",
-            "description": "Crawlee puppeteer template [TypeScript]"
+            "description": "PuppeteerCrawler template project [TypeScript]"
         },
         {
             "name": "cheerio-js",
-            "description": "Crawlee cheerio template [JavaScript]"
+            "description": "CheerioCrawler template project [JavaScript]"
         },
         {
             "name": "playwright-js",
-            "description": "Crawlee playwright template [JavaScript]"
+            "description": "PlaywrightCrawler template project [JavaScript]"
         },
         {
             "name": "puppeteer-js",
-            "description": "Crawlee puppeteer template [JavaScript]"
+            "description": "PuppeteerCrawler template project [JavaScript]"
         }
     ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/templates",
-    "version": "3.0.4-beta.5",
+    "version": "3.0.4-beta.8",
     "description": "Templates for the crawlee projects",
     "engines": {
         "node": ">=16.0.0"

package/templates/cheerio-js/.dockerignore

@@ -0,0 +1,10 @@
+# configurations
+.idea
+
+# crawlee and apify storage folders
+apify_storage
+crawlee_storage
+storage
+
+# installed files
+node_modules

package/templates/cheerio-js/Dockerfile

@@ -1,15 +1,29 @@
+# Specify the base Docker image. You can read more about
+# the available images at https://crawlee.dev/docs/guides/docker-images
+# You can also use any other image from Docker Hub.
 FROM apify/actor-node:16
 
-# copy package.json, lock file and possibly crawlee.json
-COPY package*.json crawlee?json ./
-COPY src ./src
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY package*.json ./
 
-# install only prod deps
+# Install NPM packages, skip optional and development dependencies to
+# keep the image small. Avoid logging too much and print the dependency
+# tree for debugging
 RUN npm --quiet set progress=false \
-    && npm install --only=prod --no-optional \
+    && npm install --omit=dev --omit=optional \
     && echo "Installed NPM packages:" \
-    && (npm list --only=prod --no-optional --all || true) \
+    && (npm list --omit=dev --all || true) \
     && echo "Node.js version:" \
     && node --version \
     && echo "NPM version:" \
     && npm --version
+
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after NPM install, quick build will be really fast
+# for most source file changes.
+COPY . ./
+
+
+# Run the image.
+CMD npm start --silent

package/templates/cheerio-js/package.json

@@ -8,7 +8,6 @@
     },
     "scripts": {
         "start": "node src/main.js",
-        "build": "tsc",
         "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
     },
     "author": "It's not you it's me",

package/templates/cheerio-js/src/routes.js

@@ -3,19 +3,19 @@ import { Dataset, createCheerioRouter } from 'crawlee';
 export const router = createCheerioRouter();
 
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`Handle Start URLs`);
+    log.info(`enqueueing new URLs`);
     await enqueueLinks({
         globs: ['https://crawlee.dev/**'],
-        label: 'DETAIL',
+        label: 'detail',
     });
 });
 
-router.addHandler('LIST', async ({ log }) => {
-    log.info(`Handle pagination`);
-});
-
-router.addHandler('DETAIL', async ({ request, $, log }) => {
+router.addHandler('detail', async ({ request, $, log }) => {
     const title = $('title').text();
-    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
-    await Dataset.pushData({ url: request.loadedUrl });
+    log.info(`${title}`, { url: request.loadedUrl });
+
+    await Dataset.pushData({
+        url: request.loadedUrl,
+        title,
+    });
 });

package/templates/cheerio-ts/.dockerignore

@@ -0,0 +1,10 @@
+# configurations
+.idea
+
+# crawlee and apify storage folders
+apify_storage
+crawlee_storage
+storage
+
+# installed files
+node_modules

package/templates/cheerio-ts/Dockerfile

@@ -1,32 +1,50 @@
-# First, specify the base Docker image. You can read more about
+# Specify the base Docker image. You can read more about
 # the available images at https://crawlee.dev/docs/guides/docker-images
 # You can also use any other image from Docker Hub.
 FROM apify/actor-node:16 AS builder
 
-# Second, copy the TS source files, package.json and package-lock.json since it should be
-# the only files that affects the installation and build in the next step
-COPY package*.json tsconfig.json crawlee?json ./
-COPY src ./src
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY package*.json ./
 
-# Install all dependencies (including dev deps) and build the project
-RUN npm install --include=dev && npm run build
+# Install all dependencies. Don't audit to speed up the installation.
+RUN npm install --include=dev --audit=false
+
+# Next, copy the source files using the user set
+# in the base image.
+COPY . ./
+
+# Install all dependencies and build the project.
+# Don't audit to speed up the installation.
+RUN npm run build
 
 # Create final image
 FROM apify/actor-node:16
 
-# Copy only necessary files (crawlee.json might not exist, so we need the regexp trick with `crawlee?json`)
-COPY --from=builder /usr/src/app/package.json /usr/src/app/crawlee?json ./
+# Copy only built JS files from builder image
 COPY --from=builder /usr/src/app/dist ./dist
 
-# Install only prod deps
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY package*.json ./
+
+# Install NPM packages, skip optional and development dependencies to
+# keep the image small. Avoid logging too much and print the dependency
+# tree for debugging
 RUN npm --quiet set progress=false \
-    && npm install --only=prod --no-optional \
+    && npm install --omit=dev --omit=optional \
     && echo "Installed NPM packages:" \
-    && (npm list --only=prod --no-optional --all || true) \
+    && (npm list --omit=dev --all || true) \
     && echo "Node.js version:" \
     && node --version \
     && echo "NPM version:" \
     && npm --version
 
-# Run compiled code
-CMD npm run start:prod
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after NPM install, quick build will be really fast
+# for most source file changes.
+COPY . ./
+
+
+# Run the image.
+CMD npm run start:prod --silent

package/templates/cheerio-ts/src/routes.ts

@@ -1,21 +1,21 @@
-import { createCheerioRouter, Dataset } from 'crawlee';
+import { Dataset, createCheerioRouter } from 'crawlee';
 
 export const router = createCheerioRouter();
 
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`Handle Start URLs`);
+    log.info(`enqueueing new URLs`);
     await enqueueLinks({
         globs: ['https://crawlee.dev/**'],
-        label: 'DETAIL',
+        label: 'detail',
     });
 });
 
-router.addHandler('LIST', async ({ log }) => {
-    log.info(`Handle pagination`);
-});
-
-router.addHandler('DETAIL', async ({ request, $, log }) => {
+router.addHandler('detail', async ({ request, $, log }) => {
     const title = $('title').text();
-    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
-    await Dataset.pushData({ url: request.loadedUrl });
+    log.info(`${title}`, { url: request.loadedUrl });
+
+    await Dataset.pushData({
+        url: request.loadedUrl,
+        title,
+    });
 });

package/templates/getting-started-js/.dockerignore

@@ -0,0 +1,10 @@
+# configurations
+.idea
+
+# crawlee and apify storage folders
+apify_storage
+crawlee_storage
+storage
+
+# installed files
+node_modules

package/templates/getting-started-js/Dockerfile

@@ -0,0 +1,29 @@
+# Specify the base Docker image. You can read more about
+# the available images at https://crawlee.dev/docs/guides/docker-images
+# You can also use any other image from Docker Hub.
+FROM apify/actor-node-playwright-chrome:16
+
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY --chown=myuser package*.json ./
+
+# Install NPM packages, skip optional and development dependencies to
+# keep the image small. Avoid logging too much and print the dependency
+# tree for debugging
+RUN npm --quiet set progress=false \
+    && npm install --omit=dev --omit=optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --omit=dev --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after NPM install, quick build will be really fast
+# for most source file changes.
+COPY --chown=myuser . ./
+
+
+# Run the image.
+CMD npm start --silent

package/templates/getting-started-js/README.md

@@ -0,0 +1,9 @@
+# Getting started with Crawlee
+
+This example uses `PlaywrightCrawler` to recursively crawl https://crawlee.dev using the browser automation library [Playwright](https://playwright.dev).
+
+You can find more examples and documentation at the following links:
+
+- [Step-by-step tutorial](https://crawlee.dev/docs/introduction) for Crawlee
+- `PlaywrightCrawler` [API documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
+- Other [examples](https://crawlee.dev/docs/examples/playwright-crawler)

package/templates/getting-started-js/package.json

@@ -0,0 +1,16 @@
+{
+    "name": "crawlee-getting-started-js",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "^3.0.0",
+        "playwright": "*"
+    },
+    "scripts": {
+        "start": "node src/main.js",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/getting-started-js/src/main.js

@@ -0,0 +1,24 @@
+// For more information, see https://crawlee.dev/
+import { PlaywrightCrawler, Dataset } from 'crawlee';
+
+// PlaywrightCrawler crawls the web using a headless
+// browser controlled by the Playwright library.
+const crawler = new PlaywrightCrawler({
+    // Use the requestHandler to process each of the crawled pages.
+    async requestHandler({ request, page, enqueueLinks, log }) {
+        const title = await page.title();
+        log.info(`Title of ${request.loadedUrl} is '${title}'`);
+
+        // Save results as JSON to ./storage/datasets/default
+        await Dataset.pushData({ title, url: request.loadedUrl });
+
+        // Extract links from the current page
+        // and add them to the crawling queue.
+        await enqueueLinks();
+    },
+    // Uncomment this option to see the browser window.
+    // headless: false,
+});
+
+// Add first URL to the queue and start the crawl.
+await crawler.run(['https://crawlee.dev']);

package/templates/getting-started-ts/.dockerignore

@@ -0,0 +1,10 @@
+# configurations
+.idea
+
+# crawlee and apify storage folders
+apify_storage
+crawlee_storage
+storage
+
+# installed files
+node_modules

package/templates/getting-started-ts/Dockerfile

@@ -0,0 +1,51 @@
+# Specify the base Docker image. You can read more about
+# the available images at https://crawlee.dev/docs/guides/docker-images
+# You can also use any other image from Docker Hub.
+FROM apify/actor-node-playwright-chrome:16 AS builder
+
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY --chown=myuser package*.json ./
+
+# Install all dependencies. Don't audit to speed up the installation.
+RUN npm install --include=dev --audit=false
+
+# Next, copy the source files using the user set
+# in the base image.
+COPY --chown=myuser . ./
+
+# Install all dependencies and build the project.
+# Don't audit to speed up the installation.
+RUN npm run build
+
+# Create final image
+FROM apify/actor-node-playwright-chrome:16
+
+# Copy only built JS files from builder image
+COPY --from=builder --chown=myuser /home/myuser/dist ./dist
+
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY --chown=myuser package*.json ./
+
+# Install NPM packages, skip optional and development dependencies to
+# keep the image small. Avoid logging too much and print the dependency
+# tree for debugging
+RUN npm --quiet set progress=false \
+    && npm install --omit=dev --omit=optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --omit=dev --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after NPM install, quick build will be really fast
+# for most source file changes.
+COPY --chown=myuser . ./
+
+
+# Run the image. If you know you won't need headful browsers,
+# you can remove the XVFB start script for a micro perf gain.
+CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

package/templates/getting-started-ts/README.md

@@ -0,0 +1,9 @@
+# Getting started with Crawlee
+
+This example uses `PlaywrightCrawler` to recursively crawl https://crawlee.dev using the browser automation library [Playwright](https://playwright.dev).
+
+You can find more examples and documentation at the following links:
+
+- [Step-by-step tutorial](https://crawlee.dev/docs/introduction) for Crawlee
+- `PlaywrightCrawler` [API documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
+- Other [examples](https://crawlee.dev/docs/examples/playwright-crawler)

package/templates/getting-started-ts/package.json

@@ -0,0 +1,24 @@
+{
+    "name": "crawlee-getting-started-ts",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "crawlee": "^3.0.0",
+        "playwright": "*"
+    },
+    "devDependencies": {
+        "@apify/tsconfig": "^0.1.0",
+        "ts-node": "^10.8.0",
+        "typescript": "^4.7.4"
+    },
+    "scripts": {
+        "start": "npm run start:dev",
+        "start:prod": "node dist/main.js",
+        "start:dev": "ts-node-esm -T src/main.ts",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

package/templates/getting-started-ts/src/main.ts

@@ -0,0 +1,24 @@
+// For more information, see https://crawlee.dev/
+import { PlaywrightCrawler, Dataset } from 'crawlee';
+
+// PlaywrightCrawler crawls the web using a headless
+// browser controlled by the Playwright library.
+const crawler = new PlaywrightCrawler({
+    // Use the requestHandler to process each of the crawled pages.
+    async requestHandler({ request, page, enqueueLinks, log }) {
+        const title = await page.title();
+        log.info(`Title of ${request.loadedUrl} is '${title}'`);
+
+        // Save results as JSON to ./storage/datasets/default
+        await Dataset.pushData({ title, url: request.loadedUrl });
+
+        // Extract links from the current page
+        // and add them to the crawling queue.
+        await enqueueLinks();
+    },
+    // Uncomment this option to see the browser window.
+    // headless: false,
+});
+
+// Add first URL to the queue and start the crawl.
+await crawler.run(['https://crawlee.dev']);

package/templates/getting-started-ts/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "@apify/tsconfig",
+    "compilerOptions": {
+        "module": "ES2022",
+        "target": "ES2022",
+        "outDir": "dist",
+        "noUnusedLocals": false,
+        "lib": ["DOM"]
+    },
+    "include": [
+        "./src/**/*"
+    ]
+}

package/templates/playwright-js/.dockerignore

@@ -0,0 +1,10 @@
+# configurations
+.idea
+
+# crawlee and apify storage folders
+apify_storage
+crawlee_storage
+storage
+
+# installed files
+node_modules

package/templates/playwright-js/Dockerfile

@@ -1,15 +1,29 @@
+# Specify the base Docker image. You can read more about
+# the available images at https://crawlee.dev/docs/guides/docker-images
+# You can also use any other image from Docker Hub.
 FROM apify/actor-node-playwright-chrome:16
 
-# copy package.json, lock file and possibly crawlee.json
-COPY package*.json crawlee?json ./
-COPY src ./src
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY --chown=myuser package*.json ./
 
-# install only prod deps
+# Install NPM packages, skip optional and development dependencies to
+# keep the image small. Avoid logging too much and print the dependency
+# tree for debugging
 RUN npm --quiet set progress=false \
-    && npm install --only=prod --no-optional \
+    && npm install --omit=dev --omit=optional \
     && echo "Installed NPM packages:" \
-    && (npm list --only=prod --no-optional --all || true) \
+    && (npm list --omit=dev --all || true) \
     && echo "Node.js version:" \
     && node --version \
     && echo "NPM version:" \
     && npm --version
+
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after NPM install, quick build will be really fast
+# for most source file changes.
+COPY --chown=myuser . ./
+
+
+# Run the image.
+CMD npm start --silent

package/templates/playwright-js/README.md

@@ -6,21 +6,3 @@ If you're looking for examples or want to learn more visit:
 
 - [Documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
 - [Examples](https://crawlee.dev/docs/examples/playwright-crawler)
-
-## Writing a README
-
-See our tutorial on [writing READMEs for your actors](https://help.apify.com/en/articles/2912548-how-to-write-great-readme-for-your-actors) if you need more inspiration.
-
-### Table of contents
-
-If your README requires a table of contents, use the template below and make sure to keep the `<!-- toc start -->` and `<!-- toc end -->` markers.
-
-<!-- toc start -->
-- Introduction
-- Use Cases
-    - Case 1
-    - Case 2
-- Input
-- Output
-- Miscellaneous
- <!-- toc end -->

package/templates/playwright-js/package.json

@@ -9,7 +9,6 @@
     },
     "scripts": {
         "start": "node src/main.js",
-        "build": "tsc",
         "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
     },
     "author": "It's not you it's me",

package/templates/playwright-js/src/routes.js

@@ -3,19 +3,19 @@ import { Dataset, createPlaywrightRouter } from 'crawlee';
 export const router = createPlaywrightRouter();
 
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`Handle Start URLs`);
+    log.info(`enqueueing new URLs`);
     await enqueueLinks({
         globs: ['https://crawlee.dev/**'],
-        label: 'DETAIL',
+        label: 'detail',
     });
 });
 
-router.addHandler('handleList', async ({ log }) => {
-    log.info(`Handle pagination`);
-});
-
-router.addHandler('handleDetail', async ({ request, page, log }) => {
+router.addHandler('detail', async ({ request, page, log }) => {
     const title = await page.title();
-    log.info(`Handle details: ${title} [${request.loadedUrl}]`);
-    await Dataset.pushData({ url: request.loadedUrl });
+    log.info(`${title}`, { url: request.loadedUrl });
+
+    await Dataset.pushData({
+        url: request.loadedUrl,
+        title,
+    });
 });

package/templates/playwright-ts/.dockerignore

@@ -0,0 +1,10 @@
+# configurations
+.idea
+
+# crawlee and apify storage folders
+apify_storage
+crawlee_storage
+storage
+
+# installed files
+node_modules