@crawlee/templates 3.0.5-beta.8 → 3.1.0

package/templates/playwright-js/src/routes.js

@@ -1,21 +0,0 @@
-import { Dataset, createPlaywrightRouter } from 'crawlee';
-
-export const router = createPlaywrightRouter();
-
-router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`enqueueing new URLs`);
-    await enqueueLinks({
-        globs: ['https://crawlee.dev/**'],
-        label: 'detail',
-    });
-});
-
-router.addHandler('detail', async ({ request, page, log }) => {
-    const title = await page.title();
-    log.info(`${title}`, { url: request.loadedUrl });
-
-    await Dataset.pushData({
-        url: request.loadedUrl,
-        title,
-    });
-});

package/templates/playwright-ts/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/playwright-ts/Dockerfile

@@ -1,51 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16 AS builder
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install all dependencies. Don't audit to speed up the installation.
-RUN npm install --include=dev --audit=false
-
-# Next, copy the source files using the user set
-# in the base image.
-COPY --chown=myuser . ./
-
-# Install all dependencies and build the project.
-# Don't audit to speed up the installation.
-RUN npm run build
-
-# Create final image
-FROM apify/actor-node-playwright-chrome:16
-
-# Copy only built JS files from builder image
-COPY --from=builder --chown=myuser /home/myuser/dist ./dist
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY --chown=myuser . ./
-
-
-# Run the image. If you know you won't need headful browsers,
-# you can remove the XVFB start script for a micro perf gain.
-CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

package/templates/playwright-ts/README.md

@@ -1,8 +0,0 @@
-# Crawlee + PlaywrightCrawler + TypeScript project
-
-This template is a production ready boilerplate for developing with `PlaywrightCrawler`. Use this to bootstrap your projects using the most up-to-date code.
-
-If you're looking for examples or want to learn more visit:
-
-- [Documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
-- [Examples](https://crawlee.dev/docs/examples/playwright-crawler)

package/templates/playwright-ts/package.json

@@ -1,24 +0,0 @@
-{
-    "name": "crawlee-playwright-ts",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0",
-        "playwright": "*"
-    },
-    "devDependencies": {
-        "@apify/tsconfig": "^0.1.0",
-        "ts-node": "^10.8.0",
-        "typescript": "^4.7.4"
-    },
-    "scripts": {
-        "start": "npm run start:dev",
-        "start:prod": "node dist/main.js",
-        "start:dev": "ts-node-esm -T src/main.ts",
-        "build": "tsc",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/playwright-ts/src/main.ts

@@ -1,12 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
-import { router } from './routes.js';
-
-const startUrls = ['https://crawlee.dev'];
-
-const crawler = new PlaywrightCrawler({
-    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    requestHandler: router,
-});
-
-await crawler.run(startUrls);

package/templates/playwright-ts/src/routes.ts

@@ -1,21 +0,0 @@
-import { Dataset, createPlaywrightRouter } from 'crawlee';
-
-export const router = createPlaywrightRouter();
-
-router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`enqueueing new URLs`);
-    await enqueueLinks({
-        globs: ['https://crawlee.dev/**'],
-        label: 'detail',
-    });
-});
-
-router.addHandler('detail', async ({ request, page, log }) => {
-    const title = await page.title();
-    log.info(`${title}`, { url: request.loadedUrl });
-
-    await Dataset.pushData({
-        url: request.loadedUrl,
-        title,
-    });
-});

package/templates/playwright-ts/tsconfig.json

@@ -1,13 +0,0 @@
-{
-    "extends": "@apify/tsconfig",
-    "compilerOptions": {
-        "module": "ES2022",
-        "target": "ES2022",
-        "outDir": "dist",
-        "noUnusedLocals": false,
-        "lib": ["DOM"]
-    },
-    "include": [
-        "./src/**/*"
-    ]
-}

package/templates/puppeteer-js/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/puppeteer-js/Dockerfile

@@ -1,29 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node-puppeteer-chrome:16
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY --chown=myuser . ./
-
-
-# Run the image.
-CMD npm start --silent

package/templates/puppeteer-js/README.md

@@ -1,8 +0,0 @@
-# Crawlee + PuppeteerCrawler + JavaScript project
-
-This template is a production ready boilerplate for developing with `PuppeteerCrawler`. Use this to bootstrap your projects using the most up-to-date code.
-
-If you're looking for examples or want to learn more visit:
-
-- [Documentation](https://crawlee.dev/api/puppeteer-crawler/class/PuppeteerCrawler)
-- [Examples](https://crawlee.dev/docs/examples/puppeteer-crawler)

package/templates/puppeteer-js/package.json

@@ -1,16 +0,0 @@
-{
-    "name": "crawlee-puppeteer-js",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0",
-        "puppeteer": "*"
-    },
-    "scripts": {
-        "start": "node src/main.js",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/puppeteer-js/src/main.js

@@ -1,12 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
-import { router } from './routes.js';
-
-const startUrls = ['https://crawlee.dev'];
-
-const crawler = new PuppeteerCrawler({
-    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    requestHandler: router,
-});
-
-await crawler.run(startUrls);

package/templates/puppeteer-js/src/routes.js

@@ -1,21 +0,0 @@
-import { Dataset, createPuppeteerRouter } from 'crawlee';
-
-export const router = createPuppeteerRouter();
-
-router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`enqueueing new URLs`);
-    await enqueueLinks({
-        globs: ['https://crawlee.dev/**'],
-        label: 'detail',
-    });
-});
-
-router.addHandler('detail', async ({ request, page, log }) => {
-    const title = await page.title();
-    log.info(`${title}`, { url: request.loadedUrl });
-
-    await Dataset.pushData({
-        url: request.loadedUrl,
-        title,
-    });
-});

package/templates/puppeteer-ts/.dockerignore

@@ -1,10 +0,0 @@
-# configurations
-.idea
-
-# crawlee and apify storage folders
-apify_storage
-crawlee_storage
-storage
-
-# installed files
-node_modules

package/templates/puppeteer-ts/Dockerfile

@@ -1,51 +0,0 @@
-# Specify the base Docker image. You can read more about
-# the available images at https://crawlee.dev/docs/guides/docker-images
-# You can also use any other image from Docker Hub.
-FROM apify/actor-node-puppeteer-chrome:16 AS builder
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install all dependencies. Don't audit to speed up the installation.
-RUN npm install --include=dev --audit=false
-
-# Next, copy the source files using the user set
-# in the base image.
-COPY --chown=myuser . ./
-
-# Install all dependencies and build the project.
-# Don't audit to speed up the installation.
-RUN npm run build
-
-# Create final image
-FROM apify/actor-node-puppeteer-chrome:16
-
-# Copy only built JS files from builder image
-COPY --from=builder --chown=myuser /home/myuser/dist ./dist
-
-# Copy just package.json and package-lock.json
-# to speed up the build using Docker layer cache.
-COPY --chown=myuser package*.json ./
-
-# Install NPM packages, skip optional and development dependencies to
-# keep the image small. Avoid logging too much and print the dependency
-# tree for debugging
-RUN npm --quiet set progress=false \
-    && npm install --omit=dev --omit=optional \
-    && echo "Installed NPM packages:" \
-    && (npm list --omit=dev --all || true) \
-    && echo "Node.js version:" \
-    && node --version \
-    && echo "NPM version:" \
-    && npm --version
-
-# Next, copy the remaining files and directories with the source code.
-# Since we do this after NPM install, quick build will be really fast
-# for most source file changes.
-COPY --chown=myuser . ./
-
-
-# Run the image. If you know you won't need headful browsers,
-# you can remove the XVFB start script for a micro perf gain.
-CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

package/templates/puppeteer-ts/README.md

@@ -1,8 +0,0 @@
-# Crawlee + PuppeteerCrawler + TypeScript project
-
-This template is a production ready boilerplate for developing with `PuppeteerCrawler`. Use this to bootstrap your projects using the most up-to-date code.
-
-If you're looking for examples or want to learn more visit:
-
-- [Documentation](https://crawlee.dev/api/puppeteer-crawler/class/PuppeteerCrawler)
-- [Examples](https://crawlee.dev/docs/examples/puppeteer-crawler)

package/templates/puppeteer-ts/package.json

@@ -1,24 +0,0 @@
-{
-    "name": "crawlee-puppeteer-ts",
-    "version": "0.0.1",
-    "type": "module",
-    "description": "This is an example of a Crawlee project.",
-    "dependencies": {
-        "crawlee": "^3.0.0",
-        "puppeteer": "*"
-    },
-    "devDependencies": {
-        "@apify/tsconfig": "^0.1.0",
-        "ts-node": "^10.8.0",
-        "typescript": "^4.7.4"
-    },
-    "scripts": {
-        "start": "npm run start:dev",
-        "start:prod": "node dist/main.js",
-        "start:dev": "ts-node-esm -T src/main.ts",
-        "build": "tsc",
-        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
-    },
-    "author": "It's not you it's me",
-    "license": "ISC"
-}

package/templates/puppeteer-ts/src/main.ts

@@ -1,12 +0,0 @@
-// For more information, see https://crawlee.dev/
-import { PuppeteerCrawler, log } from 'crawlee';
-import { router } from './routes.js';
-
-const startUrls = ['https://crawlee.dev'];
-
-const crawler = new PuppeteerCrawler({
-    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    requestHandler: router,
-});
-
-await crawler.run(startUrls);

package/templates/puppeteer-ts/src/routes.ts

@@ -1,21 +0,0 @@
-import { Dataset, createPuppeteerRouter } from 'crawlee';
-
-export const router = createPuppeteerRouter();
-
-router.addDefaultHandler(async ({ enqueueLinks, log }) => {
-    log.info(`enqueueing new URLs`);
-    await enqueueLinks({
-        globs: ['https://crawlee.dev/**'],
-        label: 'detail',
-    });
-});
-
-router.addHandler('detail', async ({ request, page, log }) => {
-    const title = await page.title();
-    log.info(`${title}`, { url: request.loadedUrl });
-
-    await Dataset.pushData({
-        url: request.loadedUrl,
-        title,
-    });
-});

package/templates/puppeteer-ts/tsconfig.json

@@ -1,13 +0,0 @@
-{
-    "extends": "@apify/tsconfig",
-    "compilerOptions": {
-        "module": "ES2022",
-        "target": "ES2022",
-        "outDir": "dist",
-        "noUnusedLocals": false,
-        "lib": ["DOM"]
-    },
-    "include": [
-        "./src/**/*"
-    ]
-}