@crawlee/templates 3.0.3-beta.18 → 3.0.3-beta.20

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/templates",
-    "version": "3.0.3-beta.18",
+    "version": "3.0.3-beta.20",
     "description": "Templates for the crawlee projects",
     "engines": {
         "node": ">=16.0.0"

package/templates/cheerio-js/Dockerfile

@@ -1,6 +1,8 @@
 FROM apify/actor-node:16
 
-COPY . ./
+# copy package.json, lock file and possibly crawlee.json
+COPY package*.json crawlee?json ./
+COPY src ./src
 
 # install only prod deps
 RUN npm --quiet set progress=false \

package/templates/cheerio-js/src/main.js

@@ -1,17 +1,11 @@
 // For more information, see https://crawlee.dev/
-import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';
+import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
 import { router } from './routes.js';
 
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
-
-if (debug) {
-    log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
 
 const crawler = new CheerioCrawler({
     // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    // Be nice to the websites. Remove to unleash full power.
-    maxConcurrency: 50,
     requestHandler: router,
 });
 

package/templates/cheerio-js/src/routes.js

@@ -5,7 +5,7 @@ export const router = createCheerioRouter();
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
     log.info(`Handle Start URLs`);
     await enqueueLinks({
-        globs: ['https://crawlee.dev/*'],
+        globs: ['https://crawlee.dev/**'],
         label: 'DETAIL',
     });
 });

package/templates/cheerio-ts/Dockerfile

@@ -5,7 +5,7 @@ FROM apify/actor-node:16 AS builder
 
 # Second, copy the TS source files, package.json and package-lock.json since it should be
 # the only files that affects the installation and build in the next step
-COPY package*.json ./
+COPY package*.json crawlee?json ./
 COPY src ./src
 
 # Install all dependencies (including dev deps) and build the project
@@ -14,8 +14,8 @@ RUN npm install --include=dev && npm run build
 # Create final image
 FROM apify/actor-node:16
 
-# Copy only necessary files
-COPY --from=builder /usr/src/app/package.json ./
+# Copy only necessary files (crawlee.json might not exist, so we need the regexp trick with `crawlee?json`)
+COPY --from=builder /usr/src/app/package.json /usr/src/app/crawlee?json ./
 COPY --from=builder /usr/src/app/README.md ./
 COPY --from=builder /usr/src/app/dist ./dist
 

package/templates/cheerio-ts/src/main.ts

@@ -1,22 +1,11 @@
 // For more information, see https://crawlee.dev/
-import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';
+import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
 import { router } from './routes.js';
 
-interface InputSchema {
-    startUrls: string[];
-    debug?: boolean;
-}
-
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
-
-if (debug) {
-    log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
 
 const crawler = new CheerioCrawler({
     // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    // Be nice to the websites. Remove to unleash full power.
-    maxConcurrency: 50,
     requestHandler: router,
 });
 

package/templates/cheerio-ts/src/routes.ts

@@ -5,7 +5,7 @@ export const router = createCheerioRouter();
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
     log.info(`Handle Start URLs`);
     await enqueueLinks({
-        globs: ['https://crawlee.dev/*'],
+        globs: ['https://crawlee.dev/**'],
         label: 'DETAIL',
     });
 });

package/templates/playwright-js/Dockerfile

@@ -1,6 +1,8 @@
 FROM apify/actor-node-playwright-chrome:16
 
-COPY . ./
+# copy package.json, lock file and possibly crawlee.json
+COPY package*.json crawlee?json ./
+COPY src ./src
 
 # install only prod deps
 RUN npm --quiet set progress=false \

package/templates/playwright-js/src/main.js

@@ -1,17 +1,11 @@
 // For more information, see https://crawlee.dev/
-import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
 import { router } from './routes.js';
 
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
-
-if (debug) {
-    log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
 
 const crawler = new PlaywrightCrawler({
     // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    // Be nice to the websites. Remove to unleash full power.
-    maxConcurrency: 50,
     requestHandler: router,
 });
 

package/templates/playwright-js/src/routes.js

@@ -5,7 +5,7 @@ export const router = createPlaywrightRouter();
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
     log.info(`Handle Start URLs`);
     await enqueueLinks({
-        globs: ['https://crawlee.dev/*'],
+        globs: ['https://crawlee.dev/**'],
         label: 'DETAIL',
     });
 });

package/templates/playwright-ts/Dockerfile

@@ -5,7 +5,7 @@ FROM apify/actor-node:16 AS builder
 
 # Second, copy the TS source files, package.json and package-lock.json since it should be
 # the only files that affects the installation and build in the next step
-COPY package*.json ./
+COPY package*.json crawlee?json ./
 COPY src ./src
 
 # Install all dependencies (including dev deps) and build the project
@@ -14,8 +14,8 @@ RUN npm install --include=dev && npm run build
 # Create final image
 FROM apify/actor-node-playwright-chrome:16
 
-# Copy only necessary files
-COPY --from=builder /usr/src/app/package.json ./
+# Copy only necessary files (crawlee.json might not exist, so we need the regexp trick with `crawlee?json`)
+COPY --from=builder /usr/src/app/package.json /usr/src/app/crawlee?json ./
 COPY --from=builder /usr/src/app/README.md ./
 COPY --from=builder /usr/src/app/dist ./dist
 

package/templates/playwright-ts/src/main.ts

@@ -1,22 +1,11 @@
 // For more information, see https://crawlee.dev/
-import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
 import { router } from './routes.js';
 
-interface InputSchema {
-    startUrls: string[];
-    debug?: boolean;
-}
-
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
-
-if (debug) {
-    log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
 
 const crawler = new PlaywrightCrawler({
     // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    // Be nice to the websites. Remove to unleash full power.
-    maxConcurrency: 50,
     requestHandler: router,
 });
 

package/templates/playwright-ts/src/routes.ts

@@ -5,7 +5,7 @@ export const router = createPlaywrightRouter();
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
     log.info(`Handle Start URLs`);
     await enqueueLinks({
-        globs: ['https://crawlee.dev/*'],
+        globs: ['https://crawlee.dev/**'],
         label: 'DETAIL',
     });
 });

package/templates/puppeteer-js/Dockerfile

@@ -1,6 +1,8 @@
 FROM apify/actor-node-puppeteer-chrome:16
 
-COPY . ./
+# copy package.json, lock file and possibly crawlee.json
+COPY package*.json crawlee?json ./
+COPY src ./src
 
 # install only prod deps
 RUN npm --quiet set progress=false \

package/templates/puppeteer-js/src/main.js

@@ -1,17 +1,11 @@
 // For more information, see https://crawlee.dev/
-import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
 import { router } from './routes.js';
 
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput() ?? {};
-
-if (debug) {
-    log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
 
 const crawler = new PuppeteerCrawler({
     // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    // Be nice to the websites. Remove to unleash full power.
-    maxConcurrency: 50,
     requestHandler: router,
 });
 

package/templates/puppeteer-js/src/routes.js

@@ -5,7 +5,7 @@ export const router = createPuppeteerRouter();
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
     log.info(`Handle Start URLs`);
     await enqueueLinks({
-        globs: ['https://crawlee.dev/*'],
+        globs: ['https://crawlee.dev/**'],
         label: 'DETAIL',
     });
 });

package/templates/puppeteer-ts/Dockerfile

@@ -5,7 +5,7 @@ FROM apify/actor-node:16 AS builder
 
 # Second, copy the TS source files, package.json and package-lock.json since it should be
 # the only files that affects the installation and build in the next step
-COPY package*.json ./
+COPY package*.json crawlee?json ./
 COPY src ./src
 
 # Install all dependencies (including dev deps) and build the project
@@ -14,8 +14,8 @@ RUN npm install --include=dev && npm run build
 # Create final image
 FROM apify/actor-node-puppeteer-chrome:16
 
-# Copy only necessary files
-COPY --from=builder /usr/src/app/package.json ./
+# Copy only necessary files (crawlee.json might not exist, so we need the regexp trick with `crawlee?json`)
+COPY --from=builder /usr/src/app/package.json /usr/src/app/crawlee?json ./
 COPY --from=builder /usr/src/app/README.md ./
 COPY --from=builder /usr/src/app/dist ./dist
 

package/templates/puppeteer-ts/src/main.ts

@@ -1,22 +1,11 @@
 // For more information, see https://crawlee.dev/
-import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';
+import { PuppeteerCrawler, log } from 'crawlee';
 import { router } from './routes.js';
 
-interface InputSchema {
-    startUrls: string[];
-    debug?: boolean;
-}
-
-const { startUrls = ['https://crawlee.dev'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};
-
-if (debug) {
-    log.setLevel(log.LEVELS.DEBUG);
-}
+const startUrls = ['https://crawlee.dev'];
 
 const crawler = new PuppeteerCrawler({
     // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
-    // Be nice to the websites. Remove to unleash full power.
-    maxConcurrency: 50,
     requestHandler: router,
 });
 

package/templates/puppeteer-ts/src/routes.ts

@@ -5,7 +5,7 @@ export const router = createPuppeteerRouter();
 router.addDefaultHandler(async ({ enqueueLinks, log }) => {
     log.info(`Handle Start URLs`);
     await enqueueLinks({
-        globs: ['https://crawlee.dev/*'],
+        globs: ['https://crawlee.dev/**'],
         label: 'DETAIL',
     });
 });