guidelinescraper 1.0.3 → 1.0.5

package/crawl.mjs

@@ -109,10 +109,6 @@ let failed = 0;
 const crawlStart = Date.now();
 
 console.log(`Discovered ${totalPages} pages to crawl.\n`);
-for (const { url, pdfPath } of pages) {
-  console.log(`  ${pdfPath}  ←  ${url}`);
-}
-console.log();
 
 for (const { pdfPath } of pages) {
   fs.mkdirSync(path.dirname(pdfPath), { recursive: true });
@@ -129,7 +125,7 @@ const crawler = new PlaywrightCrawler({
   launchContext: {
     launchOptions: { args: ["--disable-dev-shm-usage"] },
   },
-  maxConcurrency: 16,
+  maxConcurrency: 8,
   maxRequestRetries: 2,
   navigationTimeoutSecs: 120,
 
@@ -169,7 +165,6 @@ const crawler = new PlaywrightCrawler({
       return;
     }
 
-
     await page
       .waitForLoadState("networkidle", { timeout: 30_000 })
       .catch(() => {});
@@ -411,9 +406,12 @@ const crawler = new PlaywrightCrawler({
     const pct = Math.round((completed / totalPages) * 100);
     const avgSec = (Date.now() - crawlStart) / 1000 / completed;
     const remaining = Math.round(avgSec * (totalPages - completed));
-    const eta = remaining > 0 ? `~${formatDuration(remaining * 1000)} left` : "";
+    const eta =
+      remaining > 0 ? `~${formatDuration(remaining * 1000)} left` : "";
     const shortPath = pdfPath.replace(/\.pdf$/, "").replace(/^output\//, "");
-    log.info(`[${completed}/${totalPages}] ${pct}% · ${elapsed}s ${eta} · ${shortPath}`);
+    log.info(
+      `[${completed}/${totalPages}] ${pct}% · ${elapsed}s ${eta} · ${shortPath}`,
+    );
   },
 
   async failedRequestHandler({ request, log }) {
@@ -438,5 +436,7 @@ await crawler.run(pages.map((p) => ({ url: p.url, uniqueKey: p.url })));
 const totalMs = Date.now() - crawlStart;
 console.log(`\n${"─".repeat(50)}`);
 console.log(`Done in ${formatDuration(totalMs)}`);
-console.log(`  ${completed - failed} saved, ${failed} failed, ${totalPages} total`);
+console.log(
+  `  ${completed - failed} saved, ${failed} failed, ${totalPages} total`,
+);
 console.log(`  Output: ${domainDir}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "guidelinescraper",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "type": "module",
   "description": "Scrape a Frontify brand portal and save every page as PDF and clean HTML",
   "bin": {