guidelinescraper 1.0.0 → 1.0.2

package/crawl.mjs

@@ -103,8 +103,12 @@ const domainDir = path.join(OUTPUT_DIR, sanitize(siteTree.domain));
 const pages = collectPages(siteTree.children, path.join(domainDir, "pdf"));
 
 const urlToPdf = new Map(pages.map((p) => [p.url, p.pdfPath]));
+const totalPages = pages.length;
+let completed = 0;
+let failed = 0;
+const crawlStart = Date.now();
 
-console.log(`Discovered ${pages.length} pages to crawl.\n`);
+console.log(`Discovered ${totalPages} pages to crawl.\n`);
 for (const { url, pdfPath } of pages) {
   console.log(`  ${pdfPath}  ←  ${url}`);
 }
@@ -403,14 +407,36 @@ const crawler = new PlaywrightCrawler({
     const rawHtml = await page.content();
     fs.writeFileSync(htmlPath, purge(rawHtml));
 
-    log.info(`Saved ${pdfPath} + ${htmlPath}`);
+    completed++;
+    const elapsed = ((Date.now() - crawlStart) / 1000).toFixed(1);
+    const pct = Math.round((completed / totalPages) * 100);
+    const avgSec = (Date.now() - crawlStart) / 1000 / completed;
+    const remaining = Math.round(avgSec * (totalPages - completed));
+    const eta = remaining > 0 ? `~${formatDuration(remaining * 1000)} left` : "";
+    log.info(`[${completed}/${totalPages}] ${pct}% · ${elapsed}s elapsed ${eta} · ${path.basename(pdfPath)}`);
   },
 
   async failedRequestHandler({ request, log }) {
-    log.error(`Failed to crawl ${request.url} after retries.`);
+    failed++;
+    completed++;
+    log.error(`[${completed}/${totalPages}] Failed: ${request.url}`);
   },
 });
 
+function formatDuration(ms) {
+  const totalSec = Math.round(ms / 1000);
+  const m = Math.floor(totalSec / 60);
+  const s = totalSec % 60;
+  return m > 0 ? `${m}m ${s}s` : `${s}s`;
+}
+
+const discoverEnd = Date.now();
+console.log(`\nCrawling ${totalPages} pages…\n`);
+
 await crawler.run(pages.map((p) => ({ url: p.url, uniqueKey: p.url })));
 
-console.log("\nDone. Output saved under:", domainDir);
+const totalMs = Date.now() - crawlStart;
+console.log(`\n${"─".repeat(50)}`);
+console.log(`Done in ${formatDuration(totalMs)}`);
+console.log(`  ${completed - failed} saved, ${failed} failed, ${totalPages} total`);
+console.log(`  Output: ${domainDir}`);

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "guidelinescraper",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "type": "module",
   "description": "Scrape a Frontify brand portal and save every page as PDF and clean HTML",
   "bin": {
-    "guidelinescraper": "./crawl.mjs"
+    "guidelinescraper": "crawl.mjs"
   },
   "scripts": {
     "start": "node crawl.mjs"