guidelinescraper 1.0.4 → 1.0.6

package/crawl.mjs

@@ -20,6 +20,7 @@ const { values, positionals } = parseArgs({
     url: { type: "string", short: "u" },
     hub: { type: "string", short: "h" },
     cookie: { type: "string", short: "c" },
+    limit: { type: "string", short: "l" },
     help: { type: "boolean" },
   },
   allowPositionals: true,
@@ -33,6 +34,7 @@ Options:
   -u, --url <url>       Portal domain or URL
   -h, --hub <id>        Hub ID (auto-detected if omitted)
   -c, --cookie <str>    Cookie header for authenticated requests
+  -l, --limit <n>       Only crawl the first N pages
       --help            Show this help
 
 Environment variables (or .env file):
@@ -45,6 +47,7 @@ Environment variables (or .env file):
 const inputUrl = values.url || positionals[0] || process.env.URL;
 const hubId = values.hub || process.env.HUB_ID;
 const cookie = values.cookie || process.env.COOKIE || "";
+const limit = values.limit ? Number(values.limit) : Infinity;
 
 const OUTPUT_DIR = "output";
 
@@ -100,7 +103,8 @@ function collectPages(nodes, parentDir) {
 }
 
 const domainDir = path.join(OUTPUT_DIR, sanitize(siteTree.domain));
-const pages = collectPages(siteTree.children, path.join(domainDir, "pdf"));
+const allPages = collectPages(siteTree.children, path.join(domainDir, "pdf"));
+const pages = allPages.slice(0, limit);
 
 const urlToPdf = new Map(pages.map((p) => [p.url, p.pdfPath]));
 const totalPages = pages.length;
@@ -125,7 +129,7 @@ const crawler = new PlaywrightCrawler({
   launchContext: {
     launchOptions: { args: ["--disable-dev-shm-usage"] },
   },
-  maxConcurrency: 16,
+  maxConcurrency: 8,
   maxRequestRetries: 2,
   navigationTimeoutSecs: 120,
 
@@ -165,7 +169,6 @@ const crawler = new PlaywrightCrawler({
       return;
     }
 
-
     await page
       .waitForLoadState("networkidle", { timeout: 30_000 })
       .catch(() => {});
@@ -395,11 +398,18 @@ const crawler = new PlaywrightCrawler({
 
     fs.writeFileSync(pdfPath, pdfBuffer);
 
+    const rawHtml = await page.content();
+
+    const rawPath = pdfPath
+      .replace(/\/pdf\//, "/raw/")
+      .replace(/\.pdf$/, ".html");
+    fs.mkdirSync(path.dirname(rawPath), { recursive: true });
+    fs.writeFileSync(rawPath, rawHtml);
+
     const htmlPath = pdfPath
       .replace(/\/pdf\//, "/html/")
       .replace(/\.pdf$/, ".html");
     fs.mkdirSync(path.dirname(htmlPath), { recursive: true });
-    const rawHtml = await page.content();
     fs.writeFileSync(htmlPath, purge(rawHtml));
 
     completed++;
@@ -407,9 +417,12 @@ const crawler = new PlaywrightCrawler({
     const pct = Math.round((completed / totalPages) * 100);
     const avgSec = (Date.now() - crawlStart) / 1000 / completed;
     const remaining = Math.round(avgSec * (totalPages - completed));
-    const eta = remaining > 0 ? `~${formatDuration(remaining * 1000)} left` : "";
+    const eta =
+      remaining > 0 ? `~${formatDuration(remaining * 1000)} left` : "";
     const shortPath = pdfPath.replace(/\.pdf$/, "").replace(/^output\//, "");
-    log.info(`[${completed}/${totalPages}] ${pct}% · ${elapsed}s ${eta} · ${shortPath}`);
+    log.info(
+      `[${completed}/${totalPages}] ${pct}% · ${elapsed}s ${eta} · ${shortPath}`,
+    );
   },
 
   async failedRequestHandler({ request, log }) {
@@ -434,5 +447,7 @@ await crawler.run(pages.map((p) => ({ url: p.url, uniqueKey: p.url })));
 const totalMs = Date.now() - crawlStart;
 console.log(`\n${"─".repeat(50)}`);
 console.log(`Done in ${formatDuration(totalMs)}`);
-console.log(`  ${completed - failed} saved, ${failed} failed, ${totalPages} total`);
+console.log(
+  `  ${completed - failed} saved, ${failed} failed, ${totalPages} total`,
+);
 console.log(`  Output: ${domainDir}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "guidelinescraper",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "type": "module",
   "description": "Scrape a Frontify brand portal and save every page as PDF and clean HTML",
   "bin": {

package/purge-html.mjs

@@ -5,7 +5,7 @@ import { parseHTML } from "linkedom";
 const REMOVE_TAGS = new Set([
   "script", "style", "link", "meta", "noscript", "iframe", "svg", "canvas",
   "video", "audio", "source", "track", "object", "embed", "applet",
-  "form", "input", "textarea", "select", "button", "template",
+  "form", "input", "textarea", "select", "template",
 ]);
 
 const REMOVE_SELECTORS = [
@@ -81,7 +81,7 @@ export function purge(html) {
     document.querySelector("#classic-theme") ||
     document.body;
 
-  cleanNode(main);
+  for (const child of [...main.childNodes]) cleanNode(child);
 
   const title = document.querySelector("title")?.textContent?.trim() || "";
   const cleanHtml =