guidelinescraper 1.0.5 → 1.0.6

package/crawl.mjs

@@ -20,6 +20,7 @@ const { values, positionals } = parseArgs({
     url: { type: "string", short: "u" },
     hub: { type: "string", short: "h" },
     cookie: { type: "string", short: "c" },
+    limit: { type: "string", short: "l" },
     help: { type: "boolean" },
   },
   allowPositionals: true,
@@ -33,6 +34,7 @@ Options:
   -u, --url <url>       Portal domain or URL
   -h, --hub <id>        Hub ID (auto-detected if omitted)
   -c, --cookie <str>    Cookie header for authenticated requests
+  -l, --limit <n>       Only crawl the first N pages
       --help            Show this help
 
 Environment variables (or .env file):
@@ -45,6 +47,7 @@ Environment variables (or .env file):
 const inputUrl = values.url || positionals[0] || process.env.URL;
 const hubId = values.hub || process.env.HUB_ID;
 const cookie = values.cookie || process.env.COOKIE || "";
+const limit = values.limit ? Number(values.limit) : Infinity;
 
 const OUTPUT_DIR = "output";
 
@@ -100,7 +103,8 @@ function collectPages(nodes, parentDir) {
 }
 
 const domainDir = path.join(OUTPUT_DIR, sanitize(siteTree.domain));
-const pages = collectPages(siteTree.children, path.join(domainDir, "pdf"));
+const allPages = collectPages(siteTree.children, path.join(domainDir, "pdf"));
+const pages = allPages.slice(0, limit);
 
 const urlToPdf = new Map(pages.map((p) => [p.url, p.pdfPath]));
 const totalPages = pages.length;
@@ -394,11 +398,18 @@ const crawler = new PlaywrightCrawler({
 
     fs.writeFileSync(pdfPath, pdfBuffer);
 
+    const rawHtml = await page.content();
+
+    const rawPath = pdfPath
+      .replace(/\/pdf\//, "/raw/")
+      .replace(/\.pdf$/, ".html");
+    fs.mkdirSync(path.dirname(rawPath), { recursive: true });
+    fs.writeFileSync(rawPath, rawHtml);
+
     const htmlPath = pdfPath
       .replace(/\/pdf\//, "/html/")
       .replace(/\.pdf$/, ".html");
     fs.mkdirSync(path.dirname(htmlPath), { recursive: true });
-    const rawHtml = await page.content();
     fs.writeFileSync(htmlPath, purge(rawHtml));
 
     completed++;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "guidelinescraper",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "type": "module",
   "description": "Scrape a Frontify brand portal and save every page as PDF and clean HTML",
   "bin": {

package/purge-html.mjs

@@ -5,7 +5,7 @@ import { parseHTML } from "linkedom";
 const REMOVE_TAGS = new Set([
   "script", "style", "link", "meta", "noscript", "iframe", "svg", "canvas",
   "video", "audio", "source", "track", "object", "embed", "applet",
-  "form", "input", "textarea", "select", "button", "template",
+  "form", "input", "textarea", "select", "template",
 ]);
 
 const REMOVE_SELECTORS = [
@@ -81,7 +81,7 @@ export function purge(html) {
     document.querySelector("#classic-theme") ||
     document.body;
 
-  cleanNode(main);
+  for (const child of [...main.childNodes]) cleanNode(child);
 
   const title = document.querySelector("title")?.textContent?.trim() || "";
   const cleanHtml =