cleanscrape 0.1.0

package/README.md

@@ -0,0 +1,90 @@
+# better-wget
+
+`better-wget` is a frontend-first alternative to wget/curl for design and product work.
+
+Instead of dumping low-quality scraped output, it crawls a live site and exports:
+
+- cleaned, formatted HTML
+- merged, editable CSS
+- downloaded image/font/media assets
+- a machine-readable `manifest.json`
+
+## Why this exists
+
+Traditional download tools optimize for raw bytes. This tool optimizes for **clean editable frontend code**.
+
+## Install
+
+```bash
+# global install
+npm install -g cleanscrape
+
+# local development
+npm install
+npm run build
+npm link
+```
+
+## Usage
+
+```bash
+# direct command
+cleanscrape https://example.com -o ./output/example
+
+# interactive mode (prompts for URL/mode/etc.)
+cleanscrape
+
+# set and reuse a default URL (for example a Vercel URL)
+cleanscrape default https://your-app.vercel.app
+
+# run the scraped site locally
+cleanscrape run ./output/example --port 4173
+
+# everything mode is now default; disable with --no-everything
+cleanscrape https://example.com -o ./output/example --mode clean
+
+# strict clean pass for ultra-editable output
+cleanscrape https://example.com -o ./output/example --strict-clean
+
+# save this run as your default template
+cleanscrape https://example.com --save-default
+
+# whole-site clean crawl (default): follows internal links and strips scripts/tracker junk
+node dist/cli.js frontend https://example.com -o ./output/example --mode clean
+
+# mirror mode: keeps script tags and fetches script files when possible
+node dist/cli.js frontend https://example.com -o ./output/example-mirror --mode mirror
+
+# tune crawl scope
+node dist/cli.js frontend https://example.com -o ./output/example --depth 4 --max-pages 250
+```
+
+## Output structure
+
+```text
+output/example/
+  manifest.json
+  src/
+    index.html
+    styles.css
+    pages/
+      about/
+        index.html
+        styles.css
+      pricing/
+        index.html
+        styles.css
+  assets/
+    <hostname>/...
+```
+
+CLI prints a verification report after each scrape:
+- pages/assets/scripts/styles/fonts/images/others counts
+- `remote_urls_remaining` so you can quickly see if anything external is still referenced
+
+## Roadmap
+
+- component inference (`Hero`, `Navbar`, `Footer`) into framework templates
+- CSS deduplication and naming normalization
+- JS de-minification and source-map aware rewriting
+- multi-page crawl with route graph export

package/dist/cli.js

@@ -0,0 +1,326 @@
+#!/usr/bin/env node
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import os from "node:os";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { createInterface } from "node:readline/promises";
+import { stdin as input, stdout as output } from "node:process";
+import { Command } from "commander";
+import { extractFrontend } from "./extractor.js";
+import { startPreviewServer } from "./preview.js";
+const CONFIG_PATH = path.join(os.homedir(), ".cleanscrape", "config.json");
+async function readConfig() {
+    try {
+        const raw = await readFile(CONFIG_PATH, "utf8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return {};
+    }
+}
+async function writeConfig(config) {
+    await mkdir(path.dirname(CONFIG_PATH), { recursive: true });
+    await writeFile(CONFIG_PATH, JSON.stringify(config, null, 2) + "\n", "utf8");
+}
+async function setDefaultUrl(url) {
+    const config = await readConfig();
+    config.defaultUrl = url;
+    await writeConfig(config);
+}
+async function askInteractive(promptLabel, fallback = "") {
+    const rl = createInterface({ input, output });
+    try {
+        const suffix = fallback ? ` [${fallback}]` : "";
+        const answer = (await rl.question(`${promptLabel}${suffix}: `)).trim();
+        return answer || fallback;
+    }
+    finally {
+        rl.close();
+    }
+}
+async function askYesNo(promptLabel, fallback) {
+    const defaultText = fallback ? "Y/n" : "y/N";
+    const answer = (await askInteractive(`${promptLabel} (${defaultText})`, "")).toLowerCase();
+    if (!answer)
+        return fallback;
+    return answer === "y" || answer === "yes";
+}
+async function resolveInteractiveRun(urlArg, opts, config) {
+    const modeDefault = String(opts.mode || config.mode || "clean");
+    const outDefault = String(opts.out || config.out || "./output");
+    const timeoutDefault = String(opts.timeout || config.timeout || "60000");
+    const depthDefault = String(opts.depth || config.depth || "3");
+    const maxPagesDefault = String(opts.maxPages || config.maxPages || "100");
+    const everythingDefault = typeof opts.everything === "boolean" ? opts.everything : typeof config.everything === "boolean" ? config.everything : true;
+    const strictDefault = typeof opts.strictClean === "boolean" ? opts.strictClean : typeof config.strictClean === "boolean" ? config.strictClean : false;
+    if (urlArg) {
+        return {
+            url: urlArg,
+            opts: {
+                ...opts,
+                mode: String(opts.mode || config.mode || "clean"),
+                out: String(opts.out || config.out || "./output"),
+                timeout: String(opts.timeout || config.timeout || "60000"),
+                depth: String(opts.depth || config.depth || "3"),
+                maxPages: String(opts.maxPages || config.maxPages || "100"),
+                everything: typeof opts.everything === "boolean" ? opts.everything : config.everything ?? true,
+                strictClean: typeof opts.strictClean === "boolean" ? opts.strictClean : config.strictClean ?? false
+            }
+        };
+    }
+    const url = await askInteractive("Website URL (example: https://your-app.vercel.app)", config.defaultUrl || "");
+    if (!url) {
+        console.error("No URL provided.");
+        process.exit(1);
+    }
+    let mode = modeDefault;
+    while (mode !== "clean" && mode !== "mirror") {
+        mode = await askInteractive("Mode (clean|mirror)", "clean");
+    }
+    const out = await askInteractive("Output directory", outDefault);
+    const timeout = await askInteractive("Timeout (ms)", timeoutDefault);
+    const depth = await askInteractive("Crawl depth", depthDefault);
+    const maxPages = await askInteractive("Max pages", maxPagesDefault);
+    const everything = await askYesNo("Capture everything", everythingDefault);
+    const strictClean = await askYesNo("Enable strict clean", strictDefault);
+    const saveDefault = await askYesNo("Save these as defaults", false);
+    const mergedOpts = {
+        ...opts,
+        mode,
+        out,
+        timeout,
+        depth,
+        maxPages,
+        everything,
+        strictClean,
+        saveDefault
+    };
+    return { url, opts: mergedOpts };
+}
+async function runExtraction(url, opts) {
+    const mode = String(opts.mode);
+    if (mode !== "clean" && mode !== "mirror") {
+        console.error(`Invalid mode: ${opts.mode}. Use clean or mirror.`);
+        process.exit(1);
+    }
+    const outDir = path.resolve(String(opts.out));
+    const timeoutMs = Number(opts.timeout);
+    if (!Number.isFinite(timeoutMs) || timeoutMs < 1000) {
+        console.error(`Invalid timeout: ${opts.timeout}`);
+        process.exit(1);
+    }
+    const crawlDepth = Number(opts.depth);
+    if (!Number.isFinite(crawlDepth) || crawlDepth < 0) {
+        console.error(`Invalid depth: ${opts.depth}`);
+        process.exit(1);
+    }
+    const maxPages = Number(opts.maxPages);
+    if (!Number.isFinite(maxPages) || maxPages < 1) {
+        console.error(`Invalid max-pages: ${opts.maxPages}`);
+        process.exit(1);
+    }
+    console.log(`Extracting ${url}`);
+    console.log(`Mode: ${mode}`);
+    console.log(`Everything: ${opts.everything !== false ? "yes" : "no"}`);
+    console.log(`Strict clean: ${opts.strictClean ? "yes" : "no"}`);
+    console.log(`Depth: ${crawlDepth}`);
+    console.log(`Max pages: ${maxPages}`);
+    console.log(`Output: ${outDir}`);
+    try {
+        const summary = await extractFrontend({
+            url,
+            outDir,
+            mode,
+            everything: opts.everything !== false,
+            strictClean: Boolean(opts.strictClean),
+            timeoutMs,
+            crawlDepth,
+            maxPages,
+            userAgent: opts.userAgent ? String(opts.userAgent) : undefined
+        });
+        console.log("\nDone.");
+        console.log(`Root HTML: ${summary.htmlPath}`);
+        if (summary.cssPath) {
+            console.log(`Root CSS:  ${summary.cssPath}`);
+        }
+        console.log(`Pages exported: ${summary.pageCount}`);
+        console.log(`Assets downloaded: ${summary.assets.length}`);
+        console.log(`Verify: pages=${summary.verification.pages} assets=${summary.verification.assets} scripts=${summary.verification.scripts} styles=${summary.verification.stylesheets} fonts=${summary.verification.fonts} images=${summary.verification.images} others=${summary.verification.others} remote_urls_remaining=${summary.verification.remoteUrlsRemaining}`);
+        if (summary.warnings.length > 0) {
+            console.log("\nWarnings:");
+            for (const warning of summary.warnings) {
+                console.log(`- ${warning}`);
+            }
+        }
+        if (opts.saveDefault) {
+            await writeConfig({
+                defaultUrl: url,
+                mode,
+                out: opts.out,
+                everything: opts.everything !== false,
+                strictClean: Boolean(opts.strictClean),
+                timeout: opts.timeout,
+                depth: opts.depth,
+                maxPages: opts.maxPages
+            });
+            console.log(`Saved defaults to ${CONFIG_PATH}`);
+        }
+        console.log(`\nPreview with: cleanscrape run ${outDir}`);
+    }
+    catch (error) {
+        console.error("Extraction failed:");
+        console.error(error);
+        process.exit(1);
+    }
+}
+async function runPreview(dirArg, opts) {
+    const dir = path.resolve(dirArg || "./output");
+    const port = Number(opts.port);
+    if (!Number.isFinite(port) || port < 1 || port > 65535) {
+        console.error(`Invalid port: ${opts.port}`);
+        process.exit(1);
+    }
+    const host = String(opts.host || "127.0.0.1");
+    const server = await startPreviewServer({ dir, port, host });
+    console.log(`Serving scraped site from ${dir}`);
+    console.log(`Open: http://${host}:${port}`);
+    console.log("Press Ctrl+C to stop.");
+    const stop = () => {
+        server.close(() => process.exit(0));
+    };
+    process.on("SIGINT", stop);
+    process.on("SIGTERM", stop);
+}
+const defaultOptions = {
+    out: "./output",
+    mode: "clean",
+    everything: true,
+    timeout: "60000",
+    depth: "3",
+    maxPages: "100"
+};
+const invokedAs = path.basename(fileURLToPath(import.meta.url));
+const argv0 = path.basename(process.argv[1] || "");
+const cliAliases = new Set(["scrapify", "scraper", "cleanscrape"]);
+const activeCliName = cliAliases.has(argv0) ? argv0 : "cleanscrape";
+const isScrapify = cliAliases.has(argv0) || cliAliases.has(invokedAs.replace(/\.js$/, ""));
+if (isScrapify) {
+    if (process.argv[2] === "default") {
+        const defaultProgram = new Command();
+        defaultProgram
+            .name(`${activeCliName} default`)
+            .description("Set default URL used by interactive mode")
+            .argument("[url]", "Default website URL")
+            .action(async (url) => {
+            const resolved = typeof url === "string" && url.trim() ? url.trim() : await askInteractive("Default URL");
+            if (!resolved) {
+                console.error("No default URL provided.");
+                process.exit(1);
+            }
+            await setDefaultUrl(resolved);
+            console.log(`Default URL set: ${resolved}`);
+            console.log(`Config: ${CONFIG_PATH}`);
+        });
+        defaultProgram.parseAsync(["node", activeCliName, ...process.argv.slice(3)]).catch((err) => {
+            console.error(err);
+            process.exit(1);
+        });
+    }
+    else if (process.argv[2] === "help") {
+        const helpProgram = new Command();
+        helpProgram
+            .name(activeCliName)
+            .description("Scrape a site into clean editable frontend code")
+            .argument("[url]", "Website URL")
+            .option("-o, --out <dir>", "Output directory", defaultOptions.out)
+            .option("-m, --mode <mode>", "Export mode: clean or mirror", defaultOptions.mode)
+            .option("--everything", "Capture all discoverable assets/code and keep scripts")
+            .option("--no-everything", "Disable full capture mode")
+            .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+            .option("--save-default", "Save this run's URL/options as defaults")
+            .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
+            .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)
+            .option("--max-pages <n>", "Maximum pages to crawl", defaultOptions.maxPages)
+            .option("--user-agent <ua>", "Custom user agent");
+        helpProgram.outputHelp();
+        process.exit(0);
+    }
+    else if (process.argv[2] === "run") {
+        const runProgram = new Command();
+        runProgram
+            .name(`${activeCliName} run`)
+            .description("Serve a scraped output folder locally")
+            .argument("[dir]", "Scraped output directory", "./output")
+            .option("-p, --port <port>", "Port", "4173")
+            .option("--host <host>", "Host", "127.0.0.1")
+            .action(async (dir, opts) => {
+            await runPreview(typeof dir === "string" ? dir : undefined, opts);
+        });
+        runProgram.parseAsync(["node", activeCliName, ...process.argv.slice(3)]).catch((err) => {
+            console.error(err);
+            process.exit(1);
+        });
+    }
+    else {
+        const program = new Command();
+        program
+            .name(activeCliName)
+            .description("Scrape a site into clean editable frontend code")
+            .argument("[url]", "Website URL")
+            .option("-o, --out <dir>", "Output directory", defaultOptions.out)
+            .option("-m, --mode <mode>", "Export mode: clean or mirror", defaultOptions.mode)
+            .option("--everything", "Capture all discoverable assets/code and keep scripts")
+            .option("--no-everything", "Disable full capture mode")
+            .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+            .option("--save-default", "Save this run's URL/options as defaults")
+            .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
+            .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)
+            .option("--max-pages <n>", "Maximum pages to crawl", defaultOptions.maxPages)
+            .option("--user-agent <ua>", "Custom user agent")
+            .action(async (url, opts) => {
+            const config = await readConfig();
+            const resolved = await resolveInteractiveRun(typeof url === "string" && url.trim() ? url.trim() : undefined, opts, config);
+            await runExtraction(resolved.url, resolved.opts);
+        });
+        program.parseAsync(process.argv).catch((err) => {
+            console.error(err);
+            process.exit(1);
+        });
+    }
+}
+else {
+    const program = new Command();
+    program
+        .name("better-wget")
+        .description("Capture clean, editable frontend code from a live website")
+        .version("0.1.0");
+    program
+        .command("frontend")
+        .description("Export a frontend snapshot as clean HTML/CSS/assets")
+        .argument("<url>", "Website URL")
+        .option("-o, --out <dir>", "Output directory", defaultOptions.out)
+        .option("-m, --mode <mode>", "Export mode: clean or mirror", defaultOptions.mode)
+        .option("--everything", "Capture all discoverable assets/code and keep scripts")
+        .option("--no-everything", "Disable full capture mode")
+        .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+        .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
+        .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)
+        .option("--max-pages <n>", "Maximum pages to crawl", defaultOptions.maxPages)
+        .option("--user-agent <ua>", "Custom user agent")
+        .action(async (url, opts) => {
+        await runExtraction(url, opts);
+    });
+    program
+        .command("run")
+        .description("Serve a scraped output folder locally")
+        .argument("[dir]", "Scraped output directory", "./output")
+        .option("-p, --port <port>", "Port", "4173")
+        .option("--host <host>", "Host", "127.0.0.1")
+        .action(async (dir, opts) => {
+        await runPreview(typeof dir === "string" ? dir : undefined, opts);
+    });
+    program.parseAsync(process.argv).catch((err) => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/extractor.js

@@ -0,0 +1,642 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { URL } from "node:url";
+import { load } from "cheerio";
+import prettier from "prettier";
+const TRACKER_HOST_PATTERNS = [
+    "google-analytics.com",
+    "googletagmanager.com",
+    "doubleclick.net",
+    "segment.com",
+    "mixpanel.com",
+    "hotjar.com",
+    "sentry.io"
+];
+const NON_HTML_EXTENSIONS = new Set([
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".webp",
+    ".svg",
+    ".ico",
+    ".avif",
+    ".pdf",
+    ".zip",
+    ".gz",
+    ".rar",
+    ".7z",
+    ".mp4",
+    ".mov",
+    ".webm",
+    ".mp3",
+    ".wav",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".otf",
+    ".css",
+    ".js",
+    ".mjs",
+    ".json",
+    ".xml"
+]);
+function normalizeUrl(raw, baseUrl) {
+    try {
+        return new URL(raw, baseUrl).toString();
+    }
+    catch {
+        return null;
+    }
+}
+function normalizePageUrl(raw) {
+    try {
+        const u = new URL(raw);
+        u.hash = "";
+        u.search = "";
+        if (u.pathname.length > 1 && u.pathname.endsWith("/")) {
+            u.pathname = u.pathname.slice(0, -1);
+        }
+        return u.toString();
+    }
+    catch {
+        return null;
+    }
+}
+function toSafeFileName(input) {
+    return input.replace(/[^a-zA-Z0-9._-]+/g, "-").slice(0, 180) || "asset";
+}
+async function formatMaybe(content, parser) {
+    try {
+        return await prettier.format(content, { parser });
+    }
+    catch {
+        return content;
+    }
+}
+async function saveTextFile(filePath, content) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, content, "utf8");
+}
+function buildAssetFilePath(assetUrl, outputRoot, fallbackExt = "") {
+    const urlObj = new URL(assetUrl);
+    const ext = path.extname(urlObj.pathname) || fallbackExt;
+    const base = toSafeFileName(path.basename(urlObj.pathname, ext) || "asset");
+    const host = toSafeFileName(urlObj.hostname);
+    const dir = path.join(outputRoot, "assets", host);
+    return path.join(dir, `${base}${ext}`);
+}
+function inferBinaryAssetKind(assetUrl) {
+    const pathname = new URL(assetUrl).pathname.toLowerCase();
+    if (/\.(png|jpe?g|gif|webp|svg|ico|avif)$/.test(pathname))
+        return "image";
+    if (/\.(woff2?|ttf|otf|eot)$/.test(pathname))
+        return "font";
+    return "other";
+}
+function inferAssetKind(assetUrl) {
+    const pathname = new URL(assetUrl).pathname.toLowerCase();
+    if (pathname.endsWith(".css"))
+        return "stylesheet";
+    if (pathname.endsWith(".js") || pathname.endsWith(".mjs"))
+        return "script";
+    const binary = inferBinaryAssetKind(assetUrl);
+    if (binary === "image")
+        return "image";
+    if (binary === "font")
+        return "font";
+    return "other";
+}
+function isTrackerUrl(assetUrl) {
+    try {
+        const host = new URL(assetUrl).hostname.toLowerCase();
+        return TRACKER_HOST_PATTERNS.some((pattern) => host.includes(pattern));
+    }
+    catch {
+        return false;
+    }
+}
+function rewriteCssUrls(cssText, replacer) {
+    return cssText.replace(/url\(\s*(['"]?)([^'")]+)\1\s*\)/g, (full, quote, rawUrl) => {
+        const trimmed = rawUrl.trim();
+        if (!trimmed || trimmed.startsWith("data:") || trimmed.startsWith("#"))
+            return full;
+        const replaced = replacer(trimmed);
+        const q = quote || '"';
+        return `url(${q}${replaced}${q})`;
+    });
+}
+function getPagePaths(pageUrl, outDir) {
+    const u = new URL(pageUrl);
+    const segments = u.pathname
+        .split("/")
+        .filter(Boolean)
+        .map((s) => toSafeFileName(s));
+    if (segments.length === 0) {
+        return {
+            routeKey: "/",
+            htmlPath: path.join(outDir, "src", "index.html"),
+            cssPath: path.join(outDir, "src", "styles.css")
+        };
+    }
+    const dir = path.join(outDir, "src", "pages", ...segments);
+    return {
+        routeKey: `/${segments.join("/")}`,
+        htmlPath: path.join(dir, "index.html"),
+        cssPath: path.join(dir, "styles.css")
+    };
+}
+async function fetchWithTimeout(url, timeoutMs, userAgent) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(url, {
+            signal: controller.signal,
+            headers: {
+                "user-agent": userAgent ||
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+            }
+        });
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+async function downloadTextAsset(assetUrl, timeoutMs, userAgent) {
+    const res = await fetchWithTimeout(assetUrl, timeoutMs, userAgent);
+    if (!res.ok)
+        return null;
+    return await res.text();
+}
+async function downloadBinaryAsset(assetUrl, outDir, timeoutMs, userAgent, assetMap) {
+    const existingPath = assetMap.get(assetUrl);
+    if (existingPath)
+        return existingPath;
+    const res = await fetchWithTimeout(assetUrl, timeoutMs, userAgent);
+    if (!res.ok)
+        return null;
+    const bytes = Buffer.from(await res.arrayBuffer());
+    const targetPath = buildAssetFilePath(assetUrl, outDir);
+    await mkdir(path.dirname(targetPath), { recursive: true });
+    await writeFile(targetPath, bytes);
+    const savedPath = path.relative(outDir, targetPath);
+    assetMap.set(assetUrl, savedPath);
+    return savedPath;
+}
+function maybeAddAsset(assets, seen, record) {
+    const key = `${record.kind}:${record.savedPath}`;
+    if (seen.has(key))
+        return;
+    seen.add(key);
+    assets.push(record);
+}
+function isSameOriginHtmlLink(candidateUrl, rootOrigin) {
+    let parsed;
+    try {
+        parsed = new URL(candidateUrl);
+    }
+    catch {
+        return false;
+    }
+    if (parsed.origin !== rootOrigin)
+        return false;
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+        return false;
+    const ext = path.extname(parsed.pathname.toLowerCase());
+    if (ext && NON_HTML_EXTENSIONS.has(ext))
+        return false;
+    return true;
+}
+function toRelativeWebPath(fromDir, toPath) {
+    const rel = path.relative(fromDir, toPath).split(path.sep).join("/");
+    if (rel === "")
+        return "./";
+    return rel.startsWith(".") ? rel : `./${rel}`;
+}
+function strictCleanDom($) {
+    $("*").each((_, el) => {
+        const attrs = Object.keys(el.attribs || {});
+        for (const attrName of attrs) {
+            const lower = attrName.toLowerCase();
+            if (lower.startsWith("data-") || lower.startsWith("on")) {
+                $(el).removeAttr(attrName);
+                continue;
+            }
+            if (lower === "nonce" ||
+                lower === "integrity" ||
+                lower === "crossorigin" ||
+                lower === "fetchpriority" ||
+                lower === "referrerpolicy") {
+                $(el).removeAttr(attrName);
+            }
+        }
+    });
+}
+function findRemoteUrls(content) {
+    const matches = content.match(/(?:https?:)?\/\/[^\s"'()<>]+/g) || [];
+    return matches.map((u) => u.trim()).filter(Boolean);
+}
+async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings) {
+    const value = $(el).attr(attr);
+    if (!value)
+        return;
+    if (value.startsWith("./") ||
+        value.startsWith("../") ||
+        value.startsWith("assets/") ||
+        value.startsWith("/assets/") ||
+        value.startsWith("src/") ||
+        value.startsWith("/src/")) {
+        return;
+    }
+    const absolute = normalizeUrl(value, baseUrl);
+    if (!absolute)
+        return;
+    if (isTrackerUrl(absolute))
+        return;
+    const savedPath = await downloadBinaryAsset(absolute, outDir, options.timeoutMs, options.userAgent, assetMap);
+    if (!savedPath) {
+        warnings.push(`Failed to fetch asset: ${absolute}`);
+        return;
+    }
+    const localPath = path.join(outDir, savedPath);
+    const rel = toRelativeWebPath(pageDir, localPath);
+    $(el).attr(attr, rel);
+    maybeAddAsset(assets, seenAssetRecords, {
+        url: absolute,
+        kind: inferAssetKind(absolute),
+        savedPath
+    });
+}
+export async function extractFrontend(options) {
+    const everything = options.everything === true;
+    const strictClean = options.strictClean === true;
+    const outDir = path.resolve(options.outDir);
+    await mkdir(path.join(outDir, "src"), { recursive: true });
+    const normalizedRootUrl = normalizePageUrl(options.url);
+    if (!normalizedRootUrl) {
+        throw new Error(`Invalid URL: ${options.url}`);
+    }
+    const rootOrigin = new URL(normalizedRootUrl).origin;
+    const queue = [{ url: normalizedRootUrl, depth: 0 }];
+    const visited = new Set();
+    const assets = [];
+    const warnings = [];
+    const pages = [];
+    const remoteUrlsRemaining = new Set();
+    const assetMap = new Map();
+    const seenAssetRecords = new Set();
+    while (queue.length > 0 && visited.size < options.maxPages) {
+        const current = queue.shift();
+        if (!current)
+            break;
+        if (visited.has(current.url))
+            continue;
+        visited.add(current.url);
+        let htmlRes;
+        try {
+            htmlRes = await fetchWithTimeout(current.url, options.timeoutMs, options.userAgent);
+        }
+        catch (error) {
+            warnings.push(`Failed to fetch page: ${current.url} (${String(error)})`);
+            continue;
+        }
+        if (!htmlRes.ok) {
+            warnings.push(`Failed to fetch page: ${current.url} (HTTP ${htmlRes.status})`);
+            continue;
+        }
+        const contentType = htmlRes.headers.get("content-type") || "";
+        if (!contentType.includes("text/html")) {
+            warnings.push(`Skipped non-HTML page: ${current.url} (${contentType || "unknown content-type"})`);
+            continue;
+        }
+        const domHtml = await htmlRes.text();
+        const $ = load(domHtml);
+        if (current.depth < options.crawlDepth) {
+            const discovered = new Set();
+            $("a[href]").each((_, el) => {
+                const href = $(el).attr("href");
+                if (!href)
+                    return;
+                if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+                    return;
+                }
+                const absolute = normalizeUrl(href, current.url);
+                if (!absolute)
+                    return;
+                const normalized = normalizePageUrl(absolute);
+                if (!normalized)
+                    return;
+                if (!isSameOriginHtmlLink(normalized, rootOrigin))
+                    return;
+                if (visited.has(normalized) || discovered.has(normalized))
+                    return;
+                discovered.add(normalized);
+                queue.push({ url: normalized, depth: current.depth + 1 });
+            });
+        }
+        const pagePaths = getPagePaths(current.url, outDir);
+        const pageDir = path.dirname(pagePaths.htmlPath);
+        const cssDir = path.dirname(pagePaths.cssPath);
+        const cssBlocks = [];
+        $("style").each((_, el) => {
+            const text = $(el).html();
+            if (text?.trim())
+                cssBlocks.push(text.trim());
+        });
+        const stylesheetLinks = $("link[rel='stylesheet']").toArray();
+        for (const el of stylesheetLinks) {
+            const href = $(el).attr("href");
+            if (!href)
+                continue;
+            const absolute = normalizeUrl(href, current.url);
+            if (!absolute)
+                continue;
+            try {
+                const rawCss = await downloadTextAsset(absolute, options.timeoutMs, options.userAgent);
+                if (!rawCss) {
+                    warnings.push(`Failed to fetch stylesheet: ${absolute}`);
+                    continue;
+                }
+                let cleanedCss = rawCss;
+                const importRegex = /@import\s+(?:url\()?['"]?([^'")]+)['"]?\)?\s*;/g;
+                const importUrls = Array.from(cleanedCss.matchAll(importRegex)).map((m) => m[1]).filter(Boolean);
+                for (const importUrl of importUrls) {
+                    const importAbs = normalizeUrl(importUrl, absolute);
+                    if (!importAbs)
+                        continue;
+                    const importCss = await downloadTextAsset(importAbs, options.timeoutMs, options.userAgent);
+                    if (!importCss)
+                        continue;
+                    cleanedCss = cleanedCss.replace(`@import url(${importUrl});`, "");
+                    cleanedCss += `\n\n/* inlined import: ${importAbs} */\n${importCss}`;
+                }
+                cleanedCss = rewriteCssUrls(cleanedCss, (rawUrl) => {
+                    const abs = normalizeUrl(rawUrl, absolute);
+                    if (!abs)
+                        return rawUrl;
+                    return abs;
+                });
+                cssBlocks.push(cleanedCss);
+            }
+            catch (error) {
+                warnings.push(`Failed to fetch stylesheet: ${absolute} (${String(error)})`);
+            }
+        }
+        $("style").remove();
+        $("link[rel='stylesheet']").remove();
+        const cssHref = toRelativeWebPath(pageDir, pagePaths.cssPath);
+        $("head").append(`<link rel="stylesheet" href="${cssHref}">`);
+        const mediaNodes = $("img[src], source[src], source[srcset], video[poster]").toArray();
+        for (const el of mediaNodes) {
+            if ($(el).attr("srcset") !== undefined) {
+                const srcsetValue = $(el).attr("srcset") || "";
+                const first = srcsetValue
+                    .split(",")
+                    .map((item) => item.trim().split(/\s+/)[0])
+                    .find(Boolean);
+                if (first) {
+                    $(el).attr("src", first);
+                }
+                $(el).removeAttr("srcset");
+            }
+            const attr = $(el).attr("src") !== undefined ? "src" : "poster";
+            const value = $(el).attr(attr);
+            if (!value)
+                continue;
+            const absolute = normalizeUrl(value, current.url);
+            if (!absolute)
+                continue;
+            try {
+                const savedPath = await downloadBinaryAsset(absolute, outDir, options.timeoutMs, options.userAgent, assetMap);
+                if (!savedPath) {
+                    warnings.push(`Failed to fetch media: ${absolute}`);
+                    continue;
+                }
+                const localPath = path.join(outDir, savedPath);
+                const rel = toRelativeWebPath(pageDir, localPath);
+                $(el).attr(attr, rel);
+                maybeAddAsset(assets, seenAssetRecords, {
+                    url: absolute,
+                    kind: inferBinaryAssetKind(absolute) === "font" ? "font" : "image",
+                    savedPath
+                });
+            }
+            catch (error) {
+                warnings.push(`Failed to fetch media: ${absolute} (${String(error)})`);
+            }
+        }
+        const keepScripts = options.mode !== "clean" || everything;
+        if (!keepScripts) {
+            $("script").remove();
+            $("noscript").remove();
+            $("*[data-reactroot], *[data-reactid], *[data-v-app], *[ng-version]").removeAttr("data-reactroot data-reactid data-v-app ng-version");
+        }
+        else {
+            const scripts = $("script[src]").toArray();
+            for (const el of scripts) {
+                const src = $(el).attr("src");
+                if (!src)
+                    continue;
+                const absolute = normalizeUrl(src, current.url);
+                if (!absolute)
+                    continue;
+                try {
+                    const host = new URL(absolute).hostname;
+                    if (TRACKER_HOST_PATTERNS.some((pattern) => host.includes(pattern))) {
+                        $(el).remove();
+                        continue;
+                    }
+                }
+                catch {
+                    // ignore parse failures
+                }
+                try {
+                    const rawJs = await downloadTextAsset(absolute, options.timeoutMs, options.userAgent);
+                    if (!rawJs) {
+                        warnings.push(`Failed to fetch script: ${absolute}`);
+                        continue;
+                    }
+                    const targetPath = buildAssetFilePath(absolute, outDir, ".js");
+                    await saveTextFile(targetPath, await formatMaybe(rawJs, "babel"));
+                    const savedPath = path.relative(outDir, targetPath);
+                    assetMap.set(absolute, savedPath);
+                    const localPath = path.join(outDir, savedPath);
+                    const rel = toRelativeWebPath(pageDir, localPath);
+                    $(el).attr("src", rel);
+                    maybeAddAsset(assets, seenAssetRecords, {
+                        url: absolute,
+                        kind: "script",
+                        savedPath
+                    });
+                }
+                catch (error) {
+                    warnings.push(`Failed to fetch script: ${absolute} (${String(error)})`);
+                }
+            }
+            if (everything) {
+                let inlineIndex = 0;
+                const inlineScripts = $("script:not([src])").toArray();
+                for (const el of inlineScripts) {
+                    const raw = $(el).html();
+                    if (!raw || !raw.trim())
+                        continue;
+                    inlineIndex += 1;
+                    const targetPath = path.join(outDir, "src", "scripts", `inline-${inlineIndex}.js`);
+                    await saveTextFile(targetPath, await formatMaybe(raw, "babel"));
+                    const savedPath = path.relative(outDir, targetPath);
+                    const rel = toRelativeWebPath(pageDir, targetPath);
+                    $(el).attr("src", rel);
+                    $(el).text("");
+                    maybeAddAsset(assets, seenAssetRecords, {
+                        url: `${current.url}#inline-script-${inlineIndex}`,
+                        kind: "script",
+                        savedPath
+                    });
+                }
+            }
+        }
+        $("a[href]").each((_, el) => {
+            const href = $(el).attr("href");
+            if (!href)
+                return;
+            if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+                return;
+            }
+            const absolute = normalizeUrl(href, current.url);
+            if (!absolute)
+                return;
+            const normalized = normalizePageUrl(absolute);
+            if (!normalized)
+                return;
+            if (!isSameOriginHtmlLink(normalized, rootOrigin))
+                return;
+            const targetPage = getPagePaths(normalized, outDir);
+            const targetHtml = targetPage.htmlPath;
+            const relHref = toRelativeWebPath(pageDir, targetHtml);
+            $(el).attr("href", relHref);
+        });
+        if (everything) {
+            const attrSelectors = [
+                { selector: "img[src]", attr: "src" },
+                { selector: "video[src]", attr: "src" },
+                { selector: "audio[src]", attr: "src" },
+                { selector: "track[src]", attr: "src" },
+                { selector: "iframe[src]", attr: "src" },
+                { selector: "embed[src]", attr: "src" },
+                { selector: "object[data]", attr: "data" },
+                { selector: "input[src]", attr: "src" },
+                { selector: "link[href]", attr: "href" },
+                { selector: "source[src]", attr: "src" },
+                { selector: "image[href]", attr: "href" },
+                { selector: "image[xlink\\:href]", attr: "xlink:href" },
+                { selector: "use[href]", attr: "href" },
+                { selector: "use[xlink\\:href]", attr: "xlink:href" }
+            ];
+            for (const { selector, attr } of attrSelectors) {
+                const nodes = $(selector).toArray();
+                for (const el of nodes) {
+                    if (selector === "link[href]" && $(el).attr("rel")?.toLowerCase() === "stylesheet") {
+                        continue;
+                    }
+                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings);
+                }
+            }
+            const downloadableAnchors = $("a[href]").toArray();
+            for (const el of downloadableAnchors) {
+                const href = $(el).attr("href");
+                if (!href)
+                    continue;
+                if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+                    continue;
+                }
+                const absolute = normalizeUrl(href, current.url);
+                if (!absolute)
+                    continue;
+                const normalized = normalizePageUrl(absolute);
+                const isInternalHtml = normalized ? isSameOriginHtmlLink(normalized, rootOrigin) : false;
+                if (isInternalHtml)
+                    continue;
+                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings);
+            }
+        }
+        const cssMerged = cssBlocks.join("\n\n");
+        const cssUrls = Array.from(cssMerged.matchAll(/url\(\s*(['"]?)([^'")]+)\1\s*\)/g))
+            .map((m) => m[2])
+            .filter((u) => Boolean(u) && !u.startsWith("data:") && !u.startsWith("#"));
+        let rewrittenCss = cssMerged;
+        for (const cssUrl of cssUrls) {
+            const abs = normalizeUrl(cssUrl, current.url);
+            if (!abs)
+                continue;
+            const savedPath = await downloadBinaryAsset(abs, outDir, options.timeoutMs, options.userAgent, assetMap);
+            if (!savedPath)
+                continue;
+            const localPath = path.join(outDir, savedPath);
+            const rel = toRelativeWebPath(cssDir, localPath);
+            rewrittenCss = rewrittenCss.split(cssUrl).join(rel);
+            maybeAddAsset(assets, seenAssetRecords, {
+                url: abs,
+                kind: inferBinaryAssetKind(abs) === "font" ? "font" : "image",
+                savedPath
+            });
+        }
+        if (strictClean) {
+            strictCleanDom($);
+        }
+        const finalHtml = strictClean ? $.html().replace(/<!--[\s\S]*?-->/g, "") : $.html();
+        for (const u of findRemoteUrls(finalHtml)) {
+            remoteUrlsRemaining.add(u);
+        }
+        for (const u of findRemoteUrls(rewrittenCss)) {
+            remoteUrlsRemaining.add(u);
+        }
+        await saveTextFile(pagePaths.htmlPath, await formatMaybe(finalHtml, "html"));
+        await saveTextFile(pagePaths.cssPath, await formatMaybe(rewrittenCss, "css"));
+        pages.push({
+            url: current.url,
+            htmlPath: path.relative(outDir, pagePaths.htmlPath),
+            cssPath: path.relative(outDir, pagePaths.cssPath)
+        });
+    }
+    const scriptsCount = assets.filter((a) => a.kind === "script").length;
+    const stylesheetsCount = assets.filter((a) => a.kind === "stylesheet").length;
+    const fontsCount = assets.filter((a) => a.kind === "font").length;
+    const imagesCount = assets.filter((a) => a.kind === "image").length;
+    const othersCount = assets.filter((a) => a.kind === "other").length;
+    const verification = {
+        pages: pages.length,
+        assets: assets.length,
+        scripts: scriptsCount,
+        stylesheets: stylesheetsCount,
+        fonts: fontsCount,
+        images: imagesCount,
+        others: othersCount,
+        remoteUrlsRemaining: remoteUrlsRemaining.size
+    };
+    await saveTextFile(path.join(outDir, "manifest.json"), JSON.stringify({
+        sourceUrl: options.url,
+        mode: options.mode,
+        everything,
+        strictClean,
+        exportedAt: new Date().toISOString(),
+        crawlDepth: options.crawlDepth,
+        maxPages: options.maxPages,
+        pageCount: pages.length,
+        pages,
+        assetCount: assets.length,
+        verification,
+        warnings
+    }, null, 2));
+    const rootPagePaths = getPagePaths(normalizedRootUrl, outDir);
+    return {
+        url: options.url,
+        outDir,
+        htmlPath: rootPagePaths.htmlPath,
+        cssPath: rootPagePaths.cssPath,
+        verification,
+        pageCount: pages.length,
+        pages,
+        assets,
+        warnings
+    };
+}

package/dist/preview.js

@@ -0,0 +1,104 @@
+import { readFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import path from "node:path";
+const CONTENT_TYPES = {
+    ".html": "text/html; charset=utf-8",
+    ".css": "text/css; charset=utf-8",
+    ".js": "application/javascript; charset=utf-8",
+    ".json": "application/json; charset=utf-8",
+    ".svg": "image/svg+xml",
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".ico": "image/x-icon",
+    ".woff": "font/woff",
+    ".woff2": "font/woff2",
+    ".ttf": "font/ttf",
+    ".otf": "font/otf"
+};
+function contentTypeFor(filePath) {
+    return CONTENT_TYPES[path.extname(filePath).toLowerCase()] || "application/octet-stream";
+}
+async function tryRead(filePath) {
+    try {
+        return await readFile(filePath);
+    }
+    catch {
+        return null;
+    }
+}
+function sanitizePathname(pathname) {
+    const decoded = decodeURIComponent(pathname || "/");
+    const normalized = path.posix.normalize(decoded.startsWith("/") ? decoded : `/${decoded}`);
+    return normalized.startsWith("/") ? normalized : `/${normalized}`;
+}
+function resolveFromRoute(rootDir, reqPath) {
+    const direct = path.join(rootDir, reqPath.slice(1));
+    if (reqPath === "/") {
+        return [path.join(rootDir, "src", "index.html"), direct];
+    }
+    if (reqPath === "/manifest.json") {
+        return [path.join(rootDir, "manifest.json"), direct];
+    }
+    if (reqPath.startsWith("/assets/") || reqPath.startsWith("/src/")) {
+        return [direct];
+    }
+    if (reqPath === "/styles.css") {
+        return [path.join(rootDir, "src", "styles.css"), direct];
+    }
+    if (reqPath.endsWith("/styles.css")) {
+        const route = reqPath.slice(1, -"styles.css".length - 1).replace(/\/+$/, "");
+        if (!route)
+            return [path.join(rootDir, "src", "styles.css"), direct];
+        return [path.join(rootDir, "src", "pages", ...route.split("/"), "styles.css"), direct];
+    }
+    if (reqPath.endsWith(".html")) {
+        if (reqPath === "/index.html")
+            return [path.join(rootDir, "src", "index.html"), direct];
+        const route = reqPath.slice(1, -".html".length).replace(/\/+$/, "");
+        return [
+            path.join(rootDir, "src", "pages", ...route.split("/"), "index.html"),
+            path.join(rootDir, "src", reqPath.slice(1)),
+            direct
+        ];
+    }
+    if (path.extname(reqPath) === "") {
+        const route = reqPath.slice(1).replace(/\/+$/, "");
+        if (!route)
+            return [path.join(rootDir, "src", "index.html"), direct];
+        return [path.join(rootDir, "src", "pages", ...route.split("/"), "index.html"), direct];
+    }
+    return [direct];
+}
+export async function startPreviewServer(options) {
+    const rootDir = path.resolve(options.dir);
+    const server = createServer(async (req, res) => {
+        const parsedUrl = new URL(req.url || "/", "http://localhost");
+        const reqPath = sanitizePathname(parsedUrl.pathname);
+        if (reqPath.includes("..")) {
+            res.statusCode = 400;
+            res.end("Bad Request");
+            return;
+        }
+        const candidates = resolveFromRoute(rootDir, reqPath);
+        for (const filePath of candidates) {
+            const data = await tryRead(filePath);
+            if (!data)
+                continue;
+            res.statusCode = 200;
+            res.setHeader("Content-Type", contentTypeFor(filePath));
+            res.end(data);
+            return;
+        }
+        res.statusCode = 404;
+        res.setHeader("Content-Type", "text/plain; charset=utf-8");
+        res.end("Not found");
+    });
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(options.port, options.host, () => resolve());
+    });
+    return server;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "cleanscrape",
+  "version": "0.1.0",
+  "description": "Clean frontend extractor: convert live sites into editable HTML/CSS/assets",
+  "type": "module",
+  "bin": {
+    "cleanscrape": "dist/cli.js",
+    "scraper": "dist/cli.js",
+    "scrapify": "dist/cli.js",
+    "better-wget": "dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js",
+    "check": "tsc --noEmit -p tsconfig.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "wget",
+    "curl",
+    "scraping",
+    "frontend",
+    "playwright"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "cheerio": "^1.0.0",
+    "commander": "^14.0.0",
+    "prettier": "^3.6.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.5.1",
+    "tsx": "^4.20.5",
+    "typescript": "^5.9.2"
+  }
+}