cleanscrape 0.1.0 → 0.1.2

package/dist/cli.js

@@ -133,6 +133,7 @@ async function runExtraction(url, opts) {
             mode,
             everything: opts.everything !== false,
             strictClean: Boolean(opts.strictClean),
+            onProgress: opts.quiet ? undefined : (message) => console.log(message),
             timeoutMs,
             crawlDepth,
             maxPages,
@@ -237,6 +238,7 @@ if (isScrapify) {
             .option("--everything", "Capture all discoverable assets/code and keep scripts")
             .option("--no-everything", "Disable full capture mode")
             .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+            .option("--quiet", "Hide per-file streaming logs")
             .option("--save-default", "Save this run's URL/options as defaults")
             .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
             .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)
@@ -272,6 +274,7 @@ if (isScrapify) {
             .option("--everything", "Capture all discoverable assets/code and keep scripts")
             .option("--no-everything", "Disable full capture mode")
             .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+            .option("--quiet", "Hide per-file streaming logs")
             .option("--save-default", "Save this run's URL/options as defaults")
             .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
             .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)

package/dist/extractor.js

@@ -186,9 +186,10 @@ async function downloadBinaryAsset(assetUrl, outDir, timeoutMs, userAgent, asset
 function maybeAddAsset(assets, seen, record) {
     const key = `${record.kind}:${record.savedPath}`;
     if (seen.has(key))
-        return;
+        return false;
     seen.add(key);
     assets.push(record);
+    return true;
 }
 function isSameOriginHtmlLink(candidateUrl, rootOrigin) {
     let parsed;
@@ -207,6 +208,33 @@ function isSameOriginHtmlLink(candidateUrl, rootOrigin) {
         return false;
     return true;
 }
+function getBaseDomain(hostname) {
+    const parts = hostname.toLowerCase().split(".").filter(Boolean);
+    if (parts.length <= 2)
+        return hostname.toLowerCase();
+    return `${parts[parts.length - 2]}.${parts[parts.length - 1]}`;
+}
+function isAllowedAssetHost(assetUrl, rootHost) {
+    try {
+        const host = new URL(assetUrl).hostname.toLowerCase();
+        const root = rootHost.toLowerCase();
+        const rootBase = getBaseDomain(root);
+        const hostBase = getBaseDomain(host);
+        return host === root || host.endsWith(`.${root}`) || hostBase === rootBase;
+    }
+    catch {
+        return false;
+    }
+}
+function looksLikeFileUrl(urlStr) {
+    try {
+        const u = new URL(urlStr);
+        return path.extname(u.pathname.toLowerCase()).length > 0;
+    }
+    catch {
+        return false;
+    }
+}
 function toRelativeWebPath(fromDir, toPath) {
     const rel = path.relative(fromDir, toPath).split(path.sep).join("/");
     if (rel === "")
@@ -236,7 +264,7 @@ function findRemoteUrls(content) {
     const matches = content.match(/(?:https?:)?\/\/[^\s"'()<>]+/g) || [];
     return matches.map((u) => u.trim()).filter(Boolean);
 }
-async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings) {
+async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, onProgress, rootHost) {
     const value = $(el).attr(attr);
     if (!value)
         return;
@@ -251,8 +279,12 @@ async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir,
     const absolute = normalizeUrl(value, baseUrl);
     if (!absolute)
         return;
+    if (!looksLikeFileUrl(absolute))
+        return;
     if (isTrackerUrl(absolute))
         return;
+    if (rootHost && !isAllowedAssetHost(absolute, rootHost))
+        return;
     const savedPath = await downloadBinaryAsset(absolute, outDir, options.timeoutMs, options.userAgent, assetMap);
     if (!savedPath) {
         warnings.push(`Failed to fetch asset: ${absolute}`);
@@ -261,13 +293,19 @@ async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir,
     const localPath = path.join(outDir, savedPath);
     const rel = toRelativeWebPath(pageDir, localPath);
     $(el).attr(attr, rel);
-    maybeAddAsset(assets, seenAssetRecords, {
+    const added = maybeAddAsset(assets, seenAssetRecords, {
         url: absolute,
         kind: inferAssetKind(absolute),
         savedPath
     });
+    if (added && onProgress)
+        onProgress(`[asset:${inferAssetKind(absolute)}] ${savedPath}`);
 }
 export async function extractFrontend(options) {
+    const progress = (msg) => {
+        if (options.onProgress)
+            options.onProgress(msg);
+    };
     const everything = options.everything === true;
     const strictClean = options.strictClean === true;
     const outDir = path.resolve(options.outDir);
@@ -277,6 +315,7 @@ export async function extractFrontend(options) {
         throw new Error(`Invalid URL: ${options.url}`);
     }
     const rootOrigin = new URL(normalizedRootUrl).origin;
+    const rootHost = new URL(normalizedRootUrl).hostname;
     const queue = [{ url: normalizedRootUrl, depth: 0 }];
     const visited = new Set();
     const assets = [];
@@ -292,6 +331,7 @@ export async function extractFrontend(options) {
         if (visited.has(current.url))
             continue;
         visited.add(current.url);
+        progress(`[page] Fetching ${current.url}`);
         let htmlRes;
         try {
             htmlRes = await fetchWithTimeout(current.url, options.timeoutMs, options.userAgent);
@@ -311,6 +351,7 @@ export async function extractFrontend(options) {
         }
         const domHtml = await htmlRes.text();
         const $ = load(domHtml);
+        progress(`[page] Parsing ${current.url}`);
         if (current.depth < options.crawlDepth) {
             const discovered = new Set();
             $("a[href]").each((_, el) => {
@@ -415,11 +456,13 @@ export async function extractFrontend(options) {
                 const localPath = path.join(outDir, savedPath);
                 const rel = toRelativeWebPath(pageDir, localPath);
                 $(el).attr(attr, rel);
-                maybeAddAsset(assets, seenAssetRecords, {
+                const added = maybeAddAsset(assets, seenAssetRecords, {
                     url: absolute,
                     kind: inferBinaryAssetKind(absolute) === "font" ? "font" : "image",
                     savedPath
                 });
+                if (added)
+                    progress(`[asset:image] ${savedPath}`);
             }
             catch (error) {
                 warnings.push(`Failed to fetch media: ${absolute} (${String(error)})`);
@@ -463,11 +506,13 @@ export async function extractFrontend(options) {
                     const localPath = path.join(outDir, savedPath);
                     const rel = toRelativeWebPath(pageDir, localPath);
                     $(el).attr("src", rel);
-                    maybeAddAsset(assets, seenAssetRecords, {
+                    const added = maybeAddAsset(assets, seenAssetRecords, {
                         url: absolute,
                         kind: "script",
                         savedPath
                     });
+                    if (added)
+                        progress(`[asset:script] ${savedPath}`);
                 }
                 catch (error) {
                     warnings.push(`Failed to fetch script: ${absolute} (${String(error)})`);
@@ -487,11 +532,13 @@ export async function extractFrontend(options) {
                     const rel = toRelativeWebPath(pageDir, targetPath);
                     $(el).attr("src", rel);
                     $(el).text("");
-                    maybeAddAsset(assets, seenAssetRecords, {
+                    const added = maybeAddAsset(assets, seenAssetRecords, {
                         url: `${current.url}#inline-script-${inlineIndex}`,
                         kind: "script",
                         savedPath
                     });
+                    if (added)
+                        progress(`[asset:script] ${savedPath}`);
                 }
             }
         }
@@ -525,7 +572,10 @@ export async function extractFrontend(options) {
                 { selector: "embed[src]", attr: "src" },
                 { selector: "object[data]", attr: "data" },
                 { selector: "input[src]", attr: "src" },
-                { selector: "link[href]", attr: "href" },
+                { selector: "link[rel~='icon'][href]", attr: "href" },
+                { selector: "link[rel='manifest'][href]", attr: "href" },
+                { selector: "link[rel='preload'][href]", attr: "href" },
+                { selector: "link[rel='prefetch'][href]", attr: "href" },
                 { selector: "source[src]", attr: "src" },
                 { selector: "image[href]", attr: "href" },
                 { selector: "image[xlink\\:href]", attr: "xlink:href" },
@@ -535,10 +585,7 @@ export async function extractFrontend(options) {
             for (const { selector, attr } of attrSelectors) {
                 const nodes = $(selector).toArray();
                 for (const el of nodes) {
-                    if (selector === "link[href]" && $(el).attr("rel")?.toLowerCase() === "stylesheet") {
-                        continue;
-                    }
-                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings);
+                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress, rootHost);
                 }
             }
             const downloadableAnchors = $("a[href]").toArray();
@@ -552,11 +599,15 @@ export async function extractFrontend(options) {
                 const absolute = normalizeUrl(href, current.url);
                 if (!absolute)
                     continue;
+                if (!looksLikeFileUrl(absolute))
+                    continue;
+                if (!isAllowedAssetHost(absolute, rootHost))
+                    continue;
                 const normalized = normalizePageUrl(absolute);
                 const isInternalHtml = normalized ? isSameOriginHtmlLink(normalized, rootOrigin) : false;
                 if (isInternalHtml)
                     continue;
-                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings);
+                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress, rootHost);
             }
         }
         const cssMerged = cssBlocks.join("\n\n");
@@ -574,11 +625,13 @@ export async function extractFrontend(options) {
             const localPath = path.join(outDir, savedPath);
             const rel = toRelativeWebPath(cssDir, localPath);
             rewrittenCss = rewrittenCss.split(cssUrl).join(rel);
-            maybeAddAsset(assets, seenAssetRecords, {
+            const added = maybeAddAsset(assets, seenAssetRecords, {
                 url: abs,
                 kind: inferBinaryAssetKind(abs) === "font" ? "font" : "image",
                 savedPath
             });
+            if (added)
+                progress(`[asset:${inferBinaryAssetKind(abs)}] ${savedPath}`);
         }
         if (strictClean) {
             strictCleanDom($);
@@ -592,6 +645,8 @@ export async function extractFrontend(options) {
         }
         await saveTextFile(pagePaths.htmlPath, await formatMaybe(finalHtml, "html"));
         await saveTextFile(pagePaths.cssPath, await formatMaybe(rewrittenCss, "css"));
+        progress(`[write] ${path.relative(outDir, pagePaths.htmlPath)}`);
+        progress(`[write] ${path.relative(outDir, pagePaths.cssPath)}`);
         pages.push({
             url: current.url,
             htmlPath: path.relative(outDir, pagePaths.htmlPath),
@@ -627,6 +682,7 @@ export async function extractFrontend(options) {
         verification,
         warnings
     }, null, 2));
+    progress("[done] Wrote manifest.json");
     const rootPagePaths = getPagePaths(normalizedRootUrl, outDir);
     return {
         url: options.url,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cleanscrape",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Clean frontend extractor: convert live sites into editable HTML/CSS/assets",
   "type": "module",
   "bin": {