cleanscrape 0.1.1 → 0.1.3

package/README.md

@@ -40,6 +40,9 @@ cleanscrape default https://your-app.vercel.app
 # run the scraped site locally
 cleanscrape run ./output/example --port 4173
 
+# same preview server via npm script
+npm run dev
+
 # everything mode is now default; disable with --no-everything
 cleanscrape https://example.com -o ./output/example --mode clean
 

package/dist/extractor.js

@@ -208,6 +208,33 @@ function isSameOriginHtmlLink(candidateUrl, rootOrigin) {
         return false;
     return true;
 }
+function getBaseDomain(hostname) {
+    const parts = hostname.toLowerCase().split(".").filter(Boolean);
+    if (parts.length <= 2)
+        return hostname.toLowerCase();
+    return `${parts[parts.length - 2]}.${parts[parts.length - 1]}`;
+}
+function isAllowedAssetHost(assetUrl, rootHost) {
+    try {
+        const host = new URL(assetUrl).hostname.toLowerCase();
+        const root = rootHost.toLowerCase();
+        const rootBase = getBaseDomain(root);
+        const hostBase = getBaseDomain(host);
+        return host === root || host.endsWith(`.${root}`) || hostBase === rootBase;
+    }
+    catch {
+        return false;
+    }
+}
+function looksLikeFileUrl(urlStr) {
+    try {
+        const u = new URL(urlStr);
+        return path.extname(u.pathname.toLowerCase()).length > 0;
+    }
+    catch {
+        return false;
+    }
+}
 function toRelativeWebPath(fromDir, toPath) {
     const rel = path.relative(fromDir, toPath).split(path.sep).join("/");
     if (rel === "")
@@ -237,7 +264,7 @@ function findRemoteUrls(content) {
     const matches = content.match(/(?:https?:)?\/\/[^\s"'()<>]+/g) || [];
     return matches.map((u) => u.trim()).filter(Boolean);
 }
-async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, onProgress) {
+async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, onProgress, rootHost) {
     const value = $(el).attr(attr);
     if (!value)
         return;
@@ -252,8 +279,12 @@ async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir,
     const absolute = normalizeUrl(value, baseUrl);
     if (!absolute)
         return;
+    if (!looksLikeFileUrl(absolute))
+        return;
     if (isTrackerUrl(absolute))
         return;
+    if (rootHost && !isAllowedAssetHost(absolute, rootHost))
+        return;
     const savedPath = await downloadBinaryAsset(absolute, outDir, options.timeoutMs, options.userAgent, assetMap);
     if (!savedPath) {
         warnings.push(`Failed to fetch asset: ${absolute}`);
@@ -284,6 +315,7 @@ export async function extractFrontend(options) {
         throw new Error(`Invalid URL: ${options.url}`);
     }
     const rootOrigin = new URL(normalizedRootUrl).origin;
+    const rootHost = new URL(normalizedRootUrl).hostname;
     const queue = [{ url: normalizedRootUrl, depth: 0 }];
     const visited = new Set();
     const assets = [];
@@ -540,7 +572,10 @@ export async function extractFrontend(options) {
                 { selector: "embed[src]", attr: "src" },
                 { selector: "object[data]", attr: "data" },
                 { selector: "input[src]", attr: "src" },
-                { selector: "link[href]", attr: "href" },
+                { selector: "link[rel~='icon'][href]", attr: "href" },
+                { selector: "link[rel='manifest'][href]", attr: "href" },
+                { selector: "link[rel='preload'][href]", attr: "href" },
+                { selector: "link[rel='prefetch'][href]", attr: "href" },
                 { selector: "source[src]", attr: "src" },
                 { selector: "image[href]", attr: "href" },
                 { selector: "image[xlink\\:href]", attr: "xlink:href" },
@@ -550,10 +585,7 @@ export async function extractFrontend(options) {
             for (const { selector, attr } of attrSelectors) {
                 const nodes = $(selector).toArray();
                 for (const el of nodes) {
-                    if (selector === "link[href]" && $(el).attr("rel")?.toLowerCase() === "stylesheet") {
-                        continue;
-                    }
-                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress);
+                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress, rootHost);
                 }
             }
             const downloadableAnchors = $("a[href]").toArray();
@@ -567,11 +599,15 @@ export async function extractFrontend(options) {
                 const absolute = normalizeUrl(href, current.url);
                 if (!absolute)
                     continue;
+                if (!looksLikeFileUrl(absolute))
+                    continue;
+                if (!isAllowedAssetHost(absolute, rootHost))
+                    continue;
                 const normalized = normalizePageUrl(absolute);
                 const isInternalHtml = normalized ? isSameOriginHtmlLink(normalized, rootOrigin) : false;
                 if (isInternalHtml)
                     continue;
-                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress);
+                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress, rootHost);
             }
         }
         const cssMerged = cssBlocks.join("\n\n");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cleanscrape",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Clean frontend extractor: convert live sites into editable HTML/CSS/assets",
   "type": "module",
   "bin": {
@@ -11,7 +11,8 @@
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "dev": "tsx src/cli.ts",
+    "dev": "node dist/cli.js run ./output --port 4173",
+    "dev:cli": "tsx src/cli.ts",
     "start": "node dist/cli.js",
     "check": "tsc --noEmit -p tsconfig.json"
   },