cleanscrape 0.1.0 → 0.1.1

package/dist/cli.js

@@ -133,6 +133,7 @@ async function runExtraction(url, opts) {
             mode,
             everything: opts.everything !== false,
             strictClean: Boolean(opts.strictClean),
+            onProgress: opts.quiet ? undefined : (message) => console.log(message),
             timeoutMs,
             crawlDepth,
             maxPages,
@@ -237,6 +238,7 @@ if (isScrapify) {
             .option("--everything", "Capture all discoverable assets/code and keep scripts")
             .option("--no-everything", "Disable full capture mode")
             .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+            .option("--quiet", "Hide per-file streaming logs")
             .option("--save-default", "Save this run's URL/options as defaults")
             .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
             .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)
@@ -272,6 +274,7 @@ if (isScrapify) {
             .option("--everything", "Capture all discoverable assets/code and keep scripts")
             .option("--no-everything", "Disable full capture mode")
             .option("--strict-clean", "Aggressively strip noisy attributes/comments for cleaner manual edits")
+            .option("--quiet", "Hide per-file streaming logs")
             .option("--save-default", "Save this run's URL/options as defaults")
             .option("-t, --timeout <ms>", "Timeout in milliseconds", defaultOptions.timeout)
             .option("-d, --depth <n>", "Internal link crawl depth", defaultOptions.depth)

package/dist/extractor.js

@@ -186,9 +186,10 @@ async function downloadBinaryAsset(assetUrl, outDir, timeoutMs, userAgent, asset
 function maybeAddAsset(assets, seen, record) {
     const key = `${record.kind}:${record.savedPath}`;
     if (seen.has(key))
-        return;
+        return false;
     seen.add(key);
     assets.push(record);
+    return true;
 }
 function isSameOriginHtmlLink(candidateUrl, rootOrigin) {
     let parsed;
@@ -236,7 +237,7 @@ function findRemoteUrls(content) {
     const matches = content.match(/(?:https?:)?\/\/[^\s"'()<>]+/g) || [];
     return matches.map((u) => u.trim()).filter(Boolean);
 }
-async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings) {
+async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, onProgress) {
     const value = $(el).attr(attr);
     if (!value)
         return;
@@ -261,13 +262,19 @@ async function downloadAndRelinkAttribute($, el, attr, baseUrl, pageDir, outDir,
     const localPath = path.join(outDir, savedPath);
     const rel = toRelativeWebPath(pageDir, localPath);
     $(el).attr(attr, rel);
-    maybeAddAsset(assets, seenAssetRecords, {
+    const added = maybeAddAsset(assets, seenAssetRecords, {
         url: absolute,
         kind: inferAssetKind(absolute),
         savedPath
     });
+    if (added && onProgress)
+        onProgress(`[asset:${inferAssetKind(absolute)}] ${savedPath}`);
 }
 export async function extractFrontend(options) {
+    const progress = (msg) => {
+        if (options.onProgress)
+            options.onProgress(msg);
+    };
     const everything = options.everything === true;
     const strictClean = options.strictClean === true;
     const outDir = path.resolve(options.outDir);
@@ -292,6 +299,7 @@ export async function extractFrontend(options) {
         if (visited.has(current.url))
             continue;
         visited.add(current.url);
+        progress(`[page] Fetching ${current.url}`);
         let htmlRes;
         try {
             htmlRes = await fetchWithTimeout(current.url, options.timeoutMs, options.userAgent);
@@ -311,6 +319,7 @@ export async function extractFrontend(options) {
         }
         const domHtml = await htmlRes.text();
         const $ = load(domHtml);
+        progress(`[page] Parsing ${current.url}`);
         if (current.depth < options.crawlDepth) {
             const discovered = new Set();
             $("a[href]").each((_, el) => {
@@ -415,11 +424,13 @@ export async function extractFrontend(options) {
                 const localPath = path.join(outDir, savedPath);
                 const rel = toRelativeWebPath(pageDir, localPath);
                 $(el).attr(attr, rel);
-                maybeAddAsset(assets, seenAssetRecords, {
+                const added = maybeAddAsset(assets, seenAssetRecords, {
                     url: absolute,
                     kind: inferBinaryAssetKind(absolute) === "font" ? "font" : "image",
                     savedPath
                 });
+                if (added)
+                    progress(`[asset:image] ${savedPath}`);
             }
             catch (error) {
                 warnings.push(`Failed to fetch media: ${absolute} (${String(error)})`);
@@ -463,11 +474,13 @@ export async function extractFrontend(options) {
                     const localPath = path.join(outDir, savedPath);
                     const rel = toRelativeWebPath(pageDir, localPath);
                     $(el).attr("src", rel);
-                    maybeAddAsset(assets, seenAssetRecords, {
+                    const added = maybeAddAsset(assets, seenAssetRecords, {
                         url: absolute,
                         kind: "script",
                         savedPath
                     });
+                    if (added)
+                        progress(`[asset:script] ${savedPath}`);
                 }
                 catch (error) {
                     warnings.push(`Failed to fetch script: ${absolute} (${String(error)})`);
@@ -487,11 +500,13 @@ export async function extractFrontend(options) {
                     const rel = toRelativeWebPath(pageDir, targetPath);
                     $(el).attr("src", rel);
                     $(el).text("");
-                    maybeAddAsset(assets, seenAssetRecords, {
+                    const added = maybeAddAsset(assets, seenAssetRecords, {
                         url: `${current.url}#inline-script-${inlineIndex}`,
                         kind: "script",
                         savedPath
                     });
+                    if (added)
+                        progress(`[asset:script] ${savedPath}`);
                 }
             }
         }
@@ -538,7 +553,7 @@ export async function extractFrontend(options) {
                     if (selector === "link[href]" && $(el).attr("rel")?.toLowerCase() === "stylesheet") {
                         continue;
                     }
-                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings);
+                    await downloadAndRelinkAttribute($, el, attr, current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress);
                 }
             }
             const downloadableAnchors = $("a[href]").toArray();
@@ -556,7 +571,7 @@ export async function extractFrontend(options) {
                 const isInternalHtml = normalized ? isSameOriginHtmlLink(normalized, rootOrigin) : false;
                 if (isInternalHtml)
                     continue;
-                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings);
+                await downloadAndRelinkAttribute($, el, "href", current.url, pageDir, outDir, options, assetMap, assets, seenAssetRecords, warnings, progress);
             }
         }
         const cssMerged = cssBlocks.join("\n\n");
@@ -574,11 +589,13 @@ export async function extractFrontend(options) {
             const localPath = path.join(outDir, savedPath);
             const rel = toRelativeWebPath(cssDir, localPath);
             rewrittenCss = rewrittenCss.split(cssUrl).join(rel);
-            maybeAddAsset(assets, seenAssetRecords, {
+            const added = maybeAddAsset(assets, seenAssetRecords, {
                 url: abs,
                 kind: inferBinaryAssetKind(abs) === "font" ? "font" : "image",
                 savedPath
             });
+            if (added)
+                progress(`[asset:${inferBinaryAssetKind(abs)}] ${savedPath}`);
         }
         if (strictClean) {
             strictCleanDom($);
@@ -592,6 +609,8 @@ export async function extractFrontend(options) {
         }
         await saveTextFile(pagePaths.htmlPath, await formatMaybe(finalHtml, "html"));
         await saveTextFile(pagePaths.cssPath, await formatMaybe(rewrittenCss, "css"));
+        progress(`[write] ${path.relative(outDir, pagePaths.htmlPath)}`);
+        progress(`[write] ${path.relative(outDir, pagePaths.cssPath)}`);
         pages.push({
             url: current.url,
             htmlPath: path.relative(outDir, pagePaths.htmlPath),
@@ -627,6 +646,7 @@ export async function extractFrontend(options) {
         verification,
         warnings
     }, null, 2));
+    progress("[done] Wrote manifest.json");
     const rootPagePaths = getPagePaths(normalizedRootUrl, outDir);
     return {
         url: options.url,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cleanscrape",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Clean frontend extractor: convert live sites into editable HTML/CSS/assets",
   "type": "module",
   "bin": {