@redstone-md/mapr 0.0.2-alpha → 0.0.3-alpha

package/README.md

@@ -1,5 +1,7 @@
 # Mapr
 
+![Mapr banner](./assets/banner.svg)
+
 Mapr is a Bun-native CLI/TUI for reverse-engineering frontend websites and build outputs. It crawls a target site, collects analyzable frontend artifacts, runs a multi-agent AI analysis pipeline over chunked code, and writes a Markdown report with entry points, initialization flow, inferred call graph edges, restored names, artifact summaries, and investigation tips.
 
 This repository is public for source visibility and collaboration. The license remains source-available and restricted. Read the contribution and license sections before reusing or contributing to the codebase.

package/assets/banner.svg

@@ -0,0 +1,50 @@
+<svg width="1280" height="320" viewBox="0 0 1280 320" fill="none" xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="title desc">
+  <title id="title">Mapr repository banner</title>
+  <desc id="desc">A dark technical banner showing Mapr's flow from crawl to artifacts to report.</desc>
+  <rect width="1280" height="320" fill="#111315"/>
+  <rect x="24" y="24" width="1232" height="272" rx="14" fill="#15181B" stroke="#2A2E33"/>
+  <rect x="56" y="56" width="360" height="208" rx="12" fill="#181C20" stroke="#2E3339"/>
+  <rect x="460" y="56" width="360" height="208" rx="12" fill="#181C20" stroke="#2E3339"/>
+  <rect x="864" y="56" width="360" height="208" rx="12" fill="#181C20" stroke="#2E3339"/>
+
+  <text x="80" y="88" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="28" font-weight="700">mapr</text>
+  <text x="80" y="114" fill="#8E877C" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">frontend reverse engineering for bun</text>
+
+  <text x="80" y="154" fill="#C67A3B" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="13">crawl</text>
+  <text x="484" y="88" fill="#C67A3B" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="13">artifacts</text>
+  <text x="888" y="88" fill="#C67A3B" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="13">report</text>
+
+  <rect x="80" y="172" width="170" height="20" rx="6" fill="#22272C"/>
+  <rect x="80" y="204" width="240" height="20" rx="6" fill="#22272C"/>
+  <rect x="80" y="236" width="120" height="12" rx="6" fill="#7F6B52"/>
+  <rect x="210" y="236" width="112" height="12" rx="6" fill="#22272C"/>
+  <rect x="332" y="236" width="42" height="12" rx="6" fill="#22272C"/>
+
+  <rect x="500" y="114" width="88" height="36" rx="8" fill="#1F2428" stroke="#343A41"/>
+  <rect x="596" y="114" width="88" height="36" rx="8" fill="#1F2428" stroke="#343A41"/>
+  <rect x="692" y="114" width="88" height="36" rx="8" fill="#1F2428" stroke="#343A41"/>
+  <rect x="500" y="164" width="88" height="36" rx="8" fill="#1F2428" stroke="#343A41"/>
+  <rect x="596" y="164" width="88" height="36" rx="8" fill="#1F2428" stroke="#343A41"/>
+  <rect x="692" y="164" width="88" height="36" rx="8" fill="#1F2428" stroke="#343A41"/>
+  <text x="530" y="136" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">js</text>
+  <text x="618" y="136" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">sw</text>
+  <text x="705" y="136" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">map</text>
+  <text x="522" y="186" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">wasm</text>
+  <text x="607" y="186" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">worker</text>
+  <text x="713" y="186" fill="#EEE6DA" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="14">rag</text>
+
+  <rect x="888" y="112" width="312" height="120" rx="10" fill="#1D2226" stroke="#343941"/>
+  <rect x="914" y="134" width="84" height="8" rx="4" fill="#C67A3B"/>
+  <rect x="914" y="156" width="182" height="8" rx="4" fill="#444B52"/>
+  <rect x="914" y="176" width="212" height="8" rx="4" fill="#444B52"/>
+  <rect x="914" y="196" width="168" height="8" rx="4" fill="#444B52"/>
+  <rect x="1108" y="156" width="66" height="52" rx="8" fill="#171B1E" stroke="#343941"/>
+  <path d="M1132 171H1150M1132 183H1148M1132 195H1144" stroke="#EEE6DA" stroke-width="2" stroke-linecap="round"/>
+
+  <path d="M392 160C414 160 426 160 446 160" stroke="#7F6B52" stroke-width="2.5" stroke-linecap="round"/>
+  <path d="M796 160C816 160 828 160 848 160" stroke="#7F6B52" stroke-width="2.5" stroke-linecap="round"/>
+  <path d="M438 160L426 154V166L438 160Z" fill="#C67A3B"/>
+  <path d="M840 160L828 154V166L840 160Z" fill="#C67A3B"/>
+
+  <text x="930" y="254" fill="#8E877C" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="12">crawl -> analyze -> report</text>
+</svg>

package/lib/scraper.ts

@@ -37,6 +37,7 @@ export interface ScrapeResult {
 type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
 type NumericScraperOptions = z.input<typeof scraperOptionsSchema>;
 type QueueEntry = { candidate: ArtifactCandidate; depth: number };
+type CrawlScope = "site" | "page";
 
 export interface ScraperProgressEvent {
   message: string;
@@ -53,6 +54,11 @@ function isPageCandidate(candidate: ArtifactCandidate, rootOrigin: string): bool
   return candidate.type === "html" && new URL(candidate.url).origin === rootOrigin;
 }
 
+function isRootLikeEntry(url: string): boolean {
+  const pathname = new URL(url).pathname.toLowerCase();
+  return pathname === "/" || pathname === "" || pathname.endsWith("/index.html") || pathname.endsWith("/index.htm");
+}
+
 function shouldFollowCandidate(candidate: ArtifactCandidate, rootOrigin: string): boolean {
   if (candidate.type === "html") {
     return new URL(candidate.url).origin === rootOrigin;
@@ -150,6 +156,7 @@ export class BundleScraper {
   public async scrape(pageUrl: string): Promise<ScrapeResult> {
     const validatedPageUrl = httpUrlSchema.parse(pageUrl);
     const rootOrigin = new URL(validatedPageUrl).origin;
+    const crawlScope: CrawlScope = isRootLikeEntry(validatedPageUrl) ? "site" : "page";
     const visitedUrls = new Set<string>();
     const htmlPages = new Set<string>();
     const artifacts: DiscoveredArtifact[] = [];
@@ -164,7 +171,9 @@ export class BundleScraper {
       },
     ];
 
-    queue.push(...(await this.discoverSupplementalPages(rootOrigin)).map((candidate) => ({ candidate, depth: 1 })));
+    if (crawlScope === "site") {
+      queue.push(...(await this.discoverSupplementalPages(rootOrigin)).map((candidate) => ({ candidate, depth: 1 })));
+    }
 
     while (queue.length > 0) {
       if (artifacts.length >= this.options.maxArtifacts) {
@@ -204,7 +213,7 @@ export class BundleScraper {
         depth,
       });
 
-      const artifact = await this.fetchArtifact(candidate, depth);
+      const artifact = await this.fetchArtifact(candidate, depth, candidate.url === validatedPageUrl);
       if (!artifact) {
         continue;
       }
@@ -217,7 +226,7 @@ export class BundleScraper {
         artifacts.push(artifact);
       }
 
-      const nestedCandidates = extractNestedCandidates(artifact);
+      const nestedCandidates = this.filterNestedCandidates(extractNestedCandidates(artifact), validatedPageUrl, crawlScope);
       for (const nestedCandidate of nestedCandidates) {
         if (!visitedUrls.has(nestedCandidate.url)) {
           queue.push({ candidate: nestedCandidate, depth: depth + 1 });
@@ -279,8 +288,12 @@ export class BundleScraper {
     return [...candidates.values()];
   }
 
-  private async fetchArtifact(candidate: ArtifactCandidate, depth: number): Promise<DiscoveredArtifact | null> {
-    const response = await this.fetchResponse(candidate.url, candidate.type);
+  private async fetchArtifact(candidate: ArtifactCandidate, depth: number, required: boolean): Promise<DiscoveredArtifact | null> {
+    const response = await this.fetchResponse(candidate.url, candidate.type, depth, required);
+    if (!response) {
+      return null;
+    }
+
     const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
 
     if (isIgnoredContentType(contentType)) {
@@ -335,7 +348,12 @@ export class BundleScraper {
     });
   }
 
-  private async fetchResponse(url: string, artifactType: ArtifactCandidate["type"]): Promise<Response> {
+  private async fetchResponse(
+    url: string,
+    artifactType: ArtifactCandidate["type"],
+    depth: number,
+    required: boolean,
+  ): Promise<Response | null> {
     try {
       const response = await this.fetcher(url, {
         headers: {
@@ -344,11 +362,31 @@ export class BundleScraper {
       });
 
       if (!response.ok) {
-        throw new Error(`Failed to fetch ${artifactType} from ${url}: ${response.status} ${response.statusText}`);
+        if (required) {
+          throw new Error(`Failed to fetch ${artifactType} from ${url}: ${response.status} ${response.statusText}`);
+        }
+
+        this.emitProgress({
+          message: `Skipping ${artifactType} after ${response.status} ${response.statusText}: ${url}`,
+          url,
+          type: artifactType,
+          depth,
+        });
+        return null;
       }
 
       return response;
     } catch (error) {
+      if (!required) {
+        this.emitProgress({
+          message: `Skipping ${artifactType} after fetch error: ${url}`,
+          url,
+          type: artifactType,
+          depth,
+        });
+        return null;
+      }
+
       if (error instanceof Error) {
         throw new Error(`Unable to fetch ${artifactType} artifact ${url}: ${error.message}`);
       }
@@ -378,6 +416,50 @@ export class BundleScraper {
   private emitProgress(event: ScraperProgressEvent): void {
     this.onProgress?.(event);
   }
+
+  private filterNestedCandidates(
+    candidates: ArtifactCandidate[],
+    entryUrl: string,
+    crawlScope: CrawlScope,
+  ): ArtifactCandidate[] {
+    if (crawlScope === "site") {
+      return candidates;
+    }
+
+    const entryPath = new URL(entryUrl).pathname.toLowerCase();
+    const entryStem = entryPath.replace(/(?:index)?\.html?$/i, "").replace(/\/+$/, "") || entryPath;
+    const entryDirectory = entryPath.includes("/") ? entryPath.slice(0, entryPath.lastIndexOf("/") + 1) : "/";
+
+    return candidates.filter((candidate) => {
+      if (candidate.type !== "html") {
+        return true;
+      }
+
+      const discoveredFrom = candidate.discoveredFrom.toLowerCase();
+      if (discoveredFrom.includes("iframe") || discoveredFrom.includes("form")) {
+        return true;
+      }
+
+      const candidatePath = new URL(candidate.url).pathname.toLowerCase();
+      if (candidatePath === entryPath) {
+        return true;
+      }
+
+      if (entryDirectory !== "/") {
+        return candidatePath.startsWith(entryDirectory);
+      }
+
+      if (entryStem !== entryPath && candidatePath.startsWith(entryStem)) {
+        return true;
+      }
+
+      if (candidatePath.startsWith(`${entryPath}/`)) {
+        return true;
+      }
+
+      return false;
+    });
+  }
 }
 
 export { extractArtifactCandidates };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redstone-md/mapr",
-  "version": "0.0.2-alpha",
+  "version": "0.0.3-alpha",
   "type": "module",
   "description": "Bun-native CLI/TUI for reverse-engineering frontend websites, bundles, WASM, and service workers",
   "license": "SEE LICENSE IN LICENSE",
@@ -30,6 +30,7 @@
     "mapr": "./bin/mapr"
   },
   "files": [
+    "assets",
     "bin",
     "index.ts",
     "lib",