@akotliar/sitemap-qa 1.0.0-alpha.4 → 1.0.0-alpha.6

package/dist/index.js

@@ -6,7 +6,7 @@ import { Command as Command3 } from "commander";
 // src/commands/analyze.ts
 import { Command } from "commander";
 import chalk3 from "chalk";
-import path2 from "path";
+import path3 from "path";
 import fs4 from "fs/promises";
 
 // src/config/loader.ts
@@ -27,13 +27,18 @@ var PolicySchema = z.object({
   patterns: z.array(PatternSchema).min(1, "At least one pattern is required per category")
 });
 var ConfigSchema = z.object({
+  acceptable_patterns: z.array(PatternSchema).default([]),
   policies: z.array(PolicySchema).default([]),
   outDir: z.string().optional(),
-  outputFormat: z.enum(["json", "html", "all"]).default("all")
+  outputFormat: z.enum(["json", "html", "all"]).default("all"),
+  enforceDomainConsistency: z.boolean().default(true)
 });
 
 // src/config/defaults.ts
 var DEFAULT_POLICIES = {
+  acceptable_patterns: [],
+  outputFormat: "all",
+  enforceDomainConsistency: true,
   policies: [
     {
       category: "Security & Admin",
@@ -89,7 +94,12 @@ var ConfigLoader = class {
   static DEFAULT_CONFIG_PATH = "sitemap-qa.yaml";
   static load(configPath) {
     const targetPath = configPath || path.join(process.cwd(), this.DEFAULT_CONFIG_PATH);
-    let userConfig = { policies: [] };
+    let userConfig = {
+      acceptable_patterns: [],
+      policies: [],
+      outputFormat: "all",
+      enforceDomainConsistency: true
+    };
     if (fs.existsSync(targetPath)) {
       try {
         const fileContent = fs.readFileSync(targetPath, "utf8");
@@ -101,15 +111,18 @@ var ConfigLoader = class {
             console.error(chalk.yellow(`  - ${issue.path.join(".")}: ${issue.message}`));
           });
           process.exit(2);
+          return DEFAULT_POLICIES;
         }
         userConfig = result.data;
       } catch (error) {
         console.error(chalk.red("Failed to load configuration:"), error);
         process.exit(2);
+        return DEFAULT_POLICIES;
       }
     } else if (configPath) {
       console.error(chalk.red(`Error: Configuration file not found at ${targetPath}`));
       process.exit(2);
+      return DEFAULT_POLICIES;
     }
     return this.mergeConfigs(DEFAULT_POLICIES, userConfig);
   }
@@ -125,6 +138,7 @@ var ConfigLoader = class {
     });
     const merged = {
       ...defaults,
+      acceptable_patterns: [...defaults.acceptable_patterns || [], ...user.acceptable_patterns || []],
       policies: mergedPolicies
     };
     if (user.outDir !== void 0) {
@@ -133,15 +147,90 @@ var ConfigLoader = class {
     if (user.outputFormat !== void 0) {
       merged.outputFormat = user.outputFormat;
     }
+    if (user.enforceDomainConsistency !== void 0) {
+      merged.enforceDomainConsistency = user.enforceDomainConsistency;
+    }
     return merged;
   }
 };
 
 // src/core/discovery.ts
 import { fetch } from "undici";
+import { Readable } from "stream";
+
+// src/core/xml-parser.ts
 import { XMLParser } from "fast-xml-parser";
-var DiscoveryService = class {
+import { gunzipSync } from "zlib";
+var StreamingXmlParser = class {
   parser;
+  lastParsedXml;
+  constructor() {
+    this.parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: "@_",
+      // Ensure we always get arrays for sitemap and url tags
+      isArray: (name) => name === "sitemap" || name === "url",
+      removeNSPrefix: true
+    });
+  }
+  /**
+   * Parses an XML stream and yields typed entries as they are found.
+   * Generator-first design allows consumers to process entries without pre-collecting.
+   */
+  async *parse(stream) {
+    const xmlData = typeof stream === "string" ? stream : await this.streamToString(stream);
+    this.lastParsedXml = xmlData;
+    const jsonObj = this.parser.parse(xmlData);
+    if (jsonObj.sitemapindex?.sitemap) {
+      const sitemaps = jsonObj.sitemapindex.sitemap;
+      for (const sitemap of sitemaps) {
+        if (sitemap?.loc) {
+          yield { type: "sitemap", loc: sitemap.loc };
+        }
+      }
+    }
+    if (jsonObj.urlset?.url) {
+      const urls = jsonObj.urlset.url;
+      for (const url of urls) {
+        if (url?.loc) {
+          yield {
+            type: "url",
+            loc: url.loc,
+            lastmod: url.lastmod,
+            changefreq: url.changefreq,
+            priority: url.priority
+          };
+        }
+      }
+    }
+  }
+  /**
+   * Get the last parsed XML data (useful to avoid re-fetching).
+   */
+  getLastParsedXml() {
+    return this.lastParsedXml;
+  }
+  async streamToString(stream) {
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(Buffer.from(chunk));
+    }
+    const buffer = Buffer.concat(chunks);
+    if (buffer.length >= 2 && buffer[0] === 31 && buffer[1] === 139) {
+      try {
+        const decompressed = gunzipSync(buffer);
+        return decompressed.toString("utf8");
+      } catch (error) {
+        throw new Error(`Failed to decompress gzipped content: ${error}`);
+      }
+    }
+    return buffer.toString("utf8");
+  }
+};
+
+// src/core/discovery.ts
+var DiscoveryService = class {
+  xmlParser;
   visited = /* @__PURE__ */ new Set();
   STANDARD_PATHS = [
     "/sitemap.xml",
@@ -151,10 +240,7 @@ var DiscoveryService = class {
     "/sitemap.xml.gz"
   ];
   constructor() {
-    this.parser = new XMLParser({
-      ignoreAttributes: false,
-      attributeNamePrefix: "@_"
-    });
+    this.xmlParser = new StreamingXmlParser();
   }
   /**
    * Attempts to find sitemaps for a given base website URL.
@@ -176,9 +262,9 @@ var DiscoveryService = class {
     } catch (e) {
     }
     if (sitemaps.size === 0) {
-      for (const path4 of this.STANDARD_PATHS) {
+      for (const path5 of this.STANDARD_PATHS) {
         try {
-          const sitemapUrl = `${origin}${path4}`;
+          const sitemapUrl = `${origin}${path5}`;
           const response = await fetch(sitemapUrl, { method: "HEAD" });
           if (response.status === 200) {
             sitemaps.add(sitemapUrl);
@@ -191,6 +277,7 @@ var DiscoveryService = class {
   }
   /**
    * Recursively discovers all leaf sitemaps from a root URL.
+   * Returns both the sitemap URL and its XML data to avoid duplicate fetches.
    */
   async *discover(rootUrl) {
     const queue = [rootUrl];
@@ -201,17 +288,39 @@ var DiscoveryService = class {
       try {
         const response = await fetch(currentUrl);
         if (response.status !== 200) continue;
-        const xmlData = await response.text();
-        const jsonObj = this.parser.parse(xmlData);
-        if (jsonObj.sitemapindex) {
-          const sitemaps = Array.isArray(jsonObj.sitemapindex.sitemap) ? jsonObj.sitemapindex.sitemap : [jsonObj.sitemapindex.sitemap];
-          for (const sitemap of sitemaps) {
-            if (sitemap?.loc) {
-              queue.push(sitemap.loc);
-            }
+        let isIndex = false;
+        let isLeaf = false;
+        const childSitemaps = [];
+        let xmlData;
+        let source;
+        if (response.body) {
+          const nodeStream = Readable.fromWeb(response.body);
+          source = nodeStream;
+        } else {
+          xmlData = await response.text();
+          source = xmlData;
+        }
+        for await (const entry of this.xmlParser.parse(source)) {
+          if (entry.type === "sitemap") {
+            isIndex = true;
+            childSitemaps.push(entry.loc);
+          } else if (entry.type === "url") {
+            isLeaf = true;
           }
-        } else if (jsonObj.urlset) {
-          yield currentUrl;
+        }
+        if (isIndex) {
+          for (const loc of childSitemaps) {
+            queue.push(loc);
+          }
+        }
+        if (!xmlData) {
+          xmlData = this.xmlParser.getLastParsedXml() || "";
+        }
+        if (isLeaf || !isIndex && xmlData.includes("<urlset")) {
+          yield {
+            url: currentUrl,
+            xmlData
+          };
         }
       } catch (error) {
         console.error(`Failed to fetch or parse sitemap at ${currentUrl}:`, error);
@@ -221,47 +330,78 @@ var DiscoveryService = class {
 };
 
 // src/core/parser.ts
-import { XMLParser as XMLParser2 } from "fast-xml-parser";
+import { Readable as Readable2 } from "stream";
 import { fetch as fetch2 } from "undici";
 var SitemapParser = class {
-  parser;
+  xmlParser;
   constructor() {
-    this.parser = new XMLParser2({
-      ignoreAttributes: false,
-      attributeNamePrefix: "@_"
-    });
+    this.xmlParser = new StreamingXmlParser();
   }
   /**
    * Parses a leaf sitemap and yields SitemapUrl objects.
-   * Note: For true streaming of massive files, we'd use a SAX-like approach.
-   * fast-xml-parser's parse() is fast but loads the whole string.
-   * Given the 50k URL requirement, we'll use a more memory-efficient approach if needed,
-   * but let's start with a clean AsyncGenerator interface.
+   * Uses the shared StreamingXmlParser for consistent and efficient parsing.
+   * 
+   * @param sitemapUrlOrData - Accepts one of three input types:
+   *   - `string`: A URL string. The method will fetch the sitemap from this URL.
+   *     Use this when you need to fetch a sitemap from a remote location.
+   *   - `{ type: 'xmlData'; url: string; xmlData: string }`: An object with a URL and pre-fetched XML data.
+   *     Use this when you already have the XML content (e.g., from a cache or file)
+   *     and want to avoid an additional HTTP request.
+   *   - `{ type: 'stream'; url: string; stream: ReadableStream | Readable }`: An object with a URL and a stream.
+   *     Accepts either a Web ReadableStream or Node.js Readable stream.
+   *     Use this when you have a stream source (e.g., from a streaming HTTP response)
+   *     that should be consumed and parsed. Web streams are converted to Node.js Readable internally.
+   * 
+   * @yields {SitemapUrl} Parsed sitemap URL entries containing `loc` (URL), `source` (sitemap URL),
+   *   optional metadata (`lastmod`, `changefreq`, `priority`), and a `risks` array (initialized as empty,
+   *   populated later in the processing pipeline). Other properties like `ignored`/`ignoredBy` are not
+   *   set by this method and may be added by downstream processors.
    */
-  async *parse(sitemapUrl) {
+  async *parse(sitemapUrlOrData) {
+    const sitemapUrl = typeof sitemapUrlOrData === "string" ? sitemapUrlOrData : sitemapUrlOrData.url;
     try {
-      const response = await fetch2(sitemapUrl);
-      const xmlData = await response.text();
-      const jsonObj = this.parser.parse(xmlData);
-      if (jsonObj.urlset && jsonObj.urlset.url) {
-        const urls = Array.isArray(jsonObj.urlset.url) ? jsonObj.urlset.url : [jsonObj.urlset.url];
-        for (const url of urls) {
-          if (url.loc) {
-            yield {
-              loc: url.loc,
-              source: sitemapUrl,
-              lastmod: url.lastmod,
-              changefreq: url.changefreq,
-              priority: url.priority,
-              risks: []
-            };
-          }
+      let source;
+      if (typeof sitemapUrlOrData === "string") {
+        const response = await fetch2(sitemapUrl);
+        if (response.status !== 200) throw new Error(`Failed to fetch sitemap at ${sitemapUrl}: HTTP ${response.status}`);
+        if (response.body) {
+          source = Readable2.fromWeb(response.body);
+        } else {
+          source = await response.text();
+        }
+      } else if (sitemapUrlOrData.type === "stream") {
+        if (sitemapUrlOrData.stream instanceof Readable2) {
+          source = sitemapUrlOrData.stream;
+        } else {
+          source = Readable2.fromWeb(sitemapUrlOrData.stream);
+        }
+      } else {
+        source = sitemapUrlOrData.xmlData;
+        source = sitemapUrlOrData.xmlData;
+      }
+      for await (const entry of this.xmlParser.parse(source)) {
+        if (entry.type === "url") {
+          yield {
+            loc: entry.loc,
+            source: sitemapUrl,
+            lastmod: entry.lastmod,
+            changefreq: entry.changefreq,
+            priority: entry.priority,
+            risks: []
+          };
         }
       }
     } catch (error) {
       console.error(`Failed to parse sitemap at ${sitemapUrl}:`, error);
     }
   }
+  async streamToString(stream) {
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks).toString("utf8");
+  }
 };
 
 // src/core/extractor.ts
@@ -308,9 +448,10 @@ var ExtractorService = class {
       }
     }
     for (const startUrl of startUrls) {
-      for await (const sitemapUrl of this.discovery.discover(startUrl)) {
-        this.discoveredSitemaps.add(sitemapUrl);
-        for await (const urlObj of this.parser.parse(sitemapUrl)) {
+      for await (const discovered of this.discovery.discover(startUrl)) {
+        this.discoveredSitemaps.add(discovered.url);
+        const parserInput = { type: "xmlData", url: discovered.url, xmlData: discovered.xmlData };
+        for await (const urlObj of this.parser.parse(parserInput)) {
           const normalized = this.normalizeUrl(urlObj.loc);
           if (!this.seenUrls.has(normalized)) {
             this.seenUrls.add(normalized);
@@ -326,14 +467,42 @@ var ExtractorService = class {
 import micromatch from "micromatch";
 var MatcherService = class {
   config;
-  constructor(config) {
+  rootDomain;
+  constructor(config, rootUrl) {
     this.config = config;
+    if (rootUrl) {
+      try {
+        this.rootDomain = new URL(rootUrl).hostname.replace(/^www\./, "");
+      } catch {
+      }
+    }
   }
   /**
    * Matches a URL against all policies and returns detected risks.
    */
   match(urlObj) {
     const risks = [];
+    if (this.config.enforceDomainConsistency && this.rootDomain) {
+      try {
+        const currentDomain = new URL(urlObj.loc).hostname.replace(/^www\./, "");
+        if (currentDomain !== this.rootDomain) {
+          risks.push({
+            category: "Domain Consistency",
+            pattern: this.rootDomain,
+            type: "literal",
+            reason: `URL domain mismatch: expected ${this.rootDomain} (or www.${this.rootDomain}), but found ${currentDomain}.`
+          });
+        }
+      } catch {
+      }
+    }
+    for (const pattern of this.config.acceptable_patterns) {
+      if (this.isMatch(urlObj.loc, pattern)) {
+        urlObj.ignored = true;
+        urlObj.ignoredBy = pattern.reason;
+        return risks;
+      }
+    }
     for (const policy of this.config.policies) {
       for (const pattern of policy.patterns) {
         if (this.isMatch(urlObj.loc, pattern)) {
@@ -375,6 +544,7 @@ var ConsoleReporter = class {
     console.log(`Total URLs Scanned: ${data.totalUrls}`);
     console.log(`Total Risks Found:  ${data.totalRisks > 0 ? chalk2.red(data.totalRisks) : chalk2.green(0)}`);
     console.log(`URLs with Risks:    ${data.urlsWithRisks.length}`);
+    console.log(`URLs Ignored:       ${data.ignoredUrls.length > 0 ? chalk2.yellow(data.ignoredUrls.length) : 0}`);
     console.log(`Duration:           ${((data.endTime.getTime() - data.startTime.getTime()) / 1e3).toFixed(2)}s`);
     if (data.urlsWithRisks.length > 0) {
       console.log("\n" + chalk2.bold.yellow("Top Findings:"));
@@ -410,9 +580,11 @@ var JsonReporter = class {
       summary: {
         totalUrls: data.totalUrls,
         totalRisks: data.totalRisks,
-        urlsWithRisksCount: data.urlsWithRisks.length
+        urlsWithRisksCount: data.urlsWithRisks.length,
+        ignoredUrlsCount: data.ignoredUrls.length
       },
-      findings: data.urlsWithRisks
+      findings: data.urlsWithRisks,
+      ignored: data.ignoredUrls
     };
     await fs2.writeFile(this.outputPath, JSON.stringify(report, null, 2), "utf8");
     console.log(`JSON report generated at ${this.outputPath}`);
@@ -421,322 +593,92 @@ var JsonReporter = class {
 
 // src/reporters/html-reporter.ts
 import fs3 from "fs/promises";
+import path2 from "path";
+import { fileURLToPath } from "url";
+import Handlebars from "handlebars";
+var __filename2 = fileURLToPath(import.meta.url);
+var __dirname2 = path2.dirname(__filename2);
 var HtmlReporter = class {
   outputPath;
   constructor(outputPath = "sitemap-qa-report.html") {
     this.outputPath = outputPath;
+    Handlebars.registerHelper("json", (context) => {
+      return JSON.stringify(context);
+    });
   }
   async generate(data) {
-    const categories = this.groupRisks(data);
-    const html = this.generateHtml(data, categories);
+    const partialsDir = path2.join(__dirname2, "templates", "partials");
+    try {
+      const partialFiles = await fs3.readdir(partialsDir);
+      for (const file of partialFiles) {
+        if (file.endsWith(".hbs")) {
+          const partialName = path2.basename(file, ".hbs");
+          const partialSource = await fs3.readFile(path2.join(partialsDir, file), "utf8");
+          Handlebars.registerPartial(partialName, partialSource);
+        }
+      }
+    } catch (error) {
+      console.warn("Could not load partials:", error);
+    }
+    const templatePath = path2.join(__dirname2, "templates", "report.hbs");
+    const templateSource = await fs3.readFile(templatePath, "utf8");
+    const template = Handlebars.compile(templateSource);
+    const templateData = this.prepareTemplateData(data);
+    const html = template(templateData);
     await fs3.writeFile(this.outputPath, html, "utf8");
     console.log(`HTML report generated at ${this.outputPath}`);
   }
-  groupRisks(data) {
-    const categories = {};
+  prepareTemplateData(data) {
+    const duration = ((data.endTime.getTime() - data.startTime.getTime()) / 1e3).toFixed(1);
+    const timestamp = data.endTime.toLocaleString();
+    const categoriesMap = {};
     for (const urlObj of data.urlsWithRisks) {
       for (const risk of urlObj.risks) {
-        if (!categories[risk.category]) {
-          categories[risk.category] = {};
+        if (!categoriesMap[risk.category]) {
+          categoriesMap[risk.category] = {};
         }
-        if (!categories[risk.category][risk.pattern]) {
-          categories[risk.category][risk.pattern] = {
+        if (!categoriesMap[risk.category][risk.pattern]) {
+          categoriesMap[risk.category][risk.pattern] = {
             reason: risk.reason,
             urls: []
           };
         }
-        categories[risk.category][risk.pattern].urls.push(urlObj.loc);
+        categoriesMap[risk.category][risk.pattern].urls.push(urlObj.loc);
       }
     }
-    return categories;
-  }
-  generateHtml(data, categories) {
-    const duration = ((data.endTime.getTime() - data.startTime.getTime()) / 1e3).toFixed(1);
-    const timestamp = data.endTime.toLocaleString();
-    return `
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Sitemap Analysis - ${data.rootUrl}</title>
-    <style>
-        :root {
-            --bg-dark: #0f172a;
-            --bg-light: #f8fafc;
-            --text-main: #1e293b;
-            --text-muted: #64748b;
-            --primary: #3b82f6;
-            --danger: #ef4444;
-            --warning: #f59e0b;
-            --border: #e2e8f0;
-        }
-        body { 
-            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-            line-height: 1.5;
-            color: var(--text-main);
-            background-color: #fff;
-            margin: 0;
-            padding: 0;
-        }
-        header {
-            background-color: var(--bg-dark);
-            color: white;
-            padding: 40px 20px;
-            text-align: left;
-        }
-        .container {
-            max-width: 1200px;
-            margin: 0 auto;
-            padding: 0 20px;
-        }
-        header h1 { margin: 0; font-size: 24px; }
-        header .meta { margin-top: 10px; color: #94a3b8; font-size: 14px; }
-        
-        .summary-grid {
-            display: grid;
-            grid-template-columns: repeat(4, 1fr);
-            border-bottom: 1px solid var(--border);
-            margin-bottom: 40px;
-        }
-        .summary-card {
-            padding: 30px 20px;
-            text-align: center;
-            border-right: 1px solid var(--border);
-        }
-        .summary-card:last-child { border-right: none; }
-        .summary-card h3 { 
-            margin: 0; 
-            font-size: 12px; 
-            text-transform: uppercase; 
-            color: var(--text-muted);
-            letter-spacing: 0.05em;
-        }
-        .summary-card p { 
-            margin: 10px 0 0; 
-            font-size: 32px; 
-            font-weight: 700; 
-            color: var(--text-main);
-        }
-        .summary-card.highlight p { color: var(--danger); }
-
-        details {
-            margin-bottom: 20px;
-            border: 1px solid var(--border);
-            border-radius: 8px;
-            overflow: hidden;
-        }
-        summary {
-            padding: 15px 20px;
-            background-color: #fff;
-            cursor: pointer;
-            font-weight: 600;
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-            list-style: none;
-        }
-        summary::-webkit-details-marker { display: none; }
-        summary::after {
-            content: '\u25B6';
-            font-size: 12px;
-            color: var(--text-muted);
-            transition: transform 0.2s;
-        }
-        details[open] summary::after { transform: rotate(90deg); }
-        
-        .category-section {
-            border: 1px solid var(--warning);
-            border-radius: 8px;
-            margin-bottom: 20px;
-        }
-        .category-header {
-            padding: 15px 20px;
-            background-color: #fffbeb;
-            color: var(--warning);
-            font-weight: 600;
-            cursor: pointer;
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-        }
-        .category-content {
-            padding: 20px;
-            background-color: #fff;
-        }
-
-        .finding-group {
-            border: 1px solid var(--border);
-            border-radius: 8px;
-            padding: 20px;
-            margin-bottom: 20px;
-        }
-        .finding-header {
-            display: flex;
-            align-items: center;
-            gap: 10px;
-            margin-bottom: 10px;
-        }
-        .finding-header h4 { margin: 0; font-size: 16px; }
-        .badge {
-            background-color: var(--primary);
-            color: white;
-            padding: 2px 8px;
-            border-radius: 12px;
-            font-size: 12px;
-        }
-        .finding-description {
-            color: var(--text-muted);
-            font-size: 14px;
-            margin-bottom: 20px;
-        }
-        
-        .url-list {
-            background-color: var(--bg-light);
-            border-radius: 4px;
-            padding: 15px;
-            margin-bottom: 15px;
-        }
-        .url-item {
-            font-family: monospace;
-            font-size: 13px;
-            padding: 8px 12px;
-            background: white;
-            border: 1px solid var(--border);
-            border-radius: 4px;
-            margin-bottom: 8px;
-            white-space: nowrap;
-            overflow: hidden;
-            text-overflow: ellipsis;
-        }
-        .url-item:last-child { margin-bottom: 0; }
-        
-        .more-count {
-            font-size: 12px;
-            color: var(--text-muted);
-            font-style: italic;
-            margin-bottom: 15px;
-        }
-
-        .btn {
-            display: inline-flex;
-            align-items: center;
-            gap: 8px;
-            background-color: var(--primary);
-            color: white;
-            padding: 8px 16px;
-            border-radius: 6px;
-            text-decoration: none;
-            font-size: 13px;
-            font-weight: 500;
-        }
-        .btn:hover { opacity: 0.9; }
-
-        footer {
-            text-align: center;
-            padding: 40px;
-            color: var(--text-muted);
-            font-size: 12px;
-            border-top: 1px solid var(--border);
-            margin-top: 40px;
-        }
-    </style>
-</head>
-<body>
-    <header>
-        <div class="container">
-            <h1>Sitemap Analysis</h1>
-            <div class="meta">
-                <div>${data.rootUrl}</div>
-                <div>${timestamp}</div>
-            </div>
-        </div>
-    </header>
-
-    <div class="summary-grid">
-        <div class="summary-card">
-            <h3>Sitemaps</h3>
-            <p>${data.discoveredSitemaps.length}</p>
-        </div>
-        <div class="summary-card">
-            <h3>URLs Analyzed</h3>
-            <p>${data.totalUrls.toLocaleString()}</p>
-        </div>
-        <div class="summary-card highlight">
-            <h3>Issues Found</h3>
-            <p>${data.totalRisks}</p>
-        </div>
-        <div class="summary-card">
-            <h3>Scan Time</h3>
-            <p>${duration}s</p>
-        </div>
-    </div>
-
-    <div class="container">
-        <details>
-            <summary>Sitemaps Discovered (${data.discoveredSitemaps.length})</summary>
-            <div style="padding: 20px; background: var(--bg-light);">
-                ${data.discoveredSitemaps.map((s) => `<div class="url-item">${s}</div>`).join("")}
-            </div>
-        </details>
-
-        ${Object.entries(categories).map(([category, findings]) => {
-      const totalCategoryUrls = Object.values(findings).reduce((acc, f) => acc + f.urls.length, 0);
-      return `
-            <div class="category-section">
-                <div class="category-header">
-                    <span>${category} (${totalCategoryUrls} URLs)</span>
-                    <span>\u25BC</span>
-                </div>
-                <div class="category-content">
-                    ${Object.entries(findings).map(([pattern, finding]) => `
-                        <div class="finding-group">
-                            <div class="finding-header">
-                                <h4>${pattern}</h4>
-                                <span class="badge">${finding.urls.length} URLs</span>
-                            </div>
-                            <div class="finding-description">
-                                ${finding.reason}
-                            </div>
-                            <div class="url-list">
-                                ${finding.urls.slice(0, 3).map((url) => `
-                                    <div class="url-item">${url}</div>
-                                `).join("")}
-                            </div>
-                            ${finding.urls.length > 3 ? `
-                                <div class="more-count">... and ${finding.urls.length - 3} more</div>
-                            ` : ""}
-                            <a href="#" class="btn" onclick="downloadUrls('${pattern}', ${JSON.stringify(finding.urls).replace(/"/g, "&quot;")})">
-                                \u{1F4E5} Download All ${finding.urls.length} URLs
-                            </a>
-                        </div>
-                    `).join("")}
-                </div>
-            </div>
-            `;
-    }).join("")}
-    </div>
-
-    <footer>
-        Generated by sitemap-qa v1.0.0
-    </footer>
-
-    <script>
-        function downloadUrls(name, urls) {
-            const blob = new Blob([urls.join('\\n')], { type: 'text/plain' });
-            const url = window.URL.createObjectURL(blob);
-            const a = document.createElement('a');
-            a.href = url;
-            a.download = \`\${name.replace(/[^a-z0-9]/gi, '_').toLowerCase()}_urls.txt\`;
-            document.body.appendChild(a);
-            a.click();
-            window.URL.revokeObjectURL(url);
-            document.body.removeChild(a);
-        }
-    </script>
-</body>
-</html>
-`;
+    const categories = Object.entries(categoriesMap).map(([name, findingsMap]) => {
+      const findings = Object.entries(findingsMap).map(([pattern, finding]) => ({
+        pattern,
+        urls: finding.urls,
+        reason: finding.reason,
+        displayUrls: finding.urls.slice(0, 3),
+        moreCount: finding.urls.length > 3 ? finding.urls.length - 3 : 0
+      }));
+      const totalUrls = findings.reduce((acc, f) => acc + f.urls.length, 0);
+      return {
+        name,
+        totalUrls,
+        findings
+      };
+    });
+    const ignoredUrls = data.ignoredUrls.map((u) => {
+      const suppressedCategories = u.risks.length > 0 ? [...new Set(u.risks.map((r) => r.category))].join(", ") : void 0;
+      return {
+        loc: u.loc,
+        ignoredBy: u.ignoredBy ?? "Unknown",
+        suppressedCategories
+      };
+    });
+    return {
+      rootUrl: data.rootUrl,
+      timestamp,
+      discoveredSitemaps: data.discoveredSitemaps,
+      totalUrls: data.totalUrls.toLocaleString(),
+      totalRisks: data.totalRisks,
+      ignoredUrls,
+      duration,
+      categories
+    };
   }
 };
 
@@ -747,12 +689,13 @@ var analyzeCommand = new Command("analyze").description("Analyze a sitemap for p
   const outDir = options.outDir || config.outDir || ".";
   const outputFormat = options.output || config.outputFormat || "all";
   const extractor = new ExtractorService();
-  const matcher = new MatcherService(config);
+  const matcher = new MatcherService(config, url);
   const urlsWithRisks = [];
+  const ignoredUrls = [];
   let totalUrls = 0;
   let totalRisks = 0;
   console.log(chalk3.blue(`
-\uFFFD\uFFFD\uFFFD Starting analysis of ${url}...`));
+\u{1F680} Starting analysis of ${url}...`));
   try {
     for await (const urlObj of extractor.extract(url)) {
       totalUrls++;
@@ -761,6 +704,8 @@ var analyzeCommand = new Command("analyze").description("Analyze a sitemap for p
         urlObj.risks = risks;
         urlsWithRisks.push(urlObj);
         totalRisks += risks.length;
+      } else if (urlObj.ignored) {
+        ignoredUrls.push(urlObj);
       }
       if (totalUrls % 100 === 0) {
         process.stdout.write(chalk3.gray(`\rProcessed ${totalUrls} URLs...`));
@@ -774,17 +719,18 @@ var analyzeCommand = new Command("analyze").description("Analyze a sitemap for p
       totalUrls,
       totalRisks,
       urlsWithRisks,
+      ignoredUrls,
       startTime,
       endTime
     };
     const reporters = [new ConsoleReporter()];
     await fs4.mkdir(outDir, { recursive: true });
     if (outputFormat === "json" || outputFormat === "all") {
-      const jsonPath = path2.join(outDir, "sitemap-qa-report.json");
+      const jsonPath = path3.join(outDir, "sitemap-qa-report.json");
       reporters.push(new JsonReporter(jsonPath));
     }
     if (outputFormat === "html" || outputFormat === "all") {
-      const htmlPath = path2.join(outDir, "sitemap-qa-report.html");
+      const htmlPath = path3.join(outDir, "sitemap-qa-report.html");
       reporters.push(new HtmlReporter(htmlPath));
     }
     for (const reporter of reporters) {
@@ -804,11 +750,16 @@ var analyzeCommand = new Command("analyze").description("Analyze a sitemap for p
 // src/commands/init.ts
 import { Command as Command2 } from "commander";
 import fs5 from "fs";
-import path3 from "path";
+import path4 from "path";
 import chalk4 from "chalk";
 var DEFAULT_CONFIG = `# sitemap-qa configuration
 # This file defines the risk categories and patterns to monitor.
 
+# Tool Settings
+outDir: "./sitemap-qa/report"
+outputFormat: "all" # Options: json, html, all
+enforceDomainConsistency: true
+
 # Risk Categories
 # Each category contains a list of patterns to match against URLs found in sitemaps.
 # Patterns can be:
@@ -816,6 +767,16 @@ var DEFAULT_CONFIG = `# sitemap-qa configuration
 # - glob: Glob pattern (e.g., **/admin/**)
 # - regex: Regular expression (e.g., /\\/v[0-9]+\\//)
 
+# Acceptable Patterns
+# URLs matching these patterns will be ignored and not flagged as risks.
+acceptable_patterns:
+  - type: "literal"
+    value: "/acceptable-path"
+    reason: "Example of an acceptable path that should not be flagged."
+  - type: "glob"
+    value: "**/public-docs/**"
+    reason: "Public documentation is always acceptable."
+
 policies:
   - category: "Security & Admin"
     patterns:
@@ -845,7 +806,7 @@ policies:
         reason: "Archive or database backup files exposed."
 `;
 var initCommand = new Command2("init").description("Initialize a default sitemap-qa.yaml configuration file").action(() => {
-  const configPath = path3.join(process.cwd(), "sitemap-qa.yaml");
+  const configPath = path4.join(process.cwd(), "sitemap-qa.yaml");
   if (fs5.existsSync(configPath)) {
     console.error(chalk4.red(`Error: ${configPath} already exists.`));
     process.exit(1);