@akotliar/sitemap-qa 1.0.0-alpha.1 → 1.0.0-alpha.3

package/dist/index.js

@@ -289,31 +289,6 @@ async function fetchUrl(url, options = {}) {
 }
 
 // src/core/discovery.ts
-function normalizeBaseUrl(url) {
-  const parsed = new URL(url);
-  return parsed.origin;
-}
-async function detectCanonicalDomain(baseUrl, config) {
-  const urlObj = new URL(baseUrl);
-  const hasWww = urlObj.hostname.startsWith("www.");
-  const alternateHostname = hasWww ? urlObj.hostname.substring(4) : `www.${urlObj.hostname}`;
-  const alternateUrl = `${urlObj.protocol}//${alternateHostname}/robots.txt`;
-  try {
-    const result = await fetchUrl(alternateUrl, {
-      timeout: config.timeout,
-      maxRetries: 1
-    });
-    if (result.statusCode === 200 || result.statusCode === 404) {
-      return alternateHostname;
-    }
-    return urlObj.hostname;
-  } catch (error) {
-    if (error instanceof HttpError && error.statusCode === 301) {
-      return urlObj.hostname;
-    }
-    return urlObj.hostname;
-  }
-}
 async function tryStandardPaths(baseUrl, config) {
   const baseDomain = new URL(baseUrl).origin;
   const accessIssues = [];
@@ -457,13 +432,11 @@ function extractSitemapIndexUrls(xmlContent) {
   }
   return urls;
 }
-async function discoverAllSitemaps(initialSitemaps, config, baseUrl, canonicalDomain, _maxDepth = 10) {
+async function discoverAllSitemaps(initialSitemaps, config) {
   const finalSitemaps = [];
   const toProcess = [...initialSitemaps];
   const processed = /* @__PURE__ */ new Set();
-  const failed = /* @__PURE__ */ new Set();
-  const redirected = /* @__PURE__ */ new Set();
-  let detectedCanonical = canonicalDomain;
+  const inaccessible = /* @__PURE__ */ new Set();
   const BATCH_SIZE = config.discoveryConcurrency || 50;
   while (toProcess.length > 0) {
     const batch = toProcess.splice(0, Math.min(BATCH_SIZE, toProcess.length));
@@ -496,40 +469,12 @@ async function discoverAllSitemaps(initialSitemaps, config, baseUrl, canonicalDo
           return { type: "sitemap", url: sitemapUrl };
         }
       } catch (error) {
-        if (error instanceof HttpError && error.statusCode === 301) {
-          redirected.add(sitemapUrl);
-          if (config.verbose) {
-            if (!detectedCanonical) {
-              detectedCanonical = await detectCanonicalDomain(baseUrl, config);
-              if (config.verbose) {
-                console.log(`Canonical domain detected: ${detectedCanonical}`);
-              }
-            }
-            try {
-              const sitemapUrlObj = new URL(sitemapUrl);
-              if (sitemapUrlObj.hostname !== detectedCanonical) {
-                console.warn(`\u26A0\uFE0F  Sitemap URL redirects (301): ${sitemapUrl}`);
-                console.warn(`   Problem: The sitemap index contains a URL that redirects.`);
-                console.warn(`   Likely issue: Domain mismatch - expected "${detectedCanonical}" but got "${sitemapUrlObj.hostname}"`);
-                console.warn(`   Fix: Update sitemap index to use "https://${detectedCanonical}${sitemapUrlObj.pathname}"`);
-              } else {
-                console.warn(`\u26A0\uFE0F  Sitemap URL redirects (301): ${sitemapUrl}`);
-                console.warn(`   Fix: Update the sitemap index to reference the final URL after redirect.`);
-              }
-            } catch {
-              const message = error instanceof Error ? error.message : String(error);
-              console.warn(`Failed to fetch sitemap ${sitemapUrl}: ${message}`);
-            }
-          }
-          return { type: "redirect" };
-        } else {
-          failed.add(sitemapUrl);
-          if (config.verbose) {
-            const message = error instanceof Error ? error.message : String(error);
-            console.warn(`Failed to fetch sitemap ${sitemapUrl}: ${message}`);
-          }
-          return { type: "failed" };
+        inaccessible.add(sitemapUrl);
+        if (config.verbose) {
+          const message = error instanceof Error ? error.message : String(error);
+          console.warn(`Failed to fetch sitemap ${sitemapUrl}: ${message}`);
         }
+        return { type: "failed" };
       }
     }));
     for (const result of batchResults) {
@@ -544,117 +489,50 @@ async function discoverAllSitemaps(initialSitemaps, config, baseUrl, canonicalDo
       break;
     }
   }
-  const totalProcessed = processed.size;
-  const totalFailed = failed.size;
-  const totalRedirected = redirected.size;
-  const sitemapIndexCount = totalProcessed - finalSitemaps.length - totalFailed - totalRedirected;
-  if (finalSitemaps.length === 0 && totalProcessed > 0) {
+  if (finalSitemaps.length === 0 && inaccessible.size > 0) {
     console.warn(`
-\u26A0\uFE0F  SITEMAP DISCOVERY ISSUE`);
-    if (sitemapIndexCount > 0 && (totalFailed > 0 || totalRedirected > 0)) {
-      console.warn(`Found ${sitemapIndexCount} sitemap index(es) containing ${totalFailed + totalRedirected} child sitemap(s):`);
-      if (totalRedirected > 0) {
-        console.warn(`  - ${totalRedirected} sitemap(s) return 301 redirects (content not accessible without following redirect)`);
-      }
-      if (totalFailed > 0) {
-        console.warn(`  - ${totalFailed} sitemap(s) returned errors (404, 403, 500, or network issues)`);
-      }
-    } else if (totalRedirected > 0) {
-      console.warn(`All ${totalRedirected} sitemap(s) return 301 redirects.`);
-    } else if (totalFailed > 0) {
-      console.warn(`All ${totalFailed} sitemap(s) returned errors.`);
-      console.warn(`
-Common causes:`);
-      console.warn(`  - 403 Forbidden: Bot protection (Cloudflare, etc.) or IP blocking`);
-      console.warn(`  - 404 Not Found: Sitemaps don't exist at these URLs`);
-      console.warn(`  - 500/502/503: Server errors or maintenance`);
-      console.warn(`
-If sitemaps work in your browser but not here, the site likely has bot protection.`);
-      console.warn(`Try: Check if sitemaps load without JavaScript, or contact site administrator.`);
-    } else {
-      console.warn(`Processed ${totalProcessed} URL(s) but found no accessible sitemaps.`);
-    }
-    console.warn(`
-Note: This tool does not follow redirects for sitemap URLs.`);
-    if (totalRedirected > 0) {
-      console.warn(`
-Possible causes of redirects:`);
-      console.warn(`  - Sitemap index uses non-canonical domain (e.g., missing 'www' or vice versa)`);
-      console.warn(`  - Sitemap URLs redirect from HTTP to HTTPS`);
-      console.warn(`  - Intentional redirects in your site configuration`);
-      console.warn(`
-Recommendation: Update sitemap index URLs to match the final destination (no redirects).`);
-    }
-    console.warn(``);
-  }
-  return { sitemaps: finalSitemaps, canonicalDomain: detectedCanonical };
+\u26A0\uFE0F  All ${inaccessible.size} sitemap(s) were inaccessible`);
+    console.warn(`Common causes: 403/404 errors, network issues, or bot protection`);
+  }
+  return finalSitemaps;
 }
 async function discoverSitemaps(baseUrl, config) {
-  const normalizedUrl = normalizeBaseUrl(baseUrl);
-  let allAccessIssues = [];
-  let canonicalDomain;
+  const normalizedUrl = new URL(baseUrl).origin;
   if (config.verbose) {
-    console.log("Strategy 1: Checking robots.txt for sitemap directives...");
+    console.log("Checking robots.txt for sitemap directives...");
   }
   const robotsSitemaps = await parseRobotsTxt(normalizedUrl, config);
   if (robotsSitemaps.length > 0) {
-    const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(robotsSitemaps, config, normalizedUrl, canonicalDomain);
-    canonicalDomain = detected;
+    const sitemaps = await discoverAllSitemaps(robotsSitemaps, config);
     return {
-      sitemaps: allSitemaps,
+      sitemaps,
       source: "robots-txt",
-      accessIssues: [],
-      // Clear access issues since we found working sitemaps
-      canonicalDomain
+      accessIssues: []
     };
   }
   if (config.verbose) {
-    console.log("Strategy 2: Trying standard sitemap paths...");
+    console.log("Trying standard sitemap paths...");
   }
-  const { sitemaps: standardSitemaps, issues, redirectedToCanonical } = await tryStandardPaths(normalizedUrl, config);
-  allAccessIssues = issues;
+  const { sitemaps: standardSitemaps, issues } = await tryStandardPaths(normalizedUrl, config);
   if (standardSitemaps.length > 0) {
-    const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(standardSitemaps, config, normalizedUrl, canonicalDomain);
-    canonicalDomain = detected;
-    return {
-      sitemaps: allSitemaps,
-      source: "standard-path",
-      accessIssues: [],
-      // Clear access issues since we found working sitemaps
-      canonicalDomain
-    };
-  }
-  if (redirectedToCanonical) {
-    const canonicalUrl = `https://${redirectedToCanonical}`;
-    console.log(`
-\u{1F4A1} All requests redirected. Retrying with canonical domain: ${redirectedToCanonical}
-`);
-    const canonicalRobotsSitemaps = await parseRobotsTxt(canonicalUrl, config);
-    if (canonicalRobotsSitemaps.length > 0) {
-      const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(canonicalRobotsSitemaps, config, canonicalUrl, redirectedToCanonical);
+    const sitemaps = await discoverAllSitemaps(standardSitemaps, config);
+    if (sitemaps.length > 0) {
       return {
-        sitemaps: allSitemaps,
-        source: "robots-txt",
-        accessIssues: [],
-        canonicalDomain: detected || redirectedToCanonical
-      };
-    }
-    const { sitemaps: canonicalStandardSitemaps } = await tryStandardPaths(canonicalUrl, config);
-    if (canonicalStandardSitemaps.length > 0) {
-      const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(canonicalStandardSitemaps, config, canonicalUrl, redirectedToCanonical);
-      return {
-        sitemaps: allSitemaps,
+        sitemaps,
         source: "standard-path",
-        accessIssues: [],
-        canonicalDomain: detected || redirectedToCanonical
+        accessIssues: []
       };
     }
+    return {
+      sitemaps: [],
+      source: "standard-path",
+      accessIssues: issues
+    };
   }
   return {
     sitemaps: [],
     source: "none",
-    accessIssues: allAccessIssues,
-    canonicalDomain
+    accessIssues: issues
   };
 }
 
@@ -976,9 +854,6 @@ Top duplicates:`);
 
 // src/core/patterns/risk-patterns.ts
 var RISK_PATTERNS = [
-  // Note: Environment leakage patterns moved to domain-patterns.ts
-  // Note: Admin path patterns moved to admin-patterns.ts
-  // to avoid duplication and improve maintainability
   // Sensitive Parameter Patterns (HIGH)
   {
     name: "Authentication Parameter",
@@ -1532,7 +1407,7 @@ function summarizeRisks(request) {
 }
 
 // src/reporters/json-reporter.ts
-var TOOL_VERSION = true ? "1.0.0-alpha.1" : "dev";
+var TOOL_VERSION = true ? "1.0.0-alpha.2" : "dev";
 function generateJsonReport(summary, discoveryResult, parseResult, riskGroups, config, startTime, options = {}) {
   const {
     pretty = true,
@@ -1702,7 +1577,7 @@ function transformError(error) {
 
 // src/reporters/html-reporter.ts
 import { promises as fs } from "fs";
-var TOOL_VERSION2 = "1.0.0-alpha.1";
+var TOOL_VERSION2 = "1.0.0-alpha.2";
 function generateHtmlReport(summary, discoveryResult, totalUrls, config, errors, options = {}) {
   const maxUrls = options.maxUrlsPerGroup ?? 10;
   const timestamp = (/* @__PURE__ */ new Date()).toISOString();
@@ -2257,7 +2132,7 @@ function showCliSummary(result) {
   const riskyUrlCount = result.summary.categoryInsights.reduce((sum, g) => sum + g.count, 0);
   console.log(chalk.dim("\u2500".repeat(50)));
   if (riskyUrlCount === 0) {
-    console.log(chalk.green("\u2705 No issues found - sitemap looks clean!"));
+    console.log(chalk.green("No issues found - sitemap looks clean!"));
   } else {
     const { high, medium, low } = result.summary.severityBreakdown;
     const severityParts = [];
@@ -2350,9 +2225,9 @@ async function runAnalysisPipeline(url, config) {
   const duplicatePercentage = duplicatesRemoved / extractionResult.allUrls.length * 100;
   if (!config.silent) {
     if (duplicatesRemoved > 100 || duplicatePercentage > 1) {
-      console.log(chalk.green(`\u2713 Analyzed ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs (${consolidatedResult.uniqueUrls.length.toLocaleString()} unique)`));
+      console.log(chalk.green(`Found ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs (${consolidatedResult.uniqueUrls.length.toLocaleString()} unique)`));
     } else {
-      console.log(chalk.green(`\u2713 Analyzed ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs`));
+      console.log(chalk.green(`Found ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs`));
     }
   }
   phaseStart = Date.now();
@@ -2383,7 +2258,7 @@ async function runAnalysisPipeline(url, config) {
   } else if (!config.silent) {
     const parsingPhase = phaseTimings.find((p) => p.name === "Parsing");
     const sitemapsPerSec = parsingPhase ? (discoveryResult.sitemaps.length / (parsingPhase.duration / 1e3)).toFixed(1) : "0";
-    console.log(chalk.green(`\u2705 Analysis complete (${(executionTime / 1e3).toFixed(1)}s \xB7 ${sitemapsPerSec} sitemaps/sec)
+    console.log(chalk.green(`Analysis complete (${(executionTime / 1e3).toFixed(1)}s \xB7 ${sitemapsPerSec} sitemaps/sec)
 `));
   }
   if (config.benchmark) {
@@ -2432,7 +2307,7 @@ function handleAnalysisError(error, config) {
 }
 function displayPhaseSummary(timings, totalTime) {
   console.log(chalk.green(`
-\u2705 Analysis Complete (Total: ${(totalTime / 1e3).toFixed(1)}s)
+Analysis Complete (Total: ${(totalTime / 1e3).toFixed(1)}s)
 `));
   console.log(chalk.cyan("Phase Breakdown:"));
   for (const timing of timings) {