@rankcli/agent-runtime 0.0.36 → 0.0.38

package/dist/index.js

@@ -55,7 +55,8 @@ async function httpGet(url, config = {}) {
     timeout = DEFAULT_TIMEOUT,
     validateStatus = (status) => status >= 200 && status < 300,
     maxRedirects = 5,
-    params
+    params,
+    maxBytes
   } = config;
   let finalUrl = url;
   if (params) {
@@ -82,7 +83,7 @@ async function httpGet(url, config = {}) {
     if (contentType.includes("application/json")) {
       data = await response.json();
     } else {
-      data = await response.text();
+      data = await readText(response, maxBytes);
     }
     clearTimeout(timeoutId);
     if (!validateStatus(response.status)) {
@@ -109,6 +110,23 @@ async function httpGet(url, config = {}) {
     throw error;
   }
 }
+async function readText(response, maxBytes) {
+  if (!maxBytes || !response.body) return response.text();
+  const reader = response.body.getReader();
+  const chunks = [];
+  let received = 0;
+  while (received < maxBytes) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    received += value.byteLength;
+  }
+  if (received >= maxBytes) await reader.cancel().catch(() => void 0);
+  const decoder = new TextDecoder();
+  let text = "";
+  for (const chunk of chunks) text += decoder.decode(chunk, { stream: true });
+  return text + decoder.decode();
+}
 async function httpHead(url, config = {}) {
   const {
     headers = {},
@@ -9996,7 +10014,9 @@ function analyzeAnchorText(html, url) {
     } catch {
       internalLinks++;
     }
-    const hasAccessibleText = text.length > 0 || ariaLabel || title;
+    const imgAlt = $(el).find("img[alt]").map((_2, img) => ($(img).attr("alt") || "").trim()).get().join(" ").trim();
+    const svgTitle = $(el).find("svg title").text().trim();
+    const hasAccessibleText = text.length > 0 || ariaLabel || title || imgAlt.length > 0 || svgTitle.length > 0;
     if (!hasAccessibleText) {
       emptyAnchors++;
       if (emptyAnchorUrls.length < 5) {
@@ -10401,6 +10421,121 @@ async function checkInternalRedirects(internalLinks, batchSize = 10) {
   return { redirectLinks };
 }
 
+// src/audit/checks/crawler-access.ts
+init_http();
+var CRAWLERS = [
+  {
+    name: "Googlebot",
+    userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    matters: "Google Search indexing",
+    critical: true
+  },
+  {
+    name: "bingbot",
+    userAgent: "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    matters: "Bing indexing, and Copilot answers",
+    critical: true
+  },
+  {
+    name: "GPTBot",
+    userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)",
+    matters: "ChatGPT being able to read and cite the page",
+    critical: false
+  },
+  {
+    name: "ClaudeBot",
+    userAgent: "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+    matters: "Claude being able to read and cite the page",
+    critical: false
+  },
+  {
+    name: "PerplexityBot",
+    userAgent: "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+    matters: "Perplexity being able to read and cite the page",
+    critical: false
+  }
+];
+var BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+function isRefusal(status) {
+  return status === 401 || status === 403 || status === 429 || status === 503;
+}
+async function statusFor(url, userAgent) {
+  try {
+    const response = await httpGet(url, {
+      headers: { "User-Agent": userAgent },
+      timeout: 15e3,
+      // We want the status itself, not an exception on 4xx/5xx.
+      validateStatus: () => true
+    });
+    return response.status;
+  } catch {
+    return null;
+  }
+}
+async function analyzeCrawlerAccess(url) {
+  const issues = [];
+  const baselineStatus = await statusFor(url, BROWSER_UA);
+  const data = { baselineStatus, crawlers: [] };
+  if (baselineStatus === null || baselineStatus >= 400) {
+    return { issues, data };
+  }
+  const results = await Promise.all(
+    CRAWLERS.map(async (crawler) => ({
+      crawler,
+      status: await statusFor(url, crawler.userAgent)
+    }))
+  );
+  const blocked = results.filter(({ status }) => status !== null && isRefusal(status));
+  data.crawlers = results.map(({ crawler, status }) => ({
+    name: crawler.name,
+    status,
+    blocked: status !== null && isRefusal(status)
+  }));
+  if (blocked.length === 0) return { issues, data };
+  const search = blocked.filter(({ crawler }) => crawler.critical);
+  const ai = blocked.filter(({ crawler }) => !crawler.critical);
+  if (search.length > 0) {
+    const names = search.map(({ crawler }) => crawler.name).join(", ");
+    issues.push({
+      code: "SEARCH_CRAWLER_BLOCKED_AT_EDGE",
+      severity: "error",
+      category: "crawlability",
+      title: `${names} refused at the edge, not by robots.txt`,
+      description: `A browser request to this URL returned ${baselineStatus}, but ${names} received ${search.map(({ status }) => status).join(", ")}. Something in front of the origin - a CDN, WAF or bot rule - is turning the crawler away, and robots.txt cannot show you this.`,
+      impact: "Pages a search crawler cannot fetch fall out of the index. This is the failure mode behind Cloudflare\u2019s 2026-09-15 default, where blocking AI training also blocked Googlebot because it is classed as a mixed-use crawler.",
+      howToFix: `Check your CDN or WAF bot rules rather than robots.txt.
+
+On Cloudflare: the zone's Security > Bots settings, and any "Block AI Bots" or AI Crawler Control toggle. Because Googlebot crawls for both Search and AI training, blocking Training applies to all of its functions on ad-supported pages - allow Search crawlers explicitly, or opt out of the default.
+
+Confirm with Search Console's URL Inspection, which fetches as the verified Googlebot.`,
+      affectedUrls: [url],
+      details: {
+        baselineStatus,
+        blocked: search.map(({ crawler, status }) => ({ crawler: crawler.name, status })),
+        note: "Observed with the crawler\u2019s user-agent. Real crawlers are also verified by IP, so a site may treat them differently."
+      }
+    });
+  }
+  if (ai.length > 0) {
+    const names = ai.map(({ crawler }) => crawler.name).join(", ");
+    issues.push({
+      code: "AI_CRAWLER_BLOCKED_AT_EDGE",
+      severity: "warning",
+      category: "ai-readiness",
+      title: `${names} refused at the edge, not by robots.txt`,
+      description: `A browser request returned ${baselineStatus}, but ${names} received ${ai.map(({ status }) => status).join(", ")}. The block is in front of the origin, so allowing these agents in robots.txt will not change it.`,
+      impact: "An assistant that cannot fetch the page cannot quote it. This is invisible to a robots.txt check, which is where most AI-visibility tools stop.",
+      howToFix: `If blocking them is deliberate, nothing to do. If not, the rule is in your CDN or WAF, not robots.txt - on Cloudflare see Security > Bots and any AI Crawler Control setting.`,
+      affectedUrls: [url],
+      details: {
+        baselineStatus,
+        blocked: ai.map(({ crawler, status }) => ({ crawler: crawler.name, status }))
+      }
+    });
+  }
+  return { issues, data };
+}
+
 // src/audit/checks/ai-readiness.ts
 init_http();
 var cheerio13 = __toESM(require("cheerio"));
@@ -14450,6 +14585,107 @@ function analyzeResponsiveImages(html, url) {
 // src/audit/checks/bing-optimization.ts
 var cheerio28 = __toESM(require("cheerio"));
 init_http();
+
+// src/audit/checks/site-verification.ts
+init_http();
+var UNCHECKED = {
+  dnsChecked: false,
+  googleDns: false,
+  bingDns: false,
+  bingSiteAuthFile: false
+};
+var DNS_TIMEOUT_MS = 2e3;
+var FILE_TIMEOUT_MS = 5e3;
+var CACHE_TTL_MS = 10 * 60 * 1e3;
+var cache = /* @__PURE__ */ new Map();
+function lookupSiteVerification(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return Promise.resolve(UNCHECKED);
+  }
+  const host = parsed.hostname.toLowerCase();
+  const now = Date.now();
+  const cached = cache.get(host);
+  if (cached && cached.expires > now) return cached.result;
+  const result = Promise.all([readTxtRecords(host), fetchBingSiteAuth(parsed.origin)]).then(
+    ([txt, bingSiteAuthFile]) => ({
+      dnsChecked: txt.checked,
+      googleDns: txt.records.some((record) => /^google-site-verification=/i.test(record)),
+      bingDns: txt.records.some((record) => /^MS=/i.test(record)),
+      bingSiteAuthFile
+    })
+  );
+  cache.set(host, { expires: now + CACHE_TTL_MS, result });
+  return result;
+}
+var resolveTxtLoader;
+function loadResolveTxt() {
+  resolveTxtLoader ??= (async () => {
+    if (typeof process === "undefined" || !process.versions?.node) return null;
+    try {
+      const specifier = "node:dns";
+      const dns = await import(specifier);
+      const resolveTxt = dns?.promises?.resolveTxt ?? dns?.default?.promises?.resolveTxt;
+      return typeof resolveTxt === "function" ? resolveTxt : null;
+    } catch {
+      return null;
+    }
+  })();
+  return resolveTxtLoader;
+}
+function domainCandidates(host) {
+  if (/^[\d.]+$/.test(host) || host.includes(":")) return [];
+  const labels = host.split(".").filter(Boolean);
+  const names = [];
+  for (let n = 2; n <= labels.length && names.length < 4; n++) {
+    names.push(labels.slice(-n).join("."));
+  }
+  return names.length > 0 ? names : [host];
+}
+async function readTxtRecords(host) {
+  const resolveTxt = await loadResolveTxt();
+  if (!resolveTxt) return { checked: false, records: [] };
+  const lookups = await Promise.all(domainCandidates(host).map((name) => resolveTxtWithin(resolveTxt, name)));
+  return {
+    checked: lookups.every((records) => records !== null),
+    records: lookups.flatMap((records) => records ?? [])
+  };
+}
+var NO_RECORD_CODES = /* @__PURE__ */ new Set(["ENODATA", "ENOTFOUND"]);
+async function resolveTxtWithin(resolveTxt, name) {
+  let timer;
+  const timeout = new Promise((resolve) => {
+    timer = setTimeout(() => resolve(null), DNS_TIMEOUT_MS);
+  });
+  try {
+    return await Promise.race([
+      resolveTxt(name).then(
+        (records) => records.map((chunks) => chunks.join("").trim()),
+        (error) => NO_RECORD_CODES.has(error?.code ?? "") ? [] : null
+      ),
+      timeout
+    ]);
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function fetchBingSiteAuth(origin) {
+  try {
+    const response = await httpGet(`${origin}/BingSiteAuth.xml`, {
+      timeout: FILE_TIMEOUT_MS,
+      maxBytes: 64 * 1024
+    });
+    return response.status === 200 && typeof response.data === "string" && !looksLikeHtmlFallback(response.data, response.headers["content-type"]) && /<users[\s>]/i.test(response.data);
+  } catch {
+    return false;
+  }
+}
+
+// src/audit/checks/bing-optimization.ts
 async function analyzeBingOptimization(html, url) {
   const issues = [];
   const $ = cheerio28.load(html);
@@ -14458,12 +14694,9 @@ async function analyzeBingOptimization(html, url) {
   const detectedIssues = [];
   const bingMetaTag = $('meta[name="msvalidate.01"]').attr("content");
   const hasBingMetaTag = !!bingMetaTag && bingMetaTag.length > 0;
-  let hasBingSiteAuth = false;
-  try {
-    const authResponse = await httpGet(`${baseUrl}/BingSiteAuth.xml`, { timeout: 5e3 });
-    hasBingSiteAuth = authResponse.status === 200 && authResponse.data.includes("<?xml") && authResponse.data.toLowerCase().includes("users");
-  } catch {
-  }
+  const verification = await lookupSiteVerification(url);
+  const hasBingSiteAuth = verification.bingSiteAuthFile;
+  const hasBingDnsRecord = verification.bingDns;
   let hasBingIndexNowKey = false;
   try {
     const keyPatterns = [
@@ -14511,22 +14744,25 @@ async function analyzeBingOptimization(html, url) {
     bingSitemapAccessible = sitemapResponse.status === 200 && sitemapResponse.data.includes("<?xml");
   } catch {
   }
-  if (!hasBingMetaTag && !hasBingSiteAuth) {
+  if (!hasBingMetaTag && !hasBingSiteAuth && !hasBingDnsRecord) {
     detectedIssues.push("No Bing site verification");
     issues.push({
       code: "BING_NOT_VERIFIED",
       severity: "warning",
       category: "ai-readiness",
-      title: "Site not verified with Bing Webmaster Tools",
-      description: "ChatGPT Search uses Bing's index. Verifying your site with Bing Webmaster Tools ensures proper indexing and access to Bing-specific SEO tools.",
+      title: verification.dnsChecked ? "Site not verified with Bing Webmaster Tools" : "Bing Webmaster Tools verification not visible in the HTML",
+      description: verification.dnsChecked ? "No msvalidate.01 meta tag in the HTML, no MS= TXT record on the domain and no /BingSiteAuth.xml file. ChatGPT Search uses Bing's index. Verifying your site with Bing Webmaster Tools ensures proper indexing and access to Bing-specific SEO tools." : "No msvalidate.01 meta tag in the HTML and no /BingSiteAuth.xml file. The site may be verified through a DNS record instead, which this run could not check. ChatGPT Search uses Bing's index. Verifying your site with Bing Webmaster Tools ensures proper indexing and access to Bing-specific SEO tools.",
       impact: "May reduce visibility in ChatGPT Search results and miss Bing-specific optimization opportunities.",
-      howToFix: "Add Bing verification: 1) Go to Bing Webmaster Tools, 2) Add your site, 3) Choose verification method (meta tag or XML file), 4) Add the verification code to your site.",
+      howToFix: "If the site is not yet in Bing Webmaster Tools: 1) Go to Bing Webmaster Tools, 2) Add your site (or import it from Google Search Console), 3) Choose a verification method (meta tag, DNS record or XML file), 4) Add the verification code to your site.",
       affectedUrls: [url],
       details: {
         hasBingMetaTag,
         hasBingSiteAuth,
+        hasBingDnsRecord,
+        dnsChecked: verification.dnsChecked,
         verificationMethods: [
           'Add <meta name="msvalidate.01" content="YOUR_CODE"> to <head>',
+          "Add an MS=YOUR_CODE TXT record to the domain's DNS",
           "Upload BingSiteAuth.xml to root directory"
         ]
       }
@@ -14586,6 +14822,8 @@ async function analyzeBingOptimization(html, url) {
     data: {
       hasBingSiteAuth,
       hasBingMetaTag,
+      hasBingDnsRecord,
+      dnsChecked: verification.dnsChecked,
       hasBingIndexNowKey,
       bingPlacesDetected,
       bingSitemapAccessible,
@@ -16571,6 +16809,10 @@ function analyzePageResources(html, url) {
 // src/audit/checks/responsive-css.ts
 init_http();
 var cheerio35 = __toESM(require("cheerio"));
+var MAX_STYLESHEETS = 3;
+var MAX_STYLESHEET_BYTES = 1024 * 1024;
+var STYLESHEET_TIMEOUT_MS = 3e3;
+var RESPONSIVE_SIGNAL = /@media\b|@container\b|\bclamp\(|(?<![\w-])(?:min|max)\(/i;
 function extractMediaQueries(css) {
   const mediaQueries = [];
   const regex = /@media\s*([^{]+)/g;
@@ -16580,36 +16822,31 @@ function extractMediaQueries(css) {
   }
   return mediaQueries;
 }
+var WIDTH_RULE = /(?:(?:min|max)-width\s*:|width\s*(?:<=|>=|<|>))\s*(\d+(?:\.\d+)?)\s*(px|em|rem)?|(\d+(?:\.\d+)?)\s*(px|em|rem)?\s*(?:<=|>=|<|>)\s*width/gi;
 function classifyBreakpoints(mediaQueries) {
   const breakpoints = /* @__PURE__ */ new Set();
   let hasMobile = false;
   let hasTablet = false;
   let hasDesktop = false;
   for (const query of mediaQueries) {
-    const widthMatches = query.match(/(?:min|max)-width\s*:\s*(\d+)(?:px|em|rem)?/gi);
-    if (widthMatches) {
-      for (const match of widthMatches) {
-        const valueMatch = match.match(/(\d+)/);
-        if (valueMatch) {
-          const value = parseInt(valueMatch[1], 10);
-          breakpoints.add(`${value}px`);
-          if (value <= 480) {
-            hasMobile = true;
-          } else if (value <= 768) {
-            hasMobile = true;
-          } else if (value <= 1024) {
-            hasTablet = true;
-          } else {
-            hasDesktop = true;
-          }
-        }
-      }
-    }
-    if (/screen\s+and/i.test(query)) {
-    }
     if (/print/i.test(query)) {
       continue;
     }
+    for (const match of query.matchAll(WIDTH_RULE)) {
+      const raw = match[1] ?? match[3];
+      const unit = (match[2] ?? match[4] ?? "px").toLowerCase();
+      const value = Math.round(parseFloat(raw) * (unit === "px" ? 1 : 16));
+      breakpoints.add(`${value}px`);
+      if (value <= 480) {
+        hasMobile = true;
+      } else if (value <= 768) {
+        hasMobile = true;
+      } else if (value <= 1024) {
+        hasTablet = true;
+      } else {
+        hasDesktop = true;
+      }
+    }
   }
   return {
     hasMobile,
@@ -16627,6 +16864,7 @@ async function analyzeResponsiveCss(html, url) {
   const allMediaQueries = [];
   const externalSources = [];
   let inlineCount = 0;
+  let hasResponsiveSignals = false;
   $("style").each((_, el) => {
     const css = $(el).text();
     const queries = extractMediaQueries(css);
@@ -16634,39 +16872,43 @@ async function analyzeResponsiveCss(html, url) {
       allMediaQueries.push(...queries);
       inlineCount += queries.length;
     }
+    if (RESPONSIVE_SIGNAL.test(css)) hasResponsiveSignals = true;
   });
-  const cssLinks = $('link[rel="stylesheet"][href]').toArray().slice(0, 3).map((el) => $(el).attr("href")).filter((href) => {
-    if (!href || href.startsWith("data:")) return false;
+  const sameOrigin = [];
+  let crossOrigin = 0;
+  $("link[rel][href]").each((_, el) => {
+    if (!/\bstylesheet\b/i.test($(el).attr("rel") ?? "")) return;
+    const href = $(el).attr("href");
+    if (!href || href.startsWith("data:")) return;
     try {
-      const cssHostname = new URL(href, url).hostname;
-      return cssHostname === baseUrl.hostname;
+      const resolved = new URL(href, url);
+      if (resolved.hostname === baseUrl.hostname) sameOrigin.push(resolved.href);
+      else crossOrigin++;
     } catch {
-      return false;
+      crossOrigin++;
     }
   });
-  const cssPromises = cssLinks.map(async (cssUrl) => {
-    try {
-      const fullUrl = new URL(cssUrl, url).href;
-      const response = await httpGet(fullUrl, {
-        timeout: 3e3,
-        // Reduced timeout
-        validateStatus: () => true
-      });
-      if (response.status === 200 && typeof response.data === "string") {
+  let fetchFailed = 0;
+  await Promise.all(
+    sameOrigin.slice(0, MAX_STYLESHEETS).map(async (cssUrl) => {
+      try {
+        const response = await httpGet(cssUrl, {
+          timeout: STYLESHEET_TIMEOUT_MS,
+          maxBytes: MAX_STYLESHEET_BYTES
+        });
+        if (typeof response.data !== "string") throw new Error("not text");
         const queries = extractMediaQueries(response.data);
-        if (queries.length > 0) {
-          allMediaQueries.push(...queries);
+        if (queries.length > 0) allMediaQueries.push(...queries);
+        if (RESPONSIVE_SIGNAL.test(response.data)) {
+          hasResponsiveSignals = true;
           externalSources.push(cssUrl);
         }
+      } catch {
+        fetchFailed++;
       }
-    } catch {
-    }
-  });
-  await Promise.race([
-    Promise.all(cssPromises),
-    new Promise((resolve) => setTimeout(resolve, 6e3))
-    // 6s overall timeout
-  ]);
+    })
+  );
+  const stylesheetsCovered = crossOrigin === 0 && fetchFailed === 0 && sameOrigin.length <= MAX_STYLESHEETS;
   const hasMediaQueries = allMediaQueries.length > 0;
   const { hasMobile, hasTablet, hasDesktop, breakpoints } = classifyBreakpoints(allMediaQueries);
   if (!hasViewport) {
@@ -16681,21 +16923,22 @@ async function analyzeResponsiveCss(html, url) {
       affectedUrls: [url]
     });
   }
-  if (!hasMediaQueries) {
+  if (!hasResponsiveSignals && stylesheetsCovered) {
     issues.push({
       code: "RESPONSIVE_NO_MEDIA_QUERIES",
       severity: "warning",
       category: "mobile",
       title: "No CSS media queries detected",
-      description: "No @media queries found in inline styles or first-party stylesheets.",
+      description: "No @media or @container queries and no fluid sizing (clamp, min, max) found in inline styles or first-party stylesheets.",
       impact: "Without media queries, the site likely uses fixed layouts that do not adapt to different screen sizes. This hurts mobile usability and SEO.",
       howToFix: "Add CSS media queries to adapt layout for different screen sizes. Consider using a mobile-first approach with min-width breakpoints.",
       affectedUrls: [url],
       details: {
-        suggestion: "Common breakpoints: 480px (mobile), 768px (tablet), 1024px (desktop)"
+        suggestion: "Common breakpoints: 480px (mobile), 768px (tablet), 1024px (desktop)",
+        stylesheetsScanned: sameOrigin.length
       }
     });
-  } else if (!hasMobile && !hasTablet) {
+  } else if (hasMediaQueries && !hasMobile && !hasTablet) {
     issues.push({
       code: "RESPONSIVE_NO_MOBILE_BREAKPOINT",
       severity: "warning",
@@ -16721,6 +16964,8 @@ async function analyzeResponsiveCss(html, url) {
         inline: inlineCount,
         external: externalSources
       },
+      hasResponsiveSignals,
+      stylesheetsCovered,
       breakpoints,
       hasMobileBreakpoint: hasMobile,
       hasTabletBreakpoint: hasTablet,
@@ -17359,7 +17604,22 @@ var urlSafetyDatabase = {
 
 // src/audit/checks/tracking-verification.ts
 var cheerio36 = __toESM(require("cheerio"));
-function analyzeTrackingVerification(html, url) {
+var OTHER_ANALYTICS = [
+  { name: "Cloudflare Web Analytics", pattern: /cloudflareinsights\.com\/beacon(?:\.min)?\.js/i },
+  { name: "Plausible", pattern: /plausible\.io\/js\/|<script[^>]*data-domain=[^>]*plausible|<script[^>]*plausible[^>]*data-domain=/i },
+  { name: "Fathom", pattern: /cdn\.usefathom\.com/i },
+  // data-website-id is Umami's attribute, on the hosted script and on self-hosted ones alike.
+  { name: "Umami", pattern: /<script[^>]*\bdata-website-id=/i },
+  { name: "PostHog", pattern: /posthog\.init\s*\(|\.posthog\.com\//i },
+  { name: "Matomo", pattern: /matomo\.js|piwik\.js|_paq\s*(?:=|\.push)/i },
+  { name: "Simple Analytics", pattern: /scripts\.simpleanalyticscdn\.com/i },
+  { name: "Mixpanel", pattern: /cdn\.mxpnl\.com|mixpanel\.init\s*\(/i },
+  { name: "Segment", pattern: /cdn\.segment\.com/i }
+];
+function detectOtherAnalytics(html) {
+  return OTHER_ANALYTICS.filter(({ pattern }) => pattern.test(html)).map(({ name }) => name);
+}
+async function analyzeTrackingVerification(html, url) {
   const issues = [];
   const $ = cheerio36.load(html);
   const ga4Match = html.match(/gtag\s*\(\s*['"]config['"]\s*,\s*['"](G-[A-Z0-9]+)['"]/i) || html.match(/googletagmanager\.com\/gtag\/js\?id=(G-[A-Z0-9]+)/i);
@@ -17369,10 +17629,16 @@ function analyzeTrackingVerification(html, url) {
   const hasGtm = !!gtmMatch;
   const gtmContainerId = gtmMatch ? gtmMatch[1] : void 0;
   const hasGtmNoscript = /googletagmanager\.com\/ns\.html\?id=GTM-[A-Z0-9]+/i.test(html);
+  const otherAnalytics = detectOtherAnalytics(html);
+  const verification = await lookupSiteVerification(url);
   const gscMeta = $('meta[name="google-site-verification"]').attr("content");
-  const hasGscVerification = !!(gscMeta && gscMeta.length > 10);
+  const hasGscMeta = !!(gscMeta && gscMeta.length > 10);
+  const hasGscVerification = hasGscMeta || verification.googleDns;
+  const gscSource = hasGscMeta ? "meta" : verification.googleDns ? "dns" : void 0;
   const bingMeta = $('meta[name="msvalidate.01"]').attr("content");
-  const hasBingVerification = !!(bingMeta && bingMeta.length > 10);
+  const hasBingMeta = !!(bingMeta && bingMeta.length > 10);
+  const hasBingVerification = hasBingMeta || verification.bingDns || verification.bingSiteAuthFile;
+  const bingSource = hasBingMeta ? "meta" : verification.bingDns ? "dns" : verification.bingSiteAuthFile ? "file" : void 0;
   const jsonLdScripts = $('script[type="application/ld+json"]');
   const hasMicrodata = $('[itemtype*="schema.org"]').length > 0;
   const hasJsonLd = jsonLdScripts.length > 0;
@@ -17394,13 +17660,15 @@ function analyzeTrackingVerification(html, url) {
   const data = {
     ga4: { detected: hasGa4, measurementId: ga4MeasurementId },
     gtm: { detected: hasGtm, containerId: gtmContainerId, hasNoscript: hasGtmNoscript },
-    gsc: { verified: hasGscVerification, verificationCode: gscMeta },
-    bing: { verified: hasBingVerification, verificationCode: bingMeta },
+    otherAnalytics,
+    gsc: { verified: hasGscVerification, verificationCode: gscMeta, source: gscSource },
+    bing: { verified: hasBingVerification, verificationCode: bingMeta, source: bingSource },
+    dnsChecked: verification.dnsChecked,
     schemaOrg: { detected: hasJsonLd || hasMicrodata, types: schemaTypes, hasValidJson },
     sitemap: { inRobotsTxt: false }
     // Set later if we check robots.txt
   };
-  if (!hasGa4 && !hasGtm) {
+  if (!hasGa4 && !hasGtm && otherAnalytics.length === 0) {
     issues.push({
       code: "GA4_MISSING",
       severity: "warning",
@@ -17413,40 +17681,46 @@ function analyzeTrackingVerification(html, url) {
     });
   }
   if (!hasGscVerification) {
+    const dnsChecked = verification.dnsChecked;
     issues.push({
       code: "GSC_VERIFICATION_MISSING",
       severity: "warning",
       category: "on-page",
-      title: "Google Search Console verification not found",
-      description: "No GSC verification meta tag detected. GSC provides essential data about your search performance.",
+      title: dnsChecked ? "Google Search Console verification not found" : "Google Search Console verification not visible in the HTML",
+      description: dnsChecked ? "No google-site-verification meta tag in the HTML and no google-site-verification TXT record on the domain. GSC provides essential data about your search performance." : "No google-site-verification meta tag in the HTML. The site may be verified through a DNS TXT record instead, which this run could not check. GSC provides essential data about your search performance.",
       impact: "Missing critical insights into Google search visibility, impressions, and clicks.",
-      howToFix: "Add your GSC verification meta tag to the <head> section. Get it from search.google.com/search-console.",
-      affectedUrls: [url]
+      howToFix: dnsChecked ? "Verify the site in Google Search Console: add the meta tag to <head>, or a TXT record to the domain's DNS. Get it from search.google.com/search-console." : "If the site is not yet verified in Google Search Console, add the meta tag to <head> or a TXT record to the domain's DNS. Get it from search.google.com/search-console.",
+      affectedUrls: [url],
+      details: { dnsChecked }
     });
   }
   if (!hasBingVerification) {
+    const dnsChecked = verification.dnsChecked;
     issues.push({
       code: "BING_VERIFICATION_MISSING",
       severity: "notice",
       category: "on-page",
-      title: "Bing Webmaster Tools verification not found",
-      description: "No Bing verification meta tag detected. Bing/Yahoo account for 5-8% of search traffic.",
+      title: dnsChecked ? "Bing Webmaster Tools verification not found" : "Bing Webmaster Tools verification not visible in the HTML",
+      description: dnsChecked ? "No msvalidate.01 meta tag in the HTML, no MS= TXT record on the domain and no /BingSiteAuth.xml file. Bing/Yahoo account for 5-8% of search traffic." : "No msvalidate.01 meta tag in the HTML and no /BingSiteAuth.xml file. The site may be verified through a DNS record instead, which this run could not check. Bing/Yahoo account for 5-8% of search traffic.",
       impact: "Missing insights on Bing/Yahoo search traffic.",
-      howToFix: "Add Bing Webmaster Tools verification for insights on Bing/Yahoo traffic. Get it from bing.com/webmasters.",
-      affectedUrls: [url]
+      howToFix: "If the site is not yet in Bing Webmaster Tools, add it there: import it from Google Search Console, or verify with the meta tag, a DNS record or BingSiteAuth.xml. Get it from bing.com/webmasters.",
+      affectedUrls: [url],
+      details: { dnsChecked }
     });
   }
   if (!hasGtm) {
-    issues.push({
-      code: "GTM_MISSING",
-      severity: "notice",
-      category: "on-page",
-      title: "Google Tag Manager not detected",
-      description: "GTM not found. While not required, GTM makes it easier to manage tracking tags without code changes.",
-      impact: "Tag management requires developer involvement for each change.",
-      howToFix: "Consider using GTM for easier tag management. It allows marketing to add tags without developer help.",
-      affectedUrls: [url]
-    });
+    if (otherAnalytics.length === 0) {
+      issues.push({
+        code: "GTM_MISSING",
+        severity: "notice",
+        category: "on-page",
+        title: "Google Tag Manager not detected",
+        description: "GTM not found. While not required, GTM makes it easier to manage tracking tags without code changes.",
+        impact: "Tag management requires developer involvement for each change.",
+        howToFix: "Consider using GTM for easier tag management. It allows marketing to add tags without developer help.",
+        affectedUrls: [url]
+      });
+    }
   } else if (!hasGtmNoscript) {
     issues.push({
       code: "GTM_NOSCRIPT_MISSING",
@@ -17586,7 +17860,6 @@ async function runFullAudit(options) {
   const urlSafetyResult = analyzeUrlSafety(url);
   const domSizeResult = analyzeDomSize(html, url);
   const imageDimensionsResult = analyzeImageDimensions(html, url);
-  const trackingResult = analyzeTrackingVerification(html, url);
   const anchorResult = runExtendedChecks ? analyzeAnchorText(html, url) : { issues: [], data: {} };
   const localSeoResult = runExtendedChecks ? analyzeLocalSEO(html, url) : { issues: [], data: {} };
   const modernImageResult = runExtendedChecks ? analyzeModernImages(html, url) : { issues: [], data: {} };
@@ -17661,8 +17934,7 @@ async function runFullAudit(options) {
     ...colorContrastResult.issues,
     ...pageResourcesResult.issues,
     ...urlSafetyResult.issues,
-    ...paginationResult.issues,
-    ...trackingResult.issues
+    ...paginationResult.issues
   );
   if (!structuredDataResult.data.hasSchema) {
     const suggestions = suggestSchemaTypes(html, url);
@@ -17708,6 +17980,8 @@ async function runFullAudit(options) {
   asyncChecks.push(safeAsyncIssues("Certificate", async () => (await checkCertificate(url)).issues));
   asyncChecks.push(safeAsync("Redirects", () => analyzeRedirects(url)));
   asyncChecks.push(safeAsync("Robots Validation", () => validateRobotsTxt(url)));
+  asyncChecks.push(safeAsync("Tracking Verification", () => analyzeTrackingVerification(html, url)));
+  asyncChecks.push(safeAsyncIssues("Crawler Access", async () => (await analyzeCrawlerAccess(url)).issues));
   asyncChecks.push(safeAsync("Canonical Domain", () => analyzeCanonicalDomain(url)));
   asyncChecks.push(safeAsyncIssues("Framework Detection", async () => {
     const frameworkInfo = detectFramework({ html, headers });
@@ -33095,9 +33369,37 @@ function jsonLdConcerns(html) {
   }
   return bad;
 }
+var PLACEHOLDERS = [
+  { pattern: /\+?1[-\s.]?\(?800\)?[-\s.]?555[-\s.]?\d{4}/i, what: "a reserved 555 phone number" },
+  { pattern: /\b555[-\s.]?01\d{2}\b/, what: "a reserved 555 phone number" },
+  { pattern: /\bAnytown\b/i, what: '"Anytown"' },
+  { pattern: /\b123 Main (Street|St)\b/i, what: '"123 Main Street"' },
+  { pattern: /your_[a-z_]+/i, what: 'an unfilled "your_..." placeholder' },
+  { pattern: /\b(yourdomain|your-domain|example)\.(com|org|net)\b/i, what: "a placeholder domain" },
+  { pattern: /\blorem ipsum\b/i, what: "lorem ipsum" },
+  { pattern: /\bXXX-XXX-XXXX\b/i, what: "an unfilled phone template" }
+];
+function isSharedShell(html) {
+  return /<div[^>]+id=["'](root|app|__next|__nuxt)["']/i.test(html);
+}
 function assessReplacement(path3, original, replacement) {
   const concerns = [];
   const add = (code, detail) => concerns.push({ code, detail });
+  for (const { pattern, what } of PLACEHOLDERS) {
+    const hit = replacement.match(pattern);
+    if (hit && !(original && pattern.test(original))) {
+      add("placeholder_content", `contains ${what} (${hit[0]})`);
+    }
+  }
+  if (/\.html?$/i.test(path3) && isSharedShell(replacement)) {
+    const addsCanonical = /<link[^>]+rel=["']canonical["']/i.test(replacement) && !(original && /<link[^>]+rel=["']canonical["']/i.test(original));
+    if (addsCanonical) {
+      add(
+        "canonical_in_shared_shell",
+        `${path3} mounts a single-page app, so a canonical here applies to every route`
+      );
+    }
+  }
   if (/\.json$/i.test(path3) || /\.webmanifest$/i.test(path3)) {
     try {
       JSON.parse(replacement);