@nebulacomponents/citable 1.15.1 → 1.17.0

package/src/detectors/tech.js

@@ -1,3 +1,4 @@
+import { parse } from 'node-html-parser';
 import { defineDetector, hasHtmlDocumentMarkup, indexTargets, isHtmlDocument, isHtmlDocumentUrl, registryPageFor, pageSubject, safePath } from './framework.js';
 import { isAllowed } from '../crawler/robots.js';
 
@@ -358,6 +359,311 @@ D.push(defineDetector({
   },
 }));
 
+D.push(defineDetector({
+  id: 'TECH-019', name: 'Open Graph URL conflicts with canonical', namespace: 'TECH',
+  description: 'Open Graph URL (og:url) disagrees with the declared canonical URL, sending conflicting canonicalization and entity identity signals.',
+  discipline: ['seo', 'aeo', 'geo'], severity: 'medium', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'low', ranking: 'medium', citation: 'medium' },
+  applicable_requirement: 'SEO §2 canonical signals should agree across Open Graph and HTML declarations; AEO §2 consistent identity signals',
+  remediation: 'Align og:url with the rel=canonical target on the index-target document, or omit og:url if self-canonicalizing.',
+  verification: 'Compare og:url and rel=canonical in rendered HTML.',
+  check(ctx) {
+    const hits = [];
+    for (const p of ctx.site.pages) {
+      if (p.canonicals.length !== 1 || !p.ogUrl) continue;
+      const declared = p.canonicals[0];
+      let normCanonical;
+      let normOg;
+      try {
+        normCanonical = ctx.site.normalize(new URL(declared, p.url).href);
+        normOg = ctx.site.normalize(new URL(p.ogUrl, p.url).href);
+      } catch {
+        continue;
+      }
+      if (normCanonical !== normOg) {
+        hits.push({
+          subject: pageSubject(p),
+          summary: `Open Graph URL (${p.ogUrl}) disagrees with rel=canonical (${declared})`,
+          evidence: [`rel=canonical: ${declared}`, `og:url: ${p.ogUrl}`],
+          captured: { canonical: declared, ogUrl: p.ogUrl },
+          expected: 'matching canonical and og:url',
+        });
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'TECH-020', name: 'Missing reciprocal hreflang return link', namespace: 'TECH',
+  description: 'An alternate hreflang URL targets an internal page, but that target page does not reciprocate with a matching hreflang link back to the source page.',
+  discipline: ['seo', 'geo'], severity: 'high', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'high', ranking: 'medium', citation: 'medium' },
+  applicable_requirement: 'SEO §2 internationalization controls; RFC 8288 and search engine hreflang reciprocity requirement',
+  remediation: 'Add a reciprocal <link rel="alternate" hreflang="..." href="..."> tag on the target page pointing back to the referencing page.',
+  verification: 'Extract hreflang alternate links on both URLs and confirm bidirectional reciprocity.',
+  check(ctx) {
+    const hits = [];
+    for (const p of ctx.site.pages) {
+      if (!p.hreflangs || p.hreflangs.length === 0) continue;
+      const normSource = ctx.site.normalize(p.url);
+      for (const alt of p.hreflangs) {
+        if (!alt.href || alt.lang === 'x-default') continue;
+        let normTarget;
+        try {
+          normTarget = ctx.site.normalize(new URL(alt.href, p.url).href);
+        } catch {
+          continue;
+        }
+        if (normTarget === normSource) continue;
+        const targetPage = ctx.site.byUrl.get(normTarget);
+        if (!targetPage) continue;
+        const hasReciprocal = (targetPage.hreflangs || []).some((tAlt) => {
+          try {
+            const normBack = ctx.site.normalize(new URL(tAlt.href, targetPage.url).href);
+            return normBack === normSource;
+          } catch {
+            return false;
+          }
+        });
+        if (!hasReciprocal) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Missing reciprocal hreflang link from ${alt.href} back to ${p.url}`,
+            evidence: [
+              `${p.url} references alternate ${alt.href} (hreflang="${alt.lang}")`,
+              `${normTarget} does not link back to ${p.url} in its hreflang annotations`,
+            ],
+            captured: { target: normTarget, hreflang: alt.lang },
+            expected: `reciprocal link to ${p.url}`,
+          });
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'TECH-021', name: 'Invalid or non-standard BCP 47 hreflang syntax', namespace: 'TECH',
+  description: 'An hreflang annotation uses an invalid language code, malformed region subtag, deprecated territory, or relative URL.',
+  discipline: ['seo', 'geo'], severity: 'high', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'high', ranking: 'medium', citation: 'medium' },
+  applicable_requirement: 'SEO §2 BCP 47 language/region standards (ISO 639-1 language, ISO 3166-1 alpha-2 region)',
+  remediation: 'Use lowercase two-letter ISO 639-1 language codes, optional uppercase ISO 3166-1 region codes (e.g. en-US, es-ES, de-DE), or x-default with absolute URLs.',
+  verification: 'Validate all hreflang attribute values against BCP 47 and confirm absolute URLs.',
+  check(ctx) {
+    const hits = [];
+    const BCP47_REGEX = /^(?:x-default|[a-z]{2,3}(?:-[A-Z][a-z]{3})?(?:-(?:[A-Z]{2}|[0-9]{3}))?)$/;
+    for (const p of ctx.site.pages) {
+      if (!p.hreflangs || p.hreflangs.length === 0) continue;
+      for (const alt of p.hreflangs) {
+        const code = alt.lang;
+        const href = alt.href;
+        const issues = [];
+        if (!code) {
+          issues.push('hreflang attribute is empty');
+        } else {
+          if (code.toLowerCase() === 'x-default') {
+            if (code !== 'x-default') issues.push(`x-default must be lowercase (got "${code}")`);
+          } else if (code.includes('_')) {
+            issues.push(`hreflang uses underscore instead of hyphen ("${code}")`);
+          } else if (/\ben-uk\b/i.test(code)) {
+            issues.push(`"UK" is not an ISO 3166-1 alpha-2 country code; use "GB" (got "${code}")`);
+          } else if (!BCP47_REGEX.test(code)) {
+            issues.push(`malformed BCP 47 code format ("${code}")`);
+          }
+        }
+        if (href && !/^https?:\/\//i.test(href)) {
+          issues.push(`hreflang target must be an absolute URL with scheme (got "${href}")`);
+        }
+        if (issues.length > 0) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Invalid hreflang annotation: ${issues.join('; ')}`,
+            evidence: issues.map((iss) => `${p.url}: ${iss}`),
+            captured: { hreflang: code, href },
+            expected: 'valid BCP 47 language/region code and absolute URL',
+          });
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'TECH-022', name: 'Hreflang alternate target conflicts with rel=canonical', namespace: 'TECH',
+  description: 'An alternate hreflang URL points to a page that declares a different rel=canonical target, breaking the alternate cluster.',
+  discipline: ['seo', 'geo'], severity: 'critical', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'high', ranking: 'high', citation: 'high' },
+  applicable_requirement: 'SEO §2 canonical and alternate harmony; localized pages in an hreflang cluster must be self-canonical',
+  remediation: 'Ensure each localized alternate page declares a self-referencing rel=canonical matching its own URL.',
+  verification: 'Check that target rel=canonical matches the alternate hreflang URL.',
+  check(ctx) {
+    const hits = [];
+    for (const p of ctx.site.pages) {
+      if (!p.hreflangs || p.hreflangs.length === 0) continue;
+      for (const alt of p.hreflangs) {
+        if (!alt.href) continue;
+        let normTarget;
+        try {
+          normTarget = ctx.site.normalize(new URL(alt.href, p.url).href);
+        } catch {
+          continue;
+        }
+        const targetPage = ctx.site.byUrl.get(normTarget);
+        if (!targetPage || targetPage.canonicals.length !== 1) continue;
+        let targetCanonical;
+        try {
+          targetCanonical = ctx.site.normalize(new URL(targetPage.canonicals[0], targetPage.url).href);
+        } catch {
+          continue;
+        }
+        if (targetCanonical !== normTarget) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Hreflang alternate ${alt.href} points to non-canonical page (canonical is ${targetCanonical})`,
+            evidence: [
+              `${p.url} lists ${alt.href} as alternate (${alt.lang})`,
+              `${normTarget} declares canonical ${targetCanonical}`,
+            ],
+            captured: { target: normTarget, canonical: targetCanonical },
+            expected: `self-canonical target (${normTarget})`,
+          });
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'TECH-023', name: 'Hydration gap: critical metadata or schema missing from initial server HTML response', namespace: 'TECH',
+  description: 'A single-page application or client-rendered document returns empty content mounting shells with virtually no static text (<20 words) in initial server HTML, breaking crawler and answer engine extraction without full headless JS rendering.',
+  discipline: ['seo', 'aeo'], severity: 'high', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'high', ranking: 'medium', citation: 'medium' },
+  applicable_requirement: 'SEO §2 crawlable HTML and server-side rendering; AEO §2 immediate extractability without execution delay',
+  remediation: 'Implement Server-Side Rendering (SSR) or Static Site Generation (SSG) so critical content and metadata are present in initial server HTML.',
+  verification: 'Inspect initial server HTML response to confirm main content and metadata are present before JavaScript execution.',
+  check(ctx) {
+    const hits = [];
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.rawHtml) continue;
+      const doc = parse(p.rawHtml);
+      const hasMountShell = doc.querySelector('#root, #__next, #app, [data-reactroot]');
+      const wordCount = p.rawVisibleWordCount || p.wordCount || 0;
+      const hasScripts = (p.scriptBytes || 0) > 0 || doc.querySelectorAll('script').length > 0;
+
+      if (hasMountShell && wordCount < 20 && hasScripts) {
+        hits.push({
+          subject: pageSubject(p),
+          summary: `Client-rendered mounting shell (${hasMountShell.tagName.toLowerCase()}${hasMountShell.id ? '#' + hasMountShell.id : ''}) contains only ${wordCount} words in initial HTML response`,
+          evidence: [
+            `shell element: <${hasMountShell.tagName.toLowerCase()} id="${hasMountShell.id || ''}">`,
+            `initial static word count: ${wordCount}`,
+            'search engines and AI agents requiring raw HTML cannot extract content without full JavaScript rendering',
+          ],
+          captured: wordCount,
+          expected: '>= 20 words with server-rendered content',
+        });
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'TECH-024', name: 'Excessive render-blocking script payload exceeding crawler execution budget', namespace: 'TECH',
+  description: 'The page delivers excessive inline/head script payloads (>1.2MB) or an excessive number of render-blocking head scripts (>12 without defer/async), exhausting crawler execution budgets and delaying indexing.',
+  discipline: ['seo', 'aeo'], severity: 'medium', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'high', ranking: 'low' },
+  applicable_requirement: 'SEO §2 crawler execution budget; Google Search Central JavaScript rendering guidelines',
+  remediation: 'Reduce JavaScript payload size, code-split vendor bundles, and add defer or async to head script tags.',
+  verification: 'Verify total initial script payload is under 1.2MB and head script tags are deferred or asynchronous.',
+  check(ctx) {
+    const hits = [];
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.rawHtml) continue;
+      const doc = parse(p.rawHtml);
+      const headScripts = doc.querySelectorAll('head script');
+      let blockingCount = 0;
+      for (const s of headScripts) {
+        const type = (s.getAttribute('type') || '').toLowerCase();
+        if (type === 'application/ld+json') continue;
+        const isAsync = s.hasAttribute('async');
+        const isDefer = s.hasAttribute('defer');
+        const isModule = type === 'module';
+        if (!isAsync && !isDefer && !isModule && s.hasAttribute('src')) {
+          blockingCount++;
+        }
+      }
+
+      const scriptBytes = p.scriptBytes || 0;
+      if (scriptBytes > 1200000 || blockingCount > 12) {
+        hits.push({
+          subject: pageSubject(p),
+          summary: `Page exceeds crawler script execution budget (${scriptBytes > 1200000 ? `${(scriptBytes / 1000000).toFixed(2)}MB script bytes` : ''}${scriptBytes > 1200000 && blockingCount > 12 ? '; ' : ''}${blockingCount > 12 ? `${blockingCount} render-blocking head scripts` : ''})`,
+          evidence: [
+            `script bytes: ${scriptBytes}`,
+            `render-blocking scripts in head: ${blockingCount}`,
+            'crawlers and generative search spiders may abort JavaScript execution under heavy script payloads',
+          ],
+          captured: { scriptBytes, blockingCount },
+          expected: 'scriptBytes <= 1.2MB and blockingCount <= 12',
+        });
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'TECH-025', name: 'Oversized uncompressed image asset or heavy inline image payload', namespace: 'TECH',
+  description: 'The page embeds an excessively large inline image data URI (>100KB) or references raw uncompressed image payloads exceeding 1MB, delaying Largest Contentful Paint (LCP) and exhausting mobile data budgets.',
+  discipline: ['seo', 'aeo'], severity: 'medium', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'medium', ranking: 'low' },
+  applicable_requirement: 'SEO §2 crawler media payload budget; Google Search Central image optimization guidelines',
+  remediation: 'Compress image assets using modern formats (WebP/AVIF), resize to display dimensions, and externalize inline base64 image data.',
+  verification: 'Confirm all image references are under 1MB and inline data URIs are under 100KB.',
+  check(ctx) {
+    const hits = [];
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.images) continue;
+      for (const img of p.images) {
+        const src = img.src || '';
+        if (src.startsWith('data:image/') && src.length > 100000) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Heavy inline base64 image payload (${Math.round(src.length / 1024)}KB) bloats HTML document size`,
+            evidence: [
+              `data URI length: ${src.length} characters (~${Math.round(src.length / 1024)}KB)`,
+              'inline image payloads delay initial HTML parsing and waste crawler network budgets',
+            ],
+            captured: { dataUriLength: src.length },
+            expected: 'inline image data URI <= 100KB',
+          });
+          break;
+        } else if (img.bytes && img.bytes > 1000000) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Oversized image asset "${src}" (${(img.bytes / 1000000).toFixed(2)}MB) delays LCP`,
+            evidence: [
+              `image source: ${src}`,
+              `image size: ${img.bytes} bytes`,
+              'images exceeding 1MB exhaust mobile rendering budgets and degrade Largest Contentful Paint',
+            ],
+            captured: { src, bytes: img.bytes },
+            expected: 'image bytes <= 1MB',
+          });
+          break;
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
 export function canonicalTargetProblem(ctx, predicate, label) {
   const hits = [];
   for (const p of ctx.site.pages) {

package/src/extractor/page.js

@@ -19,6 +19,35 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
     .map((l) => l.getAttribute('href'))
     .filter(Boolean);
 
+  const hreflangs = root
+    .querySelectorAll('link[hreflang]')
+    .filter((l) => {
+      const rel = (l.getAttribute('rel') || '').toLowerCase();
+      return rel.split(/\s+/).includes('alternate');
+    })
+    .map((l) => ({
+      lang: (l.getAttribute('hreflang') || '').trim(),
+      href: (l.getAttribute('href') || '').trim(),
+    }))
+    .filter((item) => item.lang || item.href);
+
+  const linkHeader = headers['link'] || headers['Link'];
+  if (linkHeader) {
+    const headerStr = Array.isArray(linkHeader) ? linkHeader.join(', ') : String(linkHeader);
+    const parts = headerStr.split(/,(?=\s*<)/);
+    for (const part of parts) {
+      const hrefMatch = part.match(/<([^>]+)>/);
+      const relMatch = part.match(/rel=["']?([^"';\s]+)["']?/i);
+      const hreflangMatch = part.match(/hreflang=["']?([^"';\s]+)["']?/i);
+      if (hrefMatch && relMatch && relMatch[1].toLowerCase() === 'alternate' && hreflangMatch) {
+        hreflangs.push({
+          lang: hreflangMatch[1].trim(),
+          href: hrefMatch[1].trim(),
+        });
+      }
+    }
+  }
+
   const robotsMetaRaw = [...(metas['robots'] || []), ...(metas['googlebot'] || [])].join(',').toLowerCase();
   const xRobots = String(headers['x-robots-tag'] || '').toLowerCase();
   const robotsDirectives = new Set(
@@ -42,6 +71,9 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
   const images = root.querySelectorAll('img').map((i) => ({
     src: i.getAttribute('src') || '',
     alt: i.getAttribute('alt'),
+    width: i.getAttribute('width') || null,
+    height: i.getAttribute('height') || null,
+    loading: (i.getAttribute('loading') || '').toLowerCase(),
   }));
 
   const jsonLd = [];
@@ -102,10 +134,154 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
       if (t) hiddenTexts.push(t);
     }
   }
-  for (const c of root.querySelectorAll('*')) {
-    // comments handled below via rawHtml scan in detectors
+  // Extract interactive CTAs (buttons, links with button role/classes, submit inputs)
+  const CTA_TEXT_RX = /\b(sign\s*up|get\s*started|request\s*(a\s*)?demo|book\s*(a\s*)?demo|schedule\s*(a\s*)?demo|try\s*(it\s*)?free|start\s*(free\s*)?trial|contact\s*sales|talk\s*to\s*sales|buy\s*now|subscribe|view\s*pricing|see\s*pricing|get\s*a\s*quote|download\s*whitepaper|download\s*report|join\s*waitlist)\b/i;
+  const CTA_ATTR_RX = /\b(btn|cta|button|primary-action|action-btn|submit)\b/i;
+
+  const ctas = [];
+  for (const el of root.querySelectorAll('button, a, input[type=submit], input[type=button]')) {
+    const tag = el.tagName.toLowerCase();
+    const text = (tag === 'input' ? el.getAttribute('value') : el.text) || '';
+    const cleanText = text.replace(/\s+/g, ' ').trim();
+    const href = el.getAttribute('href') || null;
+    const action = el.getAttribute('formaction') || null;
+    const role = (el.getAttribute('role') || '').toLowerCase();
+    const cls = el.getAttribute('class') || '';
+
+    const matchesKeyword = CTA_TEXT_RX.test(cleanText);
+    const matchesAttr = CTA_ATTR_RX.test(cls) || role === 'button' || tag === 'button' || el.getAttribute('type') === 'submit';
+
+    if (matchesKeyword || (matchesAttr && cleanText.length > 0 && cleanText.length <= 40)) {
+      let inNav = false;
+      let inHero = false;
+      let cur = el.parentNode;
+      while (cur && cur !== root) {
+        const tn = cur.tagName ? cur.tagName.toLowerCase() : '';
+        const c = cur.getAttribute ? (cur.getAttribute('class') || '') : '';
+        const id = cur.getAttribute ? (cur.getAttribute('id') || '') : '';
+        const r = cur.getAttribute ? (cur.getAttribute('role') || '') : '';
+        if (tn === 'nav' || tn === 'header' || r === 'navigation' || r === 'banner') inNav = true;
+        if (tn === 'header' || c.includes('hero') || id.includes('hero') || c.includes('banner') || id.includes('banner')) inHero = true;
+        cur = cur.parentNode;
+      }
+
+      const style = (el.getAttribute('style') || '').toLowerCase();
+      const widthMatch = style.match(/width:\s*(\d+)px/);
+      const heightMatch = style.match(/height:\s*(\d+)px/);
+      const fontSizeMatch = style.match(/font-size:\s*(\d+)px/);
+      const inlineWidth = widthMatch ? Number(widthMatch[1]) : (el.getAttribute('width') ? Number(el.getAttribute('width')) : null);
+      const inlineHeight = heightMatch ? Number(heightMatch[1]) : (el.getAttribute('height') ? Number(el.getAttribute('height')) : null);
+      const inlineFontSize = fontSizeMatch ? Number(fontSizeMatch[1]) : null;
+
+      ctas.push({
+        tag,
+        text: cleanText,
+        target: href || action || null,
+        role: role || null,
+        className: cls || null,
+        isPrimary: matchesKeyword,
+        inNav,
+        inHero,
+        style: style || null,
+        inlineWidth,
+        inlineHeight,
+        inlineFontSize,
+      });
+    }
   }
 
+  // Extract form elements
+  const forms = [];
+  for (const form of root.querySelectorAll('form')) {
+    const action = form.getAttribute('action') || null;
+    const method = (form.getAttribute('method') || 'GET').toUpperCase();
+    const inputEls = form.querySelectorAll('input, select, textarea');
+    const inputs = [];
+    let hasSubmit = !!form.querySelector('button[type=submit], input[type=submit]');
+    if (!hasSubmit && form.querySelector('button:not([type=button]):not([type=reset])')) {
+      hasSubmit = true;
+    }
+
+    let nonHiddenCount = 0;
+    for (const inp of inputEls) {
+      const type = (inp.getAttribute('type') || (inp.tagName.toLowerCase() === 'textarea' ? 'textarea' : 'text')).toLowerCase();
+      const name = inp.getAttribute('name') || null;
+      const id = inp.getAttribute('id') || null;
+      const required = inp.hasAttribute('required');
+      const autocomplete = inp.getAttribute('autocomplete')?.trim() || null;
+      const inputmode = inp.getAttribute('inputmode')?.trim() || null;
+      const placeholder = inp.getAttribute('placeholder') || null;
+
+      let hasLabel = inp.hasAttribute('aria-label') || inp.hasAttribute('aria-labelledby');
+      if (!hasLabel && id) {
+        hasLabel = !!root.querySelector(`label[for="${id}"]`);
+      }
+      if (!hasLabel) {
+        let parent = inp.parentNode;
+        while (parent && parent !== form) {
+          if (parent.tagName && parent.tagName.toLowerCase() === 'label') {
+            hasLabel = true;
+            break;
+          }
+          parent = parent.parentNode;
+        }
+      }
+
+      if (type !== 'hidden') nonHiddenCount++;
+      inputs.push({ name, id, type, required, autocomplete, inputmode, placeholder, hasLabel });
+    }
+
+    forms.push({
+      action,
+      method,
+      inputs,
+      fieldCount: nonHiddenCount,
+      hasSubmit,
+    });
+  }
+
+  // Extract trust / proof signals in DOM
+  const TRUST_RX = /\b(soc\s*2|iso\s*27001|gdpr|hipaa|pci[- ]dss|money[- ]back\s*guarantee|cancel\s*anytime|no\s*credit\s*card\s*required|trusted\s*by|featured\s*in|customer\s*reviews|satisfaction\s*guarantee|sla\s*guarantee)\b/i;
+  const trustBadges = [];
+  for (const el of root.querySelectorAll('div, section, p, span, img, li, footer')) {
+    const txt = el.text.replace(/\s+/g, ' ').trim();
+    const alt = el.getAttribute('alt') || '';
+    const m = TRUST_RX.exec(txt) || TRUST_RX.exec(alt);
+    if (m && !trustBadges.some((t) => t.signal.toLowerCase() === m[1].toLowerCase())) {
+      trustBadges.push({
+        signal: m[1],
+        context: txt ? txt.slice(0, 100) : alt,
+      });
+    }
+  }
+
+  // Analytics & Tag Manager detection
+  const ANALYTICS_RX = /\b(googletagmanager\.com|google-analytics\.com|gtag\(|posthog|segment\.com\/analytics|mixpanel|rudderstack|plausible|fathom)\b/i;
+  const analyticsTags = [];
+  for (const s of root.querySelectorAll('script')) {
+    const src = s.getAttribute('src') || '';
+    const content = s.text || '';
+    if (ANALYTICS_RX.test(src) || ANALYTICS_RX.test(content)) {
+      analyticsTags.push(src || 'inline-tag');
+    }
+  }
+
+  // Navigation link count
+  const navLinksCount = root.querySelectorAll('nav a, header a, [role=navigation] a').length;
+
+  let domNodeCount = 0;
+  let maxDomDepth = 0;
+  const walkDom = (node, depth) => {
+    domNodeCount++;
+    if (depth > maxDomDepth) maxDomDepth = depth;
+    for (const child of node.childNodes || []) {
+      if (child.nodeType === 1) {
+        walkDom(child, depth + 1);
+      }
+    }
+  };
+  walkDom(root, 1);
+
   return {
     url,
     sourceFile,
@@ -118,6 +294,7 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
     metaDescription: metas['description']?.[0] ?? null,
     metas,
     canonicals,
+    hreflangs,
     robotsDirectives: [...robotsDirectives],
     noindex: robotsDirectives.has('noindex') || robotsDirectives.has('none'),
     nosnippet: robotsDirectives.has('nosnippet') || [...robotsDirectives].some((d) => /^max-snippet:\s*0$/.test(d)),
@@ -138,5 +315,12 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
     orderedLists,
     tables,
     hiddenTexts,
+    ctas,
+    forms,
+    trustBadges,
+    analyticsTags,
+    navLinksCount,
+    domNodeCount,
+    maxDomDepth,
   };
 }

package/src/extractor/site.js

@@ -45,6 +45,11 @@ export function buildSiteFromDir(dir, { baseUrl = 'https://example.test' } = {})
   const robotsPath = path.join(dir, 'robots.txt');
   const robotsText = fs.existsSync(robotsPath) ? fs.readFileSync(robotsPath, 'utf8') : null;
 
+  const llmsPath = path.join(dir, 'llms.txt');
+  const llmsText = fs.existsSync(llmsPath) ? fs.readFileSync(llmsPath, 'utf8') : null;
+  const llmsFullPath = path.join(dir, 'llms-full.txt');
+  const llmsFullText = fs.existsSync(llmsFullPath) ? fs.readFileSync(llmsFullPath, 'utf8') : null;
+
   const sitemaps = [];
   for (const f of fs.readdirSync(dir)) {
     if (/sitemap.*\.xml$/i.test(f)) {
@@ -52,7 +57,7 @@ export function buildSiteFromDir(dir, { baseUrl = 'https://example.test' } = {})
     }
   }
 
-  return assembleSite({ baseUrl, pages, robotsText, sitemaps, transport, mode: 'built_output', location: dir });
+  return assembleSite({ baseUrl, pages, robotsText, sitemaps, transport, mode: 'built_output', location: dir, llmsText, llmsFullText });
 }
 
 /** Build a SiteModel by fetching a deployed URL set (target URL + same-origin discovery, bounded). */
@@ -122,7 +127,7 @@ export async function buildSiteFromUrl(startUrl, {
   return site;
 }
 
-function assembleSite({ baseUrl, pages, robotsText, sitemaps, transport, mode, location, crawl = null }) {
+function assembleSite({ baseUrl, pages, robotsText, sitemaps, transport, mode, location, crawl = null, llmsText = null, llmsFullText = null }) {
   const byUrl = new Map();
   for (const p of pages) byUrl.set(normalize(p.url), p);
 
@@ -179,6 +184,8 @@ function assembleSite({ baseUrl, pages, robotsText, sitemaps, transport, mode, l
     depth,
     normalize,
     crawl,
+    llmsTxt: llmsText != null ? { raw: llmsText, found: true, path: '/llms.txt' } : null,
+    llmsFullTxt: llmsFullText != null ? { raw: llmsFullText, found: true, path: '/llms-full.txt' } : null,
   };
 }
 

package/src/installer/index.js

@@ -15,6 +15,7 @@ import {
 } from './providers.js';
 
 export { PROVIDERS, PROVIDER_IDS, parseProviderList, normalizeProviderId } from './providers.js';
+import { diagnoseMcpTransports } from '../connectors/mcp/diagnostics.js';
 
 export const EXIT_CODES = Object.freeze({
   success: 0,
@@ -106,6 +107,7 @@ export function parseInstallerArgs(argv) {
     help: false,
     version: false,
     failOnUpdate: false,
+    mcp: false,
     installMode: 'copy',
     skillSelection: null,
     _: [],
@@ -121,6 +123,7 @@ export function parseInstallerArgs(argv) {
     else if (arg === '--dry-run') args.dryRun = true;
     else if (arg === '--force') args.force = true;
     else if (arg === '--json') args.json = true;
+    else if (arg === '--mcp') args.mcp = true;
     else if (arg === '--all') { args.all = true; args.yes = true; }
     else if (arg === '--help' || arg === '-h') args.help = true;
     else if (arg === '--version' || arg === '-v') args.version = true;
@@ -1039,7 +1042,8 @@ function combineInstalledState(skillState, profileState) {
 export async function doctorCommand(args, options = {}) {
   const context = resolveTargets('doctor', args, options);
   const packageInfo = loadPackageInfo(context.roots.packageRoot);
-  const capabilities = await runtimeCapabilities(options);
+  const isMcp = Boolean(args?.mcp || options?.mcp);
+  const capabilities = await runtimeCapabilities({ ...options, mcp: isMcp, packageRoot: context.roots.packageRoot });
   const checks = [];
   const nodeMajor = Number.parseInt(process.versions.node.split('.')[0], 10);
   checks.push({ level: nodeMajor >= 20 ? 'PASS' : 'FAIL', message: `Node.js ${process.versions.node}` });
@@ -1073,8 +1077,21 @@ export async function doctorCommand(args, options = {}) {
     }
   }
 
+  let mcpSummary = null;
+  if (isMcp) {
+    const mcpDiag = await diagnoseMcpTransports({
+      packageRoot: context.roots.packageRoot,
+      projectRoot: context.roots.projectRoot,
+      ...options,
+    });
+    checks.push(...mcpDiag.checks);
+    mcpSummary = mcpDiag.summary;
+  }
+
   const hasFail = checks.some((check) => check.level === 'FAIL');
-  return { ok: !hasFail, command: 'doctor', checks, capabilities, exitCode: hasFail ? EXIT_CODES.integrityFailure : EXIT_CODES.success };
+  const result = { ok: !hasFail, command: 'doctor', checks, capabilities, exitCode: hasFail ? EXIT_CODES.integrityFailure : EXIT_CODES.success };
+  if (mcpSummary) result.mcp = mcpSummary;
+  return result;
 }
 
 async function optionalDependencyAvailable(name, options = {}) {
@@ -1148,6 +1165,24 @@ export async function runtimeCapabilities(options = {}) {
       limitations: [configured ? definition.readyLimitation : `${definition.credential} is not configured`],
     });
   }
+
+  if (options.mcp) {
+    const pkgRoot = options.packageRoot || packageRoot();
+    const pilotPath = path.resolve(pkgRoot, 'src/connectors/mcp/pilotServer.js');
+    const pilotAvailable = exists(pilotPath);
+    capabilities.push({
+      id: 'mcp_transport',
+      state: pilotAvailable ? 'ready' : 'missing_dependency',
+      requirements: ['node:child_process', 'node:readline', 'pilotServer.js'],
+      limitations: pilotAvailable
+        ? [
+            'MCP transport is read-only and restricted to explicit allowlisted servers and tools',
+            'remote endpoints require HTTPS and must not resolve to private or loopback ranges',
+          ]
+        : ['MCP stdio pilotServer.js script was not found'],
+    });
+  }
+
   return capabilities;
 }
 
@@ -1350,6 +1385,13 @@ function renderList(result) {
 
 function renderDoctor(result) {
   const lines = ['Citable doctor', '', ...result.checks.map((check) => `${check.level.padEnd(5)} ${check.message}`)];
+  if (result.mcp) {
+    lines.push('', 'MCP transport diagnostic');
+    lines.push(`  allowlist:          ${result.mcp.servers_count} server(s), ${result.mcp.tools_count} read-only tool(s)`);
+    lines.push(`  stdio execution:    ${result.mcp.stdio_ready ? 'ready' : 'failed'}`);
+    lines.push(`  http guards:        ${result.mcp.http_guards_active ? 'active' : 'failed'}`);
+    lines.push(`  registry bindings:  ${result.mcp.connections_found} configured connection(s)`);
+  }
   if (result.capabilities?.length) {
     lines.push('', 'Runtime capabilities');
     for (const capability of result.capabilities) {
@@ -1391,6 +1433,7 @@ Options
   --project, --local, -p    Install/check project-local skill locations
   --global, --user, -g      Install/check user-global skill locations
   --scope=<project|global>  Scope equivalent
+  --mcp                     Run MCP transport diagnostics (stdio, HTTP guards, allowlist, connections)
   --yes, -y                 Confirm non-interactively
   --dry-run                 Preview filesystem changes without mutation
   --force                   Replace locally modified or unmanaged citable directory