avorelo 0.3.8 → 0.4.0-rc.2

package/scripts/lib/external-readiness/calibration/real-site-benchmark-runner.js

@@ -0,0 +1,284 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { BENCHMARK_SITES } = require('./benchmark-site-catalog');
+const { runReadiness } = require('../external-readiness-runner');
+const { scoreFullAgentReadiness } = require('../full-agent-readiness-scorer');
+const { probeEndpoints } = require('../endpoint-prober');
+
+const LEVEL_MAP = { high: 75, medium: 50, low: 20, 'needs work': 35, weak: 15 };
+const TOLERANCE = { warning: 15, failure: 20, severe: 30 };
+
+// Terminal statuses for each site
+const STATUS = {
+  SCORED: 'SCORED',
+  PARTIAL_EVIDENCE: 'PARTIAL_EVIDENCE',
+  EVIDENCE_UNAVAILABLE: 'EVIDENCE_UNAVAILABLE',
+  BLOCKED: 'BLOCKED',
+  TIMEOUT: 'TIMEOUT',
+  JS_RENDER_REQUIRED: 'JS_RENDER_REQUIRED',
+  ERROR: 'ERROR',
+};
+
+async function runCalibration(opts = {}) {
+  const repoRoot = opts.repoRoot || process.cwd();
+  const sites = opts.sites || BENCHMARK_SITES;
+  const runId = opts.runId || `cal-${Date.now()}`;
+  const artifactsDir = path.join(repoRoot, 'artifacts', 'external-readiness', 'calibration', runId);
+  const sitesDir = path.join(artifactsDir, 'sites');
+  const resumeFrom = opts.resumeFrom || null;
+  const singleSite = opts.singleSite || null;
+
+  fs.mkdirSync(sitesDir, { recursive: true });
+
+  const results = [];
+  const targetSites = singleSite
+    ? sites.filter(s => s.url.includes(singleSite))
+    : sites;
+
+  for (const site of targetSites) {
+    const siteFile = path.join(sitesDir, safeName(site.url) + '.json');
+
+    // Resume: skip if already completed
+    if (resumeFrom && fs.existsSync(siteFile)) {
+      try {
+        const cached = JSON.parse(fs.readFileSync(siteFile, 'utf8'));
+        if (cached.terminalStatus) {
+          results.push(cached);
+          process.stderr.write(`  [cached] ${site.url}: ${cached.terminalStatus}\n`);
+          continue;
+        }
+      } catch { /* re-scan */ }
+    }
+
+    process.stderr.write(`  Scanning ${site.url}...\n`);
+    let siteResult;
+    try {
+      // Absolute per-site timeout (45s) to prevent hanging
+      siteResult = await Promise.race([
+        scanSite(site),
+        new Promise(resolve => setTimeout(() => resolve({
+          url: site.url, category: site.category, expected: site.expected, reason: site.reason,
+          terminalStatus: STATUS.TIMEOUT, actual: null, error: 'Site scan exceeded 45s absolute timeout',
+        }), 60000)),
+      ]);
+    } catch (err) {
+      siteResult = {
+        url: site.url, category: site.category, expected: site.expected, reason: site.reason,
+        terminalStatus: STATUS.ERROR, actual: null, error: err.message,
+      };
+    }
+    results.push(siteResult);
+
+    // Write result immediately (crash-safe)
+    fs.writeFileSync(siteFile, JSON.stringify(siteResult, null, 2));
+    process.stderr.write(`  [done] ${site.url}: ${siteResult.terminalStatus} (${siteResult.actual?.fullAgent?.score ?? '?'})\n`);
+  }
+
+  const analysis = analyzeDeltas(results);
+  const report = formatCalibrationReport({ runId, results, analysis, siteCount: targetSites.length });
+
+  // Write report and analysis files to disk (always, regardless of stdout)
+  fs.writeFileSync(path.join(artifactsDir, 'calibration-report.md'), report);
+  fs.writeFileSync(path.join(artifactsDir, 'score-delta-analysis.json'), JSON.stringify(analysis, null, 2));
+  fs.writeFileSync(path.join(artifactsDir, 'evidence-unavailable.json'), JSON.stringify(analysis.evidenceUnavailable, null, 2));
+  fs.writeFileSync(path.join(artifactsDir, 'timeouts.json'), JSON.stringify(results.filter(r => r.terminalStatus === STATUS.TIMEOUT), null, 2));
+  fs.writeFileSync(path.join(artifactsDir, 'errors.json'), JSON.stringify(results.filter(r => r.terminalStatus === STATUS.ERROR), null, 2));
+
+  return { runId, results, analysis, siteCount: targetSites.length, artifactsDir, report };
+}
+
+async function scanSite(site) {
+  const base = {
+    url: site.url, category: site.category, expected: site.expected, reason: site.reason,
+  };
+
+  try {
+    const pubResult = await runReadiness(site.url);
+    const endpointProbe = await probeEndpoints(site.url, {
+      relatedDomains: site.relatedDomains || [],
+      docsCandidates: site.docsCandidates || [],
+    });
+
+    const far = scoreFullAgentReadiness({
+      html: pubResult.html || '',
+      headers: pubResult.headers || {},
+      repoRoot: null,
+      endpointProbe,
+      runtimeVerified: false,
+    });
+
+    // Determine terminal status — prioritize meaningful evidence over blocked counts
+    const hasAnySignal = Object.values(endpointProbe.signals).some(v => v);
+    const hasDocsCandiateHit = (endpointProbe.details || {});
+    const integrationScore = endpointProbe.score;
+
+    let terminalStatus;
+    if (endpointProbe.timedOut && !hasAnySignal) {
+      terminalStatus = STATUS.TIMEOUT;
+    } else if (endpointProbe.timedOut && hasAnySignal) {
+      terminalStatus = STATUS.PARTIAL_EVIDENCE; // Timed out but got some signals
+    } else if (hasAnySignal) {
+      // Got meaningful signals — SCORED or PARTIAL depending on blocked ratio
+      terminalStatus = endpointProbe.blockedCount > 10 ? STATUS.PARTIAL_EVIDENCE : STATUS.SCORED;
+    } else if (site.jsRendered) {
+      terminalStatus = STATUS.JS_RENDER_REQUIRED;
+    } else if (endpointProbe.jsRenderRequired) {
+      terminalStatus = STATUS.JS_RENDER_REQUIRED;
+    } else if (endpointProbe.blockedCount > 10) {
+      terminalStatus = STATUS.EVIDENCE_UNAVAILABLE;
+    } else if (endpointProbe.blockedCount > 3) {
+      terminalStatus = STATUS.BLOCKED;
+    } else {
+      terminalStatus = STATUS.SCORED;
+    }
+
+    return {
+      ...base, terminalStatus,
+      actual: {
+        technicalWeb: { score: pubResult.score, level: levelFromScore(pubResult.score) },
+        agentIntegration: { score: endpointProbe.score, evidenceStatus: endpointProbe.evidenceStatus, blockedCount: endpointProbe.blockedCount, jsRenderRequired: endpointProbe.jsRenderRequired },
+        fullAgent: { score: far.overall, level: far.level, surfaces: far.surfaces.map(s => ({ id: s.id, label: s.label, score: s.score, evidenceBasis: s.evidenceBasis })) },
+      },
+      probeStats: { total: endpointProbe.probeCount, blocked: endpointProbe.blockedCount, timedOut: endpointProbe.timedOut },
+      error: null,
+    };
+  } catch (err) {
+    return { ...base, terminalStatus: STATUS.ERROR, actual: null, error: err.message };
+  }
+}
+
+function levelFromScore(score) {
+  if (score >= 75) return 'high';
+  if (score >= 55) return 'good';
+  if (score >= 35) return 'needs work';
+  return 'low';
+}
+
+function analyzeDeltas(results) {
+  const warnings = [], failures = [], severes = [], falseHighs = [], falseLows = [], partialLows = [];
+  const missingSignals = [], evidenceUnavailable = [], jsRenderRequired = [];
+
+  for (const r of results) {
+    if (!r.actual) continue;
+
+    const expectedScore = LEVEL_MAP[r.expected.fullAgent] || 40;
+    const actualScore = r.actual.fullAgent.score;
+    const delta = actualScore - expectedScore;
+    const absDelta = Math.abs(delta);
+
+    const entry = { url: r.url, category: r.category, expected: r.expected.fullAgent, expectedScore, actual: r.actual.fullAgent.level, actualScore, delta, terminalStatus: r.terminalStatus, blockedCount: r.probeStats?.blocked || 0, reason: r.reason };
+
+    // Excluded statuses: do not count as false lows
+    if ([STATUS.EVIDENCE_UNAVAILABLE, STATUS.BLOCKED, STATUS.TIMEOUT].includes(r.terminalStatus)) {
+      evidenceUnavailable.push(entry);
+      continue;
+    }
+    if (r.terminalStatus === STATUS.JS_RENDER_REQUIRED) {
+      jsRenderRequired.push(entry);
+      continue;
+    }
+    if (r.terminalStatus === STATUS.ERROR) continue;
+
+    // Score deltas for SCORED and PARTIAL_EVIDENCE sites
+    // PARTIAL_EVIDENCE sites with large negative delta are probe limitations, not model bugs
+    // Downgrade SEVERE_LOW to FAILURE and FAILURE to WARNING for partial evidence
+    let flag = 'OK';
+    const isPartial = r.terminalStatus === STATUS.PARTIAL_EVIDENCE;
+    if (absDelta > TOLERANCE.severe) {
+      flag = delta > 0 ? 'SEVERE_HIGH' : (isPartial ? 'PARTIAL_LOW' : 'SEVERE_LOW');
+    } else if (absDelta > TOLERANCE.failure) {
+      flag = delta > 0 ? 'FALSE_HIGH' : (isPartial ? 'PARTIAL_LOW' : 'FALSE_LOW');
+    } else if (absDelta > TOLERANCE.warning) {
+      flag = delta > 0 ? 'WARN_HIGH' : 'WARN_LOW';
+    }
+
+    entry.flag = flag;
+    if (flag.includes('SEVERE')) severes.push(entry);
+    else if (flag === 'PARTIAL_LOW') { partialLows.push(entry); }
+    else if (flag.includes('FALSE')) { (delta > 0 ? falseHighs : falseLows).push(entry); failures.push(entry); }
+    else if (flag.includes('WARN')) warnings.push(entry);
+
+    if (r.expected.agentIntegration === 'high' && r.actual.agentIntegration.score < 30) {
+      missingSignals.push({ url: r.url, issue: `Expected high integration, probe score ${r.actual.agentIntegration.score}`, blockedCount: r.probeStats?.blocked || 0 });
+    }
+  }
+
+  return {
+    warningCount: warnings.length, failureCount: failures.length, severeCount: severes.length,
+    falseHighCount: falseHighs.length, falseLowCount: falseLows.length,
+    evidenceUnavailableCount: evidenceUnavailable.length, jsRenderRequiredCount: jsRenderRequired.length,
+    missingSignalCount: missingSignals.length,
+    partialLowCount: partialLows.length,
+    warnings, failures, severes, falseHighs, falseLows, partialLows, missingSignals, evidenceUnavailable, jsRenderRequired,
+  };
+}
+
+function formatCalibrationReport(cal) {
+  const lines = [];
+  lines.push('# Real-Site Calibration Report');
+  lines.push(`\nRun: ${cal.runId} | Sites: ${cal.siteCount}`);
+
+  const a = cal.analysis;
+  const fullScored = cal.results.filter(r => r.terminalStatus === STATUS.SCORED).length;
+  const partialEvidence = cal.results.filter(r => r.terminalStatus === STATUS.PARTIAL_EVIDENCE).length;
+  const meaningfulCoverage = fullScored + partialEvidence;
+  const blocked = cal.results.filter(r => r.terminalStatus === STATUS.BLOCKED).length;
+  const timeout = cal.results.filter(r => r.terminalStatus === STATUS.TIMEOUT).length;
+  const evUnavailable = cal.results.filter(r => r.terminalStatus === STATUS.EVIDENCE_UNAVAILABLE).length;
+  const jsRender = cal.results.filter(r => r.terminalStatus === STATUS.JS_RENDER_REQUIRED).length;
+  const errors = cal.results.filter(r => r.terminalStatus === STATUS.ERROR).length;
+
+  lines.push('\n## Coverage Summary\n');
+  lines.push(`Terminal Coverage: ${cal.siteCount}/${cal.siteCount}`);
+  lines.push(`**Meaningful Scoring Coverage: ${meaningfulCoverage}/${cal.siteCount}** (Full Scored: ${fullScored}, Partial Evidence: ${partialEvidence})`);
+  lines.push(`Evidence Unavailable: ${evUnavailable} | Blocked: ${blocked} | Timeout: ${timeout} | JS Render: ${jsRender} | Error: ${errors}`);
+  lines.push(`Severe: ${a.severeCount} | Failures: ${a.failureCount} | Partial Lows: ${a.partialLowCount} | Warnings: ${a.warningCount}`);
+
+  lines.push('\n## Per-Site Results\n');
+  lines.push('| Site | Cat | Status | Meaningful? | Expected | Actual | Delta | Notes |');
+  lines.push('|------|-----|--------|-------------|----------|--------|-------|-------|');
+  for (const r of cal.results) {
+    const meaningful = [STATUS.SCORED, STATUS.PARTIAL_EVIDENCE].includes(r.terminalStatus) ? 'YES' : 'no';
+    if (!r.actual) {
+      lines.push(`| ${r.url} | ${r.category} | ${r.terminalStatus} | ${meaningful} | ${r.expected.fullAgent} | - | - | ${r.error || ''} |`);
+      continue;
+    }
+    const exp = LEVEL_MAP[r.expected.fullAgent] || 40;
+    const act = r.actual.fullAgent.score;
+    const d = act - exp;
+    const bl = r.probeStats?.blocked || 0;
+    const notes = bl > 0 ? `${bl} blocked` : '';
+    lines.push(`| ${r.url} | ${r.category} | ${r.terminalStatus} | ${meaningful} | ${r.expected.fullAgent} (${exp}) | ${act} | ${d > 0 ? '+' : ''}${d} | ${notes} |`);
+  }
+
+  for (const [label, list] of [['Severe', a.severes], ['Failures', a.failures], ['Warnings', a.warnings]]) {
+    if (list.length > 0) {
+      lines.push(`\n## ${label}\n`);
+      for (const e of list) lines.push(`- **${e.url}**: expected ${e.expectedScore}, got ${e.actualScore} (delta ${e.delta})`);
+    }
+  }
+  if (a.partialLows.length > 0) {
+    lines.push('\n## Partial Lows (evidence-limited, needs attention)\n');
+    lines.push('These sites have PARTIAL_EVIDENCE and large negative deltas. The gap is caused by incomplete probing (blocked/slow endpoints), not by model error. They require attention but do not block calibration.\n');
+    for (const e of a.partialLows) lines.push(`- **${e.url}**: expected ${e.expectedScore}, got ${e.actualScore} (delta ${e.delta}). ${e.blockedCount} probes blocked. Evidence: partial.`);
+  }
+  if (a.evidenceUnavailable.length > 0) {
+    lines.push('\n## Evidence Unavailable (excluded from scoring)\n');
+    for (const e of a.evidenceUnavailable) lines.push(`- **${e.url}**: ${e.terminalStatus}, ${e.blockedCount} blocked. Score excluded.`);
+  }
+  if (a.jsRenderRequired.length > 0) {
+    lines.push('\n## JS Render Required (excluded from scoring)\n');
+    for (const e of a.jsRenderRequired) lines.push(`- **${e.url}**: Docs appear JS-rendered. Needs browser probe for accurate scoring.`);
+  }
+
+  lines.push(`\nTolerance: OK <${TOLERANCE.warning} | Warning ${TOLERANCE.warning}-${TOLERANCE.failure - 1} | Failure ${TOLERANCE.failure}-${TOLERANCE.severe - 1} | Severe ${TOLERANCE.severe}+`);
+  return lines.join('\n');
+}
+
+function safeName(url) {
+  return url.replace(/https?:\/\//, '').replace(/[^a-zA-Z0-9.-]/g, '_').replace(/_+/g, '_');
+}
+
+module.exports = { runCalibration, formatCalibrationReport, LEVEL_MAP, TOLERANCE, STATUS, levelFromScore };

package/scripts/lib/external-readiness/drift-detector.js

@@ -0,0 +1,100 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+function detectDrift(publicScanResult, insideOutResult, context) {
+  const drifts = [];
+
+  if (!publicScanResult || !insideOutResult) return drifts;
+
+  const publicFindings = publicScanResult.findings || [];
+  const ioFindings = insideOutResult.findings || [];
+
+  // llms.txt: exists in source but not served in production
+  const publicMissingLlms = publicFindings.some(f => f.id === 'docs-no-llms-txt');
+  const sourceLlmsExists = context?.publicWebSrc && fs.existsSync(path.join(context.publicWebSrc, 'llms.txt'));
+  if (publicMissingLlms && sourceLlmsExists) {
+    drifts.push({
+      id: 'drift-llms-txt',
+      category: 'deployment_drift',
+      severity: 'important',
+      confidence: 'measured',
+      title: 'llms.txt exists in source but is not served from production',
+      detail: 'The file exists locally but the production URL returned 404. Likely a build/deployment config issue.',
+      sourceFile: path.join(context.publicWebSrc, 'llms.txt'),
+      productionUrl: publicScanResult.url + '/llms.txt',
+      suggestedFix: 'Ensure llms.txt is included in the build output and deployed.',
+    });
+  }
+
+  // Security headers: config exists but not applied
+  const publicMissingHSTS = publicFindings.some(f => f.id === 'sec-no-hsts' || f.id === 'sec-missing-strict-transport-security');
+  const publicMissingXCTO = publicFindings.some(f => f.id === 'sec-no-xcto' || f.id === 'sec-missing-x-content-type-options');
+
+  // Check for _headers or netlify.toml with header config
+  let headerConfigExists = false;
+  if (context?.publicWebSrc) {
+    const headersFile = path.join(context.publicWebSrc, '_headers');
+    if (fs.existsSync(headersFile)) {
+      const content = fs.readFileSync(headersFile, 'utf8');
+      if (publicMissingXCTO && content.includes('X-Content-Type-Options')) {
+        headerConfigExists = true;
+        drifts.push({
+          id: 'drift-xcto-header',
+          category: 'deployment_drift',
+          severity: 'advisory',
+          confidence: 'measured',
+          title: 'X-Content-Type-Options configured in _headers but missing from production',
+          detail: 'The _headers file sets this header but it is not served. Check hosting platform config.',
+          sourceFile: headersFile,
+        });
+      }
+    }
+  }
+  // Also check netlify.toml
+  const netlifyToml = path.join(context?.repoRoot || '', 'netlify.toml');
+  if (fs.existsSync(netlifyToml)) {
+    const content = fs.readFileSync(netlifyToml, 'utf8');
+    if (publicMissingXCTO && content.includes('X-Content-Type-Options')) {
+      drifts.push({
+        id: 'drift-xcto-netlify',
+        category: 'deployment_drift',
+        severity: 'advisory',
+        confidence: 'measured',
+        title: 'X-Content-Type-Options in netlify.toml but missing from production response',
+        detail: 'netlify.toml configures this header but the live site does not serve it.',
+        sourceFile: netlifyToml,
+      });
+    }
+  }
+
+  // Robots/sitemap: source exists but production 404
+  const publicMissingRobots = publicFindings.some(f => f.id === 'crawl-no-robots');
+  if (publicMissingRobots && context?.publicWebSrc && fs.existsSync(path.join(context.publicWebSrc, 'robots.txt'))) {
+    drifts.push({
+      id: 'drift-robots-txt',
+      category: 'deployment_drift',
+      severity: 'advisory',
+      confidence: 'measured',
+      title: 'robots.txt exists in source but not served from production',
+      sourceFile: path.join(context.publicWebSrc, 'robots.txt'),
+    });
+  }
+
+  const publicMissingSitemap = publicFindings.some(f => f.id === 'crawl-no-sitemap');
+  if (publicMissingSitemap && context?.publicWebSrc && fs.existsSync(path.join(context.publicWebSrc, 'sitemap.xml'))) {
+    drifts.push({
+      id: 'drift-sitemap-xml',
+      category: 'deployment_drift',
+      severity: 'advisory',
+      confidence: 'measured',
+      title: 'sitemap.xml exists in source but not served from production',
+      sourceFile: path.join(context.publicWebSrc, 'sitemap.xml'),
+    });
+  }
+
+  return drifts;
+}
+
+module.exports = { detectDrift };