@nforma.ai/nforma 0.2.1 → 0.29.0

package/bin/harness-diagnostic.cjs

@@ -0,0 +1,326 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * bin/harness-diagnostic.cjs — Unified harness diagnostic tool for nForma.
+ *
+ * Cross-references scoreboard, conformance events, token usage, stall detector,
+ * and circuit breaker state into a single structured health report with
+ * actionable recommendations.
+ *
+ * Exports: generateReport, formatTerminalReport
+ * CLI: node bin/harness-diagnostic.cjs [--json] [--cwd /path]
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+// ─── Module resolution helpers ──────────────────────────────────────────────
+
+function tryRequire(modulePath) {
+  try { return require(modulePath); } catch (_) { return null; }
+}
+
+function findModule(name) {
+  return tryRequire(path.join(__dirname, name)) ||
+         tryRequire(path.join(__dirname, '..', 'bin', name));
+}
+
+// ─── generateReport ─────────────────────────────────────────────────────────
+
+/**
+ * Generate a structured harness diagnostic report.
+ * @param {string} cwd - Project root directory.
+ * @param {Object} [options]
+ * @param {Object} [options.config] - Pre-loaded config (optional).
+ * @returns {Object} Structured report object.
+ */
+function generateReport(cwd, options = {}) {
+  const report = {
+    timestamp: new Date().toISOString(),
+    slot_availability: [],
+    pass_at_k: { total: 0, pass_at_1: 0, pass_at_3: 0, avg_k: 0 },
+    token_spend: {
+      total_records: 0,
+      total_input: 0,
+      total_output: 0,
+      by_slot: {},
+      note: 'Token values are currently unreliable (all zeros in Claude Code subagent transcripts)',
+    },
+    stall_events: [],
+    circuit_breaker: {
+      active: false,
+      disabled: false,
+      last_triggered: null,
+    },
+    recommendations: [],
+  };
+
+  const planningPaths = findModule('planning-paths.cjs');
+
+  // ── Section 1: Slot Availability from scoreboard ──────────────────────────
+  try {
+    let scoreboardPath;
+    if (planningPaths) {
+      scoreboardPath = planningPaths.resolveWithFallback(cwd, 'quorum-scoreboard');
+    } else {
+      scoreboardPath = path.join(cwd, '.planning', 'quorum', 'scoreboard.json');
+    }
+
+    if (fs.existsSync(scoreboardPath)) {
+      const scoreboard = JSON.parse(fs.readFileSync(scoreboardPath, 'utf8'));
+      const slots = scoreboard.slots || {};
+      const slotAgg = {};
+
+      for (const [key, data] of Object.entries(slots)) {
+        const slotName = key.split(':')[0];
+        if (!slotAgg[slotName]) slotAgg[slotName] = { tp: 0, fn: 0 };
+        slotAgg[slotName].tp += (data.tp || 0);
+        slotAgg[slotName].fn += (data.fn || 0);
+      }
+
+      for (const [slotName, agg] of Object.entries(slotAgg)) {
+        const total = agg.tp + agg.fn;
+        const rate = total > 0 ? agg.tp / total : 0;
+        let status = 'unknown';
+        if (total > 0) {
+          if (rate >= 0.8) status = 'healthy';
+          else if (rate >= 0.5) status = 'degraded';
+          else status = 'critical';
+        }
+        report.slot_availability.push({
+          slot: slotName,
+          total_rounds: total,
+          successes: agg.tp,
+          failures: agg.fn,
+          success_rate: Math.round(rate * 1000) / 1000,
+          status,
+        });
+      }
+    }
+  } catch (_) {}
+
+  // ── Section 2: Pass@k from conformance events ────────────────────────────
+  try {
+    const vqh = findModule('verify-quorum-health.cjs');
+    if (vqh && vqh.computePassAtKRates) {
+      let conformancePath;
+      if (planningPaths) {
+        conformancePath = planningPaths.resolveWithFallback(cwd, 'conformance-events');
+      } else {
+        conformancePath = path.join(cwd, '.planning', 'telemetry', 'conformance-events.jsonl');
+      }
+      report.pass_at_k = vqh.computePassAtKRates(conformancePath);
+    }
+  } catch (_) {}
+
+  // ── Section 3: Token spend from token-usage.jsonl ─────────────────────────
+  try {
+    let tokenPath;
+    if (planningPaths) {
+      tokenPath = planningPaths.resolveWithFallback(cwd, 'token-usage');
+    } else {
+      tokenPath = path.join(cwd, '.planning', 'telemetry', 'token-usage.jsonl');
+    }
+
+    if (fs.existsSync(tokenPath)) {
+      const lines = fs.readFileSync(tokenPath, 'utf8').split('\n').filter(l => l.trim());
+      report.token_spend.total_records = lines.length;
+      const bySlot = {};
+      for (const line of lines) {
+        try {
+          const r = JSON.parse(line);
+          const slot = r.slot || 'unknown';
+          if (!bySlot[slot]) bySlot[slot] = { input: 0, output: 0, rounds: 0 };
+          bySlot[slot].input += (r.input_tokens || 0);
+          bySlot[slot].output += (r.output_tokens || 0);
+          bySlot[slot].rounds++;
+          report.token_spend.total_input += (r.input_tokens || 0);
+          report.token_spend.total_output += (r.output_tokens || 0);
+        } catch (_) {}
+      }
+      report.token_spend.by_slot = bySlot;
+    }
+  } catch (_) {}
+
+  // ── Section 4: Stall events from stall-detector ──────────────────────────
+  try {
+    const stallDetector = findModule('stall-detector.cjs');
+    if (stallDetector && stallDetector.detectStalledSlots) {
+      let config = options.config;
+      if (!config) {
+        try {
+          const configLoader = tryRequire(path.join(__dirname, '..', 'hooks', 'config-loader'));
+          if (configLoader) config = configLoader.loadConfig(cwd);
+        } catch (_) {}
+      }
+      report.stall_events = stallDetector.detectStalledSlots(cwd, config || {}) || [];
+    }
+  } catch (_) {}
+
+  // ── Section 5: Circuit breaker status ─────────────────────────────────────
+  try {
+    const cbPath = path.join(cwd, '.claude', 'circuit-breaker-state.json');
+    if (fs.existsSync(cbPath)) {
+      const cb = JSON.parse(fs.readFileSync(cbPath, 'utf8'));
+      report.circuit_breaker.active = !!cb.active;
+      report.circuit_breaker.disabled = !!cb.disabled;
+      report.circuit_breaker.last_triggered = cb.last_triggered || cb.lastTriggered || null;
+    }
+  } catch (_) {}
+
+  // ── Section 6: Recommendations ────────────────────────────────────────────
+  const hasData = report.slot_availability.length > 0 ||
+                  report.pass_at_k.total > 0 ||
+                  report.token_spend.total_records > 0;
+
+  if (!hasData) {
+    report.recommendations.push('No quorum data available yet -- run quorum rounds to populate diagnostic data.');
+  } else {
+    // Slot health recommendations
+    for (const slot of report.slot_availability) {
+      if (slot.success_rate < 0.5 && slot.total_rounds > 0) {
+        const pct = Math.round(slot.success_rate * 100);
+        report.recommendations.push(
+          `Slot ${slot.slot} has ${pct}% success rate -- consider removing from quorum_active or checking provider health`
+        );
+      }
+    }
+
+    // Stall recommendations
+    for (const stall of report.stall_events) {
+      if ((stall.consecutiveTimeouts || 0) >= 2) {
+        report.recommendations.push(
+          `Slot ${stall.slot} has ${stall.consecutiveTimeouts} consecutive timeouts -- run \`node bin/check-mcp-health.cjs\``
+        );
+      }
+    }
+
+    // Pass@k recommendations
+    if (report.pass_at_k.total > 0 && report.pass_at_k.pass_at_1 < 0.7) {
+      const pct = Math.round(report.pass_at_k.pass_at_1 * 100);
+      report.recommendations.push(
+        `pass@1 rate at ${pct}% -- quorum requires multiple rounds frequently. Consider adjusting maxDeliberation.`
+      );
+    }
+
+    // Circuit breaker recommendations
+    if (report.circuit_breaker.active) {
+      report.recommendations.push(
+        'Circuit breaker is ACTIVE -- oscillation detected. Follow resolution procedure.'
+      );
+    }
+
+    if (report.recommendations.length === 0) {
+      report.recommendations.push('No issues detected. Harness is healthy.');
+    }
+  }
+
+  return report;
+}
+
+// ─── formatTerminalReport ───────────────────────────────────────────────────
+
+/**
+ * Format report object into a terminal-friendly string.
+ * @param {Object} report - Report from generateReport().
+ * @returns {string}
+ */
+function formatTerminalReport(report) {
+  const lines = [];
+
+  lines.push('nForma Harness Diagnostic Report');
+  lines.push('=================================');
+  lines.push(`Generated: ${report.timestamp}`);
+  lines.push('');
+
+  // Slot Availability
+  lines.push('## Slot Availability');
+  if (report.slot_availability.length === 0) {
+    lines.push('  No slot data available.');
+  } else {
+    lines.push('  slot           rounds   success   rate     status');
+    lines.push('  -----------------------------------------------');
+    for (const s of report.slot_availability) {
+      const rate = (s.success_rate * 100).toFixed(1) + '%';
+      lines.push(
+        '  ' +
+        s.slot.padEnd(15) +
+        String(s.total_rounds).padStart(6) +
+        String(s.successes).padStart(10) +
+        rate.padStart(9) +
+        '    ' + s.status
+      );
+    }
+  }
+  lines.push('');
+
+  // Pass@k
+  lines.push('## Pass@k Consensus Efficiency');
+  if (report.pass_at_k.total === 0) {
+    lines.push('  No pass@k data available.');
+  } else {
+    lines.push(`  pass@1:    ${(report.pass_at_k.pass_at_1 * 100).toFixed(1)}%`);
+    lines.push(`  pass@3:    ${(report.pass_at_k.pass_at_3 * 100).toFixed(1)}%`);
+    lines.push(`  avg rounds: ${report.pass_at_k.avg_k.toFixed(1)}`);
+    lines.push(`  total events: ${report.pass_at_k.total}`);
+  }
+  lines.push('');
+
+  // Token Spend
+  lines.push('## Token Spend');
+  lines.push('  Note: Token values unreliable (all zeros)');
+  lines.push(`  total records: ${report.token_spend.total_records}`);
+  lines.push('');
+
+  // Stall Events
+  lines.push('## Stall Events');
+  if (!report.stall_events || report.stall_events.length === 0) {
+    lines.push('  No stalled slots detected.');
+  } else {
+    for (const s of report.stall_events) {
+      lines.push(`  ${s.slot}: ${s.consecutiveTimeouts} consecutive timeouts (last: ${s.lastSeen || 'unknown'})`);
+    }
+  }
+  lines.push('');
+
+  // Circuit Breaker
+  lines.push('## Circuit Breaker');
+  if (report.circuit_breaker.active) {
+    lines.push('  Status: ACTIVE (oscillation detected)');
+  } else if (report.circuit_breaker.disabled) {
+    lines.push('  Status: disabled');
+  } else {
+    lines.push('  Status: inactive');
+  }
+  lines.push('');
+
+  // Recommendations
+  lines.push('## Recommendations');
+  for (const r of report.recommendations) {
+    lines.push(`  - ${r}`);
+  }
+
+  return lines.join('\n');
+}
+
+// ─── CLI ─────────────────────────────────────────────────────────────────────
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const jsonFlag = args.includes('--json');
+  const cwdIdx = args.indexOf('--cwd');
+  const cwd = cwdIdx >= 0 && args[cwdIdx + 1] ? args[cwdIdx + 1] : process.cwd();
+
+  const report = generateReport(cwd);
+
+  if (jsonFlag) {
+    process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+  } else {
+    process.stdout.write(formatTerminalReport(report) + '\n');
+  }
+
+  process.exit(0);
+}
+
+module.exports = { generateReport, formatTerminalReport };

package/bin/hazard-model.cjs

@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * hazard-model.cjs — FMEA hazard model for Layer 3 (Reasoning).
+ *
+ * Applies FMEA scoring (Severity x Occurrence x Detection = RPN) to every
+ * state-event pair in the L2 observed FSM. Outputs hazard-model.json with
+ * derived_from traceability links.
+ *
+ * Requirements: RSN-01
+ *
+ * Usage:
+ *   node bin/hazard-model.cjs            # print summary to stdout
+ *   node bin/hazard-model.cjs --json     # print full results JSON to stdout
+ */
+
+const fs   = require('fs');
+const path = require('path');
+
+const ROOT = process.env.PROJECT_ROOT || path.join(__dirname, '..');
+const FORMAL = path.join(ROOT, '.planning', 'formal');
+const REASONING_DIR = path.join(FORMAL, 'reasoning');
+const OUT_FILE = path.join(REASONING_DIR, 'hazard-model.json');
+
+const JSON_FLAG = process.argv.includes('--json');
+
+// ── Severity lookup table (domain judgment) ─────────────────────────────────
+
+/**
+ * Compute severity score (1-10) for a (fromState, event) pair.
+ * Based on domain impact if the transition fails.
+ */
+function computeSeverity(fromState, event) {
+  // DECIDED -> any self-loop: already terminal, harmless (check first — overrides event rules)
+  if (fromState === 'DECIDED') return 2;
+
+  // DELIBERATING -> DECIDE: wrong verdict = incorrect PASS/BLOCK
+  if (fromState === 'DELIBERATING' && event === 'DECIDE') return 8;
+
+  // COLLECTING_VOTES -> VOTES_COLLECTED: incomplete votes = stalled quorum
+  if (fromState === 'COLLECTING_VOTES' && event === 'VOTES_COLLECTED') return 6;
+
+  // Any -> CIRCUIT_BREAK: false positive break = workflow interruption
+  if (event === 'CIRCUIT_BREAK') return 6;
+
+  // IDLE -> QUORUM_START: start failure = degraded, can retry
+  if (fromState === 'IDLE' && event === 'QUORUM_START') return 4;
+
+  // Default: moderate
+  return 4;
+}
+
+// ── Occurrence computation from trace data ──────────────────────────────────
+
+/**
+ * Compute occurrence score (1-10) based on transition frequency.
+ * ratio = transition.count / totalSessions
+ */
+function computeOccurrenceScore(transitionCount, totalSessions) {
+  if (totalSessions === 0) return 1;
+  const ratio = transitionCount / totalSessions;
+  if (ratio > 0.8) return 10;
+  if (ratio > 0.5) return 8;
+  if (ratio > 0.2) return 6;
+  if (ratio > 0.05) return 4;
+  if (ratio > 0) return 2;
+  return 1;
+}
+
+// ── Detection computation from formalism + test coverage ────────────────────
+
+// Mapping of states/events to related requirement prefixes for formalism lookup
+const STATE_REQ_MAP = {
+  'IDLE':             ['STOP', 'SPEC'],
+  'COLLECTING_VOTES': ['PLAN', 'LOOP'],
+  'DELIBERATING':     ['PLAN', 'IMPR', 'LOOP'],
+  'DECIDED':          ['ORES', 'DETECT'],
+};
+
+const EVENT_REQ_MAP = {
+  'QUORUM_START':     ['PLAN', 'CRED'],
+  'VOTES_COLLECTED':  ['PLAN', 'LOOP'],
+  'DECIDE':           ['PLAN', 'IMPR', 'SAFE'],
+  'CIRCUIT_BREAK':    ['DETECT', 'ORES'],
+};
+
+/**
+ * Compute detection score (1-10) based on formalism and test coverage.
+ * - Has formalism AND tests: 2
+ * - Has formalism OR tests: 4
+ * - Neither (only conformance events): 8
+ * - Nothing at all: 10
+ */
+function computeDetectionScore(fromState, event, failureTaxonomy, unitTestCoverage) {
+  // Check if any formal check covers related requirements
+  const relatedPrefixes = [
+    ...(STATE_REQ_MAP[fromState] || []),
+    ...(EVENT_REQ_MAP[event] || []),
+  ];
+
+  const logicViolations = failureTaxonomy?.categories?.logic_violation || [];
+  const hasFormalism = logicViolations.some(f =>
+    (f.requirement_ids || []).some(rid =>
+      relatedPrefixes.some(pfx => rid.startsWith(pfx))
+    )
+  );
+
+  // Check if there are tests covering related requirement IDs
+  const coveredReqs = unitTestCoverage?.requirements || {};
+  const hasTests = relatedPrefixes.some(pfx =>
+    Object.keys(coveredReqs).some(rid => rid.startsWith(pfx) && coveredReqs[rid]?.covered)
+  );
+
+  if (hasFormalism && hasTests) return 2;
+  if (hasFormalism || hasTests) return 4;
+
+  // Check if at least conformance events cover this transition
+  // (all transitions in observed-fsm.json have conformance traces by definition)
+  return 8;
+}
+
+// ── Main generation ─────────────────────────────────────────────────────────
+
+function generateHazardModel(observedFsm, traceStats, failureTaxonomy, unitTestCoverage) {
+  const totalSessions = traceStats?.sessions?.length || 349;
+  const hazards = [];
+
+  for (const [fromState, events] of Object.entries(observedFsm.observed_transitions)) {
+    for (const [event, data] of Object.entries(events)) {
+      const severity = computeSeverity(fromState, event);
+      const occurrence = computeOccurrenceScore(data.count, totalSessions);
+      const detection = computeDetectionScore(fromState, event, failureTaxonomy, unitTestCoverage);
+      const rpn = severity * occurrence * detection;
+
+      // Build derived_from links
+      const derivedFrom = [
+        {
+          layer: 'L2',
+          artifact: 'semantics/observed-fsm.json',
+          ref: `observed_transitions.${fromState}.${event}`,
+        },
+        {
+          layer: 'L1',
+          artifact: 'evidence/trace-corpus-stats.json',
+          ref: `sessions[*].actions`,
+        },
+      ];
+
+      // Add invariant link if any invariant relates to this state
+      const invariantConfigs = {
+        'IDLE':             'MCQGSDQuorum',
+        'COLLECTING_VOTES': 'MCQGSDQuorum',
+        'DELIBERATING':     'MCdeliberation',
+        'DECIDED':          'MCQGSDQuorum',
+      };
+      const config = invariantConfigs[fromState];
+      if (config) {
+        derivedFrom.push({
+          layer: 'L2',
+          artifact: 'semantics/invariant-catalog.json',
+          ref: `invariants[config=${config}]`,
+        });
+      }
+
+      hazards.push({
+        id: `HAZARD-${fromState}-${event}`,
+        state: fromState,
+        event,
+        to_state: data.to_state,
+        severity,
+        occurrence,
+        detection,
+        rpn,
+        derived_from: derivedFrom,
+      });
+    }
+  }
+
+  // Sort by RPN descending
+  hazards.sort((a, b) => b.rpn - a.rpn);
+
+  const criticalCount = hazards.filter(h => h.rpn >= 200).length;
+  const highCount = hazards.filter(h => h.rpn >= 100 && h.rpn < 200).length;
+  const maxRpn = hazards.length > 0 ? hazards[0].rpn : 0;
+
+  return {
+    schema_version: '1',
+    generated: new Date().toISOString(),
+    methodology: 'FMEA (IEC 60812)',
+    scoring_scale: {
+      severity: '1-10: 1=no impact, 2=cosmetic, 4=degraded, 6=stalled/interrupted, 8=incorrect verdict, 10=crash/data loss',
+      occurrence: '1-10: based on transition count / total sessions (349). >80%=10, >50%=8, >20%=6, >5%=4, >0%=2, 0=1',
+      detection: '1-10: 2=formalism+tests, 4=formalism OR tests, 8=conformance events only, 10=nothing',
+      rpn: 'Severity x Occurrence x Detection (range 1-1000)',
+    },
+    hazards,
+    summary: {
+      total: hazards.length,
+      max_rpn: maxRpn,
+      critical_count: criticalCount,
+      high_count: highCount,
+    },
+  };
+}
+
+// ── Entry point ─────────────────────────────────────────────────────────────
+
+function main() {
+  // Load L2 observed FSM
+  const fsmPath = path.join(FORMAL, 'semantics', 'observed-fsm.json');
+  if (!fs.existsSync(fsmPath)) {
+    console.error('ERROR: observed-fsm.json not found at', fsmPath);
+    process.exit(1);
+  }
+  const observedFsm = JSON.parse(fs.readFileSync(fsmPath, 'utf8'));
+
+  // Load L1 trace corpus stats
+  const tracePath = path.join(FORMAL, 'evidence', 'trace-corpus-stats.json');
+  let traceStats = { sessions: [] };
+  if (fs.existsSync(tracePath)) {
+    traceStats = JSON.parse(fs.readFileSync(tracePath, 'utf8'));
+  }
+
+  // Load failure taxonomy
+  const taxPath = path.join(FORMAL, 'evidence', 'failure-taxonomy.json');
+  let failureTaxonomy = { categories: { logic_violation: [] } };
+  if (fs.existsSync(taxPath)) {
+    failureTaxonomy = JSON.parse(fs.readFileSync(taxPath, 'utf8'));
+  }
+
+  // Load unit test coverage
+  const covPath = path.join(FORMAL, 'unit-test-coverage.json');
+  let unitTestCoverage = { requirements: {} };
+  if (fs.existsSync(covPath)) {
+    unitTestCoverage = JSON.parse(fs.readFileSync(covPath, 'utf8'));
+  }
+
+  const output = generateHazardModel(observedFsm, traceStats, failureTaxonomy, unitTestCoverage);
+
+  // Write output
+  fs.mkdirSync(REASONING_DIR, { recursive: true });
+  fs.writeFileSync(OUT_FILE, JSON.stringify(output, null, 2) + '\n');
+
+  if (JSON_FLAG) {
+    process.stdout.write(JSON.stringify(output));
+  } else {
+    console.log(`Hazard Model (FMEA)`);
+    console.log(`  Total hazards: ${output.summary.total}`);
+    console.log(`  Max RPN: ${output.summary.max_rpn}`);
+    console.log(`  Critical (RPN>=200): ${output.summary.critical_count}`);
+    console.log(`  High (100<=RPN<200): ${output.summary.high_count}`);
+    console.log(`  Output: ${OUT_FILE}`);
+  }
+
+  process.exit(0);
+}
+
+if (require.main === module) main();
+
+module.exports = { computeSeverity, computeOccurrenceScore, computeDetectionScore, generateHazardModel, main };

package/bin/install-formal-tools.cjs

@@ -4,7 +4,7 @@
 /**
  * install-formal-tools.cjs
  *
- * Cross-platform installer for QGSD formal verification tools:
+ * Cross-platform installer for nForma formal verification tools:
  *   TLA+    — downloads tla2tools.jar into .planning/formal/tla/
  *   Alloy   — downloads org.alloytools.alloy.dist.jar into .planning/formal/alloy/
  *   PRISM   — downloads and installs platform-specific binary