@nforma.ai/nforma 0.2.1

package/bin/verify-quorum-health.cjs

@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+'use strict';
+// bin/verify-quorum-health.cjs
+// Verifies that the XState machine's maxDeliberation is calibrated for the
+// actual empirical reliability of the quorum agents.
+//
+// The formal PRISM model uses conservative priors (tp=0.85, unavail=0.15).
+// This tool substitutes real scoreboard rates and recomputes:
+//   - P(majority per round)  — joint probability across all agents
+//   - Expected rounds         — average rounds to decide
+//   - P(within MaxDeliberation rounds) — actual confidence for the current setting
+//   - Recommended MaxDeliberation for target confidence
+//
+// Exits 1 if actual P(within MaxDeliberation) < TARGET_CONFIDENCE.
+// This makes it a CI gate: if agents degrade, the build breaks and flags it.
+//
+// Usage:
+//   node bin/verify-quorum-health.cjs              # target: 95%
+//   node bin/verify-quorum-health.cjs --target=0.99
+//   node bin/verify-quorum-health.cjs --auto-apply # auto-applies maxDeliberation if below target
+
+const fs   = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+
+// ── Pure Functions ────────────────────────────────────────────────────────────
+
+// suggestMaxDeliberation(pPerRound, targetConfidence)
+// Computes the recommended maxDeliberation value using the geometric distribution formula:
+//   k = ceil(log(1 - targetConfidence) / log(1 - pPerRound))
+function suggestMaxDeliberation(pPerRound, targetConfidence) {
+  if (pPerRound <= 0) return Infinity;
+  if (pPerRound >= 1) return 1;
+  return Math.ceil(Math.log(1 - targetConfidence) / Math.log(1 - pPerRound));
+}
+
+// computeRates(slot, rounds)
+// Computes per-agent rates from a list of rounds.
+function computeRates(slot, rounds) {
+  const SLOTS = ['gemini', 'opencode', 'copilot', 'codex'];
+  const MIN_ROUNDS = 30;
+  const TP_PRIOR = 0.85;
+  const UNAVAIL_PRIOR = 0.15;
+
+  // Exclude Mode A rounds (empty-string result) and UNAVAILABLE typo variant —
+  // neither carries a valid binary signal for rate calculation.
+  const relevant = rounds.filter(r => {
+    const v = r.votes && r.votes[slot];
+    return v !== undefined && v !== '' && v !== 'UNAVAILABLE';
+  });
+  const n = relevant.length;
+  if (n < MIN_ROUNDS) {
+    return { n, tpRate: TP_PRIOR, unavailRate: UNAVAIL_PRIOR, usedPrior: true };
+  }
+  const approvals = relevant.filter(r => r.votes[slot] === 'TP' || r.votes[slot] === 'TP+').length;
+  const unavails  = relevant.filter(r => r.votes[slot] === 'UNAVAIL').length;
+  return {
+    n,
+    tpRate:      approvals / n,
+    unavailRate: unavails  / n,
+    usedPrior:   false,
+  };
+}
+
+// pMajorityExternal(agents)
+// Computes P(at least 2 of the 4 external agents approve) using inclusion-exclusion.
+function pMajorityExternal(agents) {
+  const names = Object.keys(agents);
+  const n = names.length;
+  let pAtLeast2 = 0;
+
+  for (let mask = 0; mask < (1 << n); mask++) {
+    const selected = names.filter((_, i) => mask & (1 << i));
+    if (selected.length < 2) continue;
+    const prob = names.reduce((acc, name, i) =>
+      acc * ((mask & (1 << i)) ? agents[name].pApprove : (1 - agents[name].pApprove))
+    , 1);
+    pAtLeast2 += prob;
+  }
+  return pAtLeast2;
+}
+
+// applyMaxDeliberationUpdate(newValue, options)
+// Atomically updates maxDeliberation in the XState machine and config.json,
+// then regenerates formal specs. Returns { success, newValue, machineUpdated, configUpdated, specsRegenerated, error, rolledBack }
+function applyMaxDeliberationUpdate(newValue, options = {}) {
+  const machineFile = options.machineFile || path.join(ROOT, 'src', 'machines', 'qgsd-workflow.machine.ts');
+  const configFile = options.configFile || path.join(ROOT, '.planning', 'config.json');
+  const skipSpecGen = options.skipSpecGen || false; // For testing without generate-formal-specs.cjs
+
+  // Backup originals for rollback
+  const backups = {};
+  const results = { machineUpdated: false, configUpdated: false, specsRegenerated: false };
+
+  try {
+    // 1. Update machine.ts
+    const machineSrc = fs.readFileSync(machineFile, 'utf8');
+    backups.machine = { path: machineFile, content: machineSrc };
+    const updated = machineSrc.replace(/maxDeliberation:\s*\d+/, 'maxDeliberation:    ' + newValue);
+    fs.writeFileSync(machineFile, updated, 'utf8');
+    results.machineUpdated = true;
+
+    // 2. Update config.json if it exists
+    if (fs.existsSync(configFile)) {
+      const configSrc = fs.readFileSync(configFile, 'utf8');
+      backups.config = { path: configFile, content: configSrc };
+      const config = JSON.parse(configSrc);
+      if (!config.workflow) config.workflow = {};
+      config.workflow.maxDeliberation = newValue;
+      fs.writeFileSync(configFile, JSON.stringify(config, null, 2) + '\n', 'utf8');
+      results.configUpdated = true;
+    }
+
+    // 3. Regenerate formal specs (unless skipped for testing)
+    if (!skipSpecGen) {
+      const { spawnSync } = require('child_process');
+      const genResult = spawnSync('node', ['bin/generate-formal-specs.cjs'], { cwd: ROOT, timeout: 30000 });
+      if (genResult.error || genResult.status !== 0) throw new Error('Formal spec generation failed');
+      results.specsRegenerated = true;
+
+      // 4. Verify spec sync
+      const syncResult = spawnSync('node', ['bin/check-spec-sync.cjs'], { cwd: ROOT, timeout: 10000 });
+      if (syncResult.error || syncResult.status !== 0) throw new Error('Spec sync verification failed');
+    }
+
+    return { success: true, newValue, ...results };
+
+  } catch (err) {
+    // Rollback all backups
+    for (const backup of Object.values(backups)) {
+      try { fs.writeFileSync(backup.path, backup.content, 'utf8'); } catch (_) {}
+    }
+    return { success: false, error: err.message, rolledBack: true };
+  }
+}
+
+// ── CLI Handler ───────────────────────────────────────────────────────────────
+
+function main() {
+  // ── Config ────────────────────────────────────────────────────────────────────
+  const DEFAULT_TARGET = 0.95;
+  const targetArg = process.argv.find(a => a.startsWith('--target='));
+  const TARGET_CONFIDENCE = targetArg ? parseFloat(targetArg.split('=')[1]) : DEFAULT_TARGET;
+
+  // ── Load XState machine (source of truth for MaxDeliberation) ─────────────────
+  const machineSrc = fs.readFileSync(
+    path.join(ROOT, 'src', 'machines', 'qgsd-workflow.machine.ts'), 'utf8'
+  );
+  const maxDelibMatch = machineSrc.match(/maxDeliberation:\s*(\d+)/);
+  const maxDelib = maxDelibMatch ? parseInt(maxDelibMatch[1], 10) : null;
+  if (!maxDelib) {
+    process.stderr.write('[verify-quorum-health] Cannot read maxDeliberation from XState machine.\n');
+    process.exit(1);
+  }
+
+  // ── Load scoreboard ────────────────────────────────────────────────────────────
+  let scoreboardPath;
+  try {
+    const pp = require('./planning-paths.cjs');
+    scoreboardPath = pp.resolveWithFallback(ROOT, 'quorum-scoreboard');
+  } catch (_) {
+    scoreboardPath = path.join(ROOT, '.planning', 'quorum-scoreboard.json');
+  }
+  if (!fs.existsSync(scoreboardPath)) {
+    process.stderr.write('[verify-quorum-health] No scoreboard found — cannot compute empirical rates.\n');
+    process.stderr.write('[verify-quorum-health] Run some quorum rounds first.\n');
+    process.exit(1);
+  }
+  const scoreboard = JSON.parse(fs.readFileSync(scoreboardPath, 'utf8'));
+  const rounds = scoreboard.rounds || [];
+
+  // ── Compute per-agent rates ───────────────────────────────────────────────────
+  const SLOTS = ['gemini', 'opencode', 'copilot', 'codex'];
+
+  const agentRates = {};
+  for (const slot of SLOTS) {
+    const r = computeRates(slot, rounds);
+    agentRates[slot] = {
+      ...r,
+      pApprove: r.tpRate * (1 - r.unavailRate),  // P(agent votes APPROVE in one round)
+    };
+  }
+
+  const pPerRound  = pMajorityExternal(agentRates);
+  const expected   = 1 / pPerRound;
+  const pWithinK   = k => 1 - Math.pow(1 - pPerRound, k);
+  const kForTarget = suggestMaxDeliberation(pPerRound, TARGET_CONFIDENCE);
+  const pActual    = pWithinK(maxDelib);
+
+  // ── Conservative-prior comparison ────────────────────────────────────────────
+  const P_PRIOR_PER_ROUND = 0.85 * (1 - 0.15);  // 0.7225 — what quorum.props assumed
+  const pPriorWithinK = k => 1 - Math.pow(1 - P_PRIOR_PER_ROUND, k);
+
+  // ── Report ────────────────────────────────────────────────────────────────────
+  process.stdout.write('\n[verify-quorum-health] QGSD Quorum Reliability Report\n');
+  process.stdout.write('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n');
+
+  process.stdout.write('Per-agent effective approval probability (tp_rate × availability):\n');
+  for (const [slot, r] of Object.entries(agentRates)) {
+    const flag = r.usedPrior ? ' [prior]' : ' [empirical, n=' + r.n + ']';
+    process.stdout.write(
+      '  ' + slot.padEnd(10) + 'tp=' + r.tpRate.toFixed(4) +
+      '  unavail=' + r.unavailRate.toFixed(4) +
+      '  p_approve=' + r.pApprove.toFixed(4) + flag + '\n'
+    );
+  }
+
+  process.stdout.write('\nJoint majority probability (P(≥2 of 4 external approve | Claude always approves)):\n');
+  process.stdout.write('  Conservative priors: p_round = ' + P_PRIOR_PER_ROUND.toFixed(4) + '\n');
+  process.stdout.write('  Empirical rates:     p_round = ' + pPerRound.toFixed(4) + '\n');
+
+  process.stdout.write('\nConvergence analysis with maxDeliberation=' + maxDelib + ' (from XState machine):\n');
+  process.stdout.write('  Expected rounds (empirical): ' + expected.toFixed(2) + '\n');
+  process.stdout.write('  P(decide within ' + maxDelib + ' rounds):\n');
+  process.stdout.write('    Prior assumption:    ' + (pPriorWithinK(maxDelib) * 100).toFixed(1) + '% (spec was designed for this)\n');
+  process.stdout.write('    Empirical reality:   ' + (pActual * 100).toFixed(1) + '%  ← actual system\n');
+
+  const gap = pPriorWithinK(maxDelib) - pActual;
+  if (gap > 0.01) {
+    process.stdout.write('    Gap: -' + (gap * 100).toFixed(1) + ' percentage points from designed confidence\n');
+  }
+
+  process.stdout.write('\nRecommended maxDeliberation for target confidence levels:\n');
+  for (const [label, t] of [['90%', 0.90], ['95%', 0.95], ['99%', 0.99]]) {
+    const k = suggestMaxDeliberation(pPerRound, t);
+    const mark = t === TARGET_CONFIDENCE ? ' ← target' : '';
+    process.stdout.write('  ' + label + ' confidence: maxDeliberation = ' + k + mark + '\n');
+  }
+
+  process.stdout.write('\n');
+
+  // ── Gate ──────────────────────────────────────────────────────────────────────
+  const pass = pActual >= TARGET_CONFIDENCE;
+
+  if (pass) {
+    process.stdout.write('✓ PASS  P(within ' + maxDelib + ' rounds) = ' + (pActual * 100).toFixed(1) + '% ≥ ' + (TARGET_CONFIDENCE * 100).toFixed(0) + '% target\n\n');
+  } else {
+    process.stderr.write(
+      '✗ FAIL  P(within ' + maxDelib + ' rounds) = ' + (pActual * 100).toFixed(1) + '% < ' + (TARGET_CONFIDENCE * 100).toFixed(0) + '% target\n' +
+      '        Recommended: update maxDeliberation to ' + kForTarget + '\n'
+    );
+
+    // ── Handle --auto-apply flag ───────────────────────────────────────────
+    const autoApply = process.argv.includes('--auto-apply');
+    if (autoApply) {
+      process.stderr.write('Applying maxDeliberation = ' + kForTarget + ' (--auto-apply)...\n');
+      const result = applyMaxDeliberationUpdate(kForTarget);
+      if (result.success) {
+        process.stderr.write('Applied successfully. Machine: ' + result.machineUpdated +
+          ', Config: ' + result.configUpdated + ', Specs: ' + result.specsRegenerated + '\n');
+        process.exit(0); // Success after auto-apply
+      } else {
+        process.stderr.write('Auto-apply failed: ' + result.error + ' (rolled back)\n');
+        process.exit(1);
+      }
+    } else {
+      process.stderr.write('        Run with --auto-apply to apply automatically, or manually update:\n' +
+        '        src/machines/qgsd-workflow.machine.ts and re-run: node bin/generate-formal-specs.cjs && node bin/check-spec-sync.cjs\n\n');
+      process.exit(1);
+    }
+  }
+}
+
+// ── Exports ───────────────────────────────────────────────────────────────────
+
+module.exports = { suggestMaxDeliberation, applyMaxDeliberationUpdate, computeRates, pMajorityExternal };
+
+// ── CLI Execution ─────────────────────────────────────────────────────────────
+
+if (require.main === module) {
+  main();
+}

package/bin/write-check-result.cjs

@@ -0,0 +1,106 @@
+'use strict';
+
+const fs   = require('fs');
+const path = require('path');
+
+const VALID_RESULTS    = ['pass', 'fail', 'warn', 'inconclusive'];
+const VALID_FORMALISMS = ['tla', 'alloy', 'prism', 'trace', 'redaction', 'uppaal'];
+
+/**
+ * Path to the NDJSON output file.
+ * Priority: CHECK_RESULTS_PATH (exact path) > CHECK_RESULTS_ROOT (root-relative) > __dirname fallback.
+ * Use CHECK_RESULTS_PATH env var to redirect in tests (avoids polluting real output).
+ * Use CHECK_RESULTS_ROOT env var when --project-root is set (writes to ROOT/.planning/formal/).
+ */
+const NDJSON_PATH = process.env.CHECK_RESULTS_PATH ||
+  (process.env.CHECK_RESULTS_ROOT
+    ? path.join(process.env.CHECK_RESULTS_ROOT, '.planning', 'formal', 'check-results.ndjson')
+    : path.join(__dirname, '..', '.planning', 'formal', 'check-results.ndjson'));
+
+/**
+ * Append one normalized check result line to .planning/formal/check-results.ndjson.
+ *
+ * @param {Object} entry
+ * @param {string} entry.tool       - Name of the tool/runner (e.g. 'run-tlc')
+ * @param {string} entry.formalism  - One of VALID_FORMALISMS
+ * @param {string} entry.result     - One of VALID_RESULTS
+ * @param {string} entry.check_id   - Unique step identifier (e.g. 'tla:quorum-safety'), required
+ * @param {string} entry.surface    - STEPS tool group (e.g. 'tla', 'alloy', 'prism', 'ci'), required
+ * @param {string} entry.property   - Human-readable check description, required
+ * @param {number} entry.runtime_ms - Wall-clock elapsed milliseconds from tool start to this call, required. Stored as Math.round(runtime_ms) integer.
+ * @param {string} entry.summary    - One-line outcome (e.g. 'pass: MCsafety in 1823ms'), required
+ * @param {string[]} [entry.triage_tags] - Optional anomaly tags. Defaults to [].
+ * @param {object} [entry.observation_window] - Optional stochastic check window metadata (PRISM-critical). Contains { window_start, window_end, n_traces, n_events, window_days }.
+ * @param {string[]} [entry.requirement_ids] - Optional array of requirement IDs this check covers. Defaults to [].
+ * @param {Object} [entry.metadata] - Optional extra fields (spec, config, etc.)
+ * @throws {Error} On validation failure
+ */
+function writeCheckResult(entry) {
+  if (!entry || typeof entry.tool !== 'string' || entry.tool.length === 0) {
+    throw new Error('[write-check-result] tool is required and must be a non-empty string');
+  }
+  if (!VALID_FORMALISMS.includes(entry.formalism)) {
+    throw new Error(
+      '[write-check-result] formalism must be one of: ' + VALID_FORMALISMS.join(', ') +
+      ' (got: ' + entry.formalism + ')'
+    );
+  }
+  if (!VALID_RESULTS.includes(entry.result)) {
+    throw new Error(
+      '[write-check-result] result must be one of: ' + VALID_RESULTS.join(', ') +
+      ' (got: ' + entry.result + ')'
+    );
+  }
+
+  // v2.1 required field validations
+  if (typeof entry.check_id !== 'string' || entry.check_id.length === 0) {
+    throw new Error('[write-check-result] check_id is required and must be a non-empty string');
+  }
+  if (typeof entry.surface !== 'string' || entry.surface.length === 0) {
+    throw new Error('[write-check-result] surface is required and must be a non-empty string');
+  }
+  if (typeof entry.property !== 'string' || entry.property.length === 0) {
+    throw new Error('[write-check-result] property is required and must be a non-empty string');
+  }
+  if (typeof entry.runtime_ms !== 'number') {
+    throw new Error('[write-check-result] runtime_ms is required and must be a number');
+  }
+  if (typeof entry.summary !== 'string' || entry.summary.length === 0) {
+    throw new Error('[write-check-result] summary is required and must be a non-empty string');
+  }
+
+  // requirement_ids: optional array of requirement IDs this check covers
+  if (entry.requirement_ids !== undefined) {
+    if (!Array.isArray(entry.requirement_ids)) {
+      throw new Error('[write-check-result] requirement_ids must be an array');
+    }
+    for (const id of entry.requirement_ids) {
+      if (typeof id !== 'string') {
+        throw new Error('[write-check-result] requirement_ids must contain only strings (got: ' + typeof id + ')');
+      }
+    }
+  }
+
+  const record = {
+    tool:       entry.tool,
+    formalism:  entry.formalism,
+    result:     entry.result,
+    timestamp:  new Date().toISOString(),
+    check_id:   entry.check_id,
+    surface:    entry.surface,
+    property:   entry.property,
+    runtime_ms: Math.round(entry.runtime_ms),
+    summary:    entry.summary,
+    triage_tags: entry.triage_tags || [],
+    requirement_ids: Array.isArray(entry.requirement_ids) ? entry.requirement_ids : [],
+    metadata:   entry.metadata || {},
+  };
+
+  if (entry.observation_window !== undefined) {
+    record.observation_window = entry.observation_window;
+  }
+
+  fs.appendFileSync(NDJSON_PATH, JSON.stringify(record) + '\n', 'utf8');
+}
+
+module.exports = { writeCheckResult, NDJSON_PATH, VALID_RESULTS, VALID_FORMALISMS };