@nforma.ai/nforma 0.2.1 → 0.29.0

package/bin/detect-coverage-gaps.cjs

@@ -5,7 +5,7 @@
 // Requirements: SIG-01
 //
 // Usage:
-//   node bin/detect-coverage-gaps.cjs [--spec=QGSDQuorum] [--log=path]
+//   node bin/detect-coverage-gaps.cjs [--spec=NFQuorum] [--log=path]
 //
 // Output:
 //   .planning/formal/coverage-gaps.md — structured test backlog when gaps exist
@@ -19,15 +19,15 @@ const path = require('path');
 // Maps each TLA+ spec to its state variable name and value-to-name mapping.
 // Source of truth: state comments in the TLA+ spec files.
 const STATE_MAPS = {
-  'QGSDQuorum': {
+  'NFQuorum': {
     variable: 's',
     values: { '0': 'COLLECTING_VOTES', '1': 'DECIDED', '2': 'DELIBERATING' },
   },
-  'QGSDStopHook': {
+  'NFStopHook': {
     variable: 'phase',
     values: { '0': 'IDLE', '1': 'READING', '2': 'DECIDING', '3': 'BLOCKED' },
   },
-  'QGSDCircuitBreaker': {
+  'NFCircuitBreaker': {
     variable: 'state',
     values: { '0': 'MONITORING', '1': 'TRIGGERED', '2': 'RECOVERING' },
   },
@@ -51,7 +51,7 @@ const ACTION_TO_STATE = {
 
 /**
  * parseTlcStates(specName) — returns the full set of named states for a spec.
- * @param {string} specName - TLA+ spec name (e.g. 'QGSDQuorum')
+ * @param {string} specName - TLA+ spec name (e.g. 'NFQuorum')
  * @returns {{ specName: string, states: Set<string>, variable: string } | null}
  */
 function parseTlcStates(specName) {
@@ -102,7 +102,7 @@ function parseTraceStates(logPath) {
  * @returns {{ status: string, gaps?: string[], outputPath?: string, reason?: string }}
  */
 function detectCoverageGaps(options = {}) {
-  const specName   = options.specName || 'QGSDQuorum';
+  const specName   = options.specName || 'NFQuorum';
   const pp2 = require('./planning-paths.cjs');
   const logPath    = options.logPath || pp2.resolveWithFallback(process.cwd(), 'conformance-events');
   const outputPath = options.outputPath || path.join(process.cwd(), '.planning', 'formal', 'coverage-gaps.md');
@@ -186,7 +186,7 @@ if (require.main === module) {
   const specArg = args.find(a => a.startsWith('--spec='));
   const logArg  = args.find(a => a.startsWith('--log='));
 
-  const specName = specArg ? specArg.split('=')[1] : 'QGSDQuorum';
+  const specName = specArg ? specArg.split('=')[1] : 'NFQuorum';
   const logPath  = logArg ? logArg.split('=')[1] : undefined;
 
   const result = detectCoverageGaps({ specName, logPath });

package/bin/failure-mode-catalog.cjs

@@ -0,0 +1,227 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * failure-mode-catalog.cjs — Failure mode enumeration for Layer 3 (Reasoning).
+ *
+ * Enumerates concrete failure modes (omission, commission, corruption) per
+ * L2 state-event pair. Uses hazard-model.json severity scores for corruption
+ * thresholds and observed-fsm.json model_comparison for commission conditions.
+ *
+ * Requirements: RSN-02
+ *
+ * Usage:
+ *   node bin/failure-mode-catalog.cjs            # print summary to stdout
+ *   node bin/failure-mode-catalog.cjs --json     # print full results JSON to stdout
+ */
+
+const fs   = require('fs');
+const path = require('path');
+
+const ROOT = process.env.PROJECT_ROOT || path.join(__dirname, '..');
+const FORMAL = path.join(ROOT, '.planning', 'formal');
+const REASONING_DIR = path.join(FORMAL, 'reasoning');
+const OUT_FILE = path.join(REASONING_DIR, 'failure-mode-catalog.json');
+
+const JSON_FLAG = process.argv.includes('--json');
+
+// ── Severity class mapping ──────────────────────────────────────────────────
+
+function classifySeverity(severity, mode) {
+  if (mode === 'commission') return 'model_gap';
+  if (severity >= 8) return 'critical';
+  if (severity >= 6) return 'stalled';
+  if (severity >= 4) return 'degraded';
+  if (severity >= 2) return 'cosmetic';
+  return 'cosmetic';
+}
+
+// ── Failure mode descriptions ───────────────────────────────────────────────
+
+function omissionDescription(fromState, event, toState) {
+  return `Transition ${fromState} --[${event}]--> ${toState} does not fire when expected`;
+}
+
+function omissionEffect(fromState) {
+  return `System stays in ${fromState}; expected state change does not occur`;
+}
+
+function commissionDescription(fromState, event, toState) {
+  return `Transition ${fromState} --[${event}]--> ${toState} fires but is not modeled in XState`;
+}
+
+function commissionEffect(fromState, event, toState) {
+  return `Unmodeled state transition from ${fromState} to ${toState} on ${event}; behavior diverges from spec`;
+}
+
+function corruptionDescription(fromState, event, toState) {
+  return `Transition ${fromState} --[${event}]--> fires but produces wrong target state (not ${toState})`;
+}
+
+function corruptionEffect(fromState, toState) {
+  return `System enters incorrect state instead of ${toState}; downstream behavior undefined`;
+}
+
+// ── Mismatch enrichment ─────────────────────────────────────────────────────
+
+function findMismatches(fromState, event, mismatches) {
+  return mismatches.filter(m => {
+    // Mismatches have expected/actual states; match if the event context aligns
+    // Since mismatches don't have from/event fields directly, we match on state overlap
+    return m.expected_state === fromState || m.actual_state === fromState;
+  });
+}
+
+// ── Core enumeration ────────────────────────────────────────────────────────
+
+function enumerateFailureModes(observedFsm, hazardModel, mismatches) {
+  const missingInModel = new Set(
+    (observedFsm.model_comparison?.missing_in_model || [])
+      .map(m => `${m.from}-${m.event}`)
+  );
+
+  // Build severity lookup from hazard model
+  const severityMap = {};
+  for (const h of (hazardModel?.hazards || [])) {
+    severityMap[`${h.state}-${h.event}`] = h.severity;
+  }
+
+  const failureModes = [];
+
+  for (const [fromState, events] of Object.entries(observedFsm.observed_transitions)) {
+    for (const [event, data] of Object.entries(events)) {
+      const key = `${fromState}-${event}`;
+      const severity = severityMap[key] || 4;
+      const toState = data.to_state;
+      const relMismatches = findMismatches(fromState, event, mismatches);
+      const mismatchNote = relMismatches.length > 0
+        ? ` (${relMismatches.length} observed mismatch(es): ${relMismatches.map(m => m.id).join(', ')})`
+        : '';
+
+      // 1. Omission (always)
+      failureModes.push({
+        id: `FM-${fromState}-${event}-OMISSION`,
+        state: fromState,
+        event,
+        to_state: toState,
+        failure_mode: 'omission',
+        description: omissionDescription(fromState, event, toState) + mismatchNote,
+        effect: omissionEffect(fromState),
+        severity_class: classifySeverity(severity, 'omission'),
+        derived_from: [
+          { layer: 'L2', artifact: 'semantics/observed-fsm.json', ref: `observed_transitions.${fromState}.${event}` },
+          { layer: 'L3', artifact: 'reasoning/hazard-model.json', ref: `hazards[id=HAZARD-${fromState}-${event}]` },
+        ],
+      });
+
+      // 2. Commission (conditional: only if in missing_in_model)
+      if (missingInModel.has(key)) {
+        failureModes.push({
+          id: `FM-${fromState}-${event}-COMMISSION`,
+          state: fromState,
+          event,
+          to_state: toState,
+          failure_mode: 'commission',
+          description: commissionDescription(fromState, event, toState) + mismatchNote,
+          effect: commissionEffect(fromState, event, toState),
+          severity_class: classifySeverity(severity, 'commission'),
+          derived_from: [
+            { layer: 'L2', artifact: 'semantics/observed-fsm.json', ref: `model_comparison.missing_in_model` },
+            { layer: 'L2', artifact: 'semantics/observed-fsm.json', ref: `observed_transitions.${fromState}.${event}` },
+            { layer: 'L3', artifact: 'reasoning/hazard-model.json', ref: `hazards[id=HAZARD-${fromState}-${event}]` },
+          ],
+        });
+      }
+
+      // 3. Corruption (conditional: only if severity >= 6)
+      if (severity >= 6) {
+        failureModes.push({
+          id: `FM-${fromState}-${event}-CORRUPTION`,
+          state: fromState,
+          event,
+          to_state: toState,
+          failure_mode: 'corruption',
+          description: corruptionDescription(fromState, event, toState) + mismatchNote,
+          effect: corruptionEffect(fromState, toState),
+          severity_class: classifySeverity(severity, 'corruption'),
+          derived_from: [
+            { layer: 'L2', artifact: 'semantics/observed-fsm.json', ref: `observed_transitions.${fromState}.${event}` },
+            { layer: 'L3', artifact: 'reasoning/hazard-model.json', ref: `hazards[id=HAZARD-${fromState}-${event}]` },
+          ],
+        });
+      }
+    }
+  }
+
+  // Count by mode and severity class
+  const byMode = { omission: 0, commission: 0, corruption: 0 };
+  const bySeverityClass = {};
+  for (const fm of failureModes) {
+    byMode[fm.failure_mode] = (byMode[fm.failure_mode] || 0) + 1;
+    bySeverityClass[fm.severity_class] = (bySeverityClass[fm.severity_class] || 0) + 1;
+  }
+
+  return {
+    schema_version: '1',
+    generated: new Date().toISOString(),
+    failure_modes: failureModes,
+    summary: {
+      total: failureModes.length,
+      by_mode: byMode,
+      by_severity_class: bySeverityClass,
+    },
+  };
+}
+
+// ── Entry point ─────────────────────────────────────────────────────────────
+
+function main() {
+  // Load L2 observed FSM
+  const fsmPath = path.join(FORMAL, 'semantics', 'observed-fsm.json');
+  if (!fs.existsSync(fsmPath)) {
+    console.error('ERROR: observed-fsm.json not found at', fsmPath);
+    process.exit(1);
+  }
+  const observedFsm = JSON.parse(fs.readFileSync(fsmPath, 'utf8'));
+
+  // Load L3 hazard model (produced by hazard-model.cjs)
+  const hazardPath = path.join(REASONING_DIR, 'hazard-model.json');
+  if (!fs.existsSync(hazardPath)) {
+    console.error('ERROR: hazard-model.json not found at', hazardPath);
+    console.error('Run bin/hazard-model.cjs first.');
+    process.exit(1);
+  }
+  const hazardModel = JSON.parse(fs.readFileSync(hazardPath, 'utf8'));
+
+  // Load mismatch register
+  const mismatchPath = path.join(FORMAL, 'semantics', 'mismatch-register.jsonl');
+  let mismatches = [];
+  if (fs.existsSync(mismatchPath)) {
+    mismatches = fs.readFileSync(mismatchPath, 'utf8')
+      .trim().split('\n')
+      .filter(Boolean)
+      .map(line => JSON.parse(line));
+  }
+
+  const output = enumerateFailureModes(observedFsm, hazardModel, mismatches);
+
+  // Write output
+  fs.mkdirSync(REASONING_DIR, { recursive: true });
+  fs.writeFileSync(OUT_FILE, JSON.stringify(output, null, 2) + '\n');
+
+  if (JSON_FLAG) {
+    process.stdout.write(JSON.stringify(output));
+  } else {
+    console.log(`Failure Mode Catalog`);
+    console.log(`  Total failure modes: ${output.summary.total}`);
+    console.log(`  By mode: ${JSON.stringify(output.summary.by_mode)}`);
+    console.log(`  By severity class: ${JSON.stringify(output.summary.by_severity_class)}`);
+    console.log(`  Output: ${OUT_FILE}`);
+  }
+
+  process.exit(0);
+}
+
+if (require.main === module) main();
+
+module.exports = { enumerateFailureModes, classifySeverity };

package/bin/failure-taxonomy.cjs

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+'use strict';
+// bin/failure-taxonomy.cjs
+// Classifies check-result failures into 5 categories:
+// crash, timeout, logic_violation, drift, degradation
+//
+// Requirement: EVID-03
+
+const fs   = require('fs');
+const path = require('path');
+
+const ROOT = process.env.PROJECT_ROOT || path.join(__dirname, '..');
+const EVIDENCE_DIR = path.join(ROOT, '.planning', 'formal', 'evidence');
+const CHECK_RESULTS_PATH = path.join(ROOT, '.planning', 'formal', 'check-results.ndjson');
+const DEBT_PATH = path.join(ROOT, '.planning', 'formal', 'debt.json');
+const OUTPUT_PATH = path.join(EVIDENCE_DIR, 'failure-taxonomy.json');
+
+const JSON_FLAG = process.argv.includes('--json');
+
+// Timeout threshold in ms (> 60 seconds suggests TLC state explosion)
+const TIMEOUT_THRESHOLD_MS = 60000;
+
+// ── Classification rules ────────────────────────────────────────────────────
+
+/**
+ * Classify a check-result failure into exactly one category.
+ * Decision rules per research Pitfall 5:
+ *
+ * - crash: formalism tool itself crashed (non-zero exit, no structured result, stack trace)
+ * - timeout: runtime_ms > threshold AND result=fail (TLC state explosion, model checker timeout)
+ * - logic_violation: TLC/Alloy counterexample, assertion failure
+ * - drift: validate-traces reports divergence (formalism "trace" with divergence count)
+ * - degradation: metrics trending worse (reserved; falls back to logic_violation when no baseline)
+ */
+function classifyFailure(entry) {
+  // Check for crash indicators
+  if (entry.metadata && entry.metadata.stack_trace) {
+    return { category: 'crash', reason: 'Stack trace present in metadata' };
+  }
+  if (entry.summary && /error|crash|exception|ENOENT|spawn/i.test(entry.summary) &&
+      !/counterexample|divergence|fail:/i.test(entry.summary)) {
+    return { category: 'crash', reason: `Error pattern in summary: ${entry.summary.substring(0, 80)}` };
+  }
+
+  // Check for timeout
+  if (entry.runtime_ms && entry.runtime_ms > TIMEOUT_THRESHOLD_MS) {
+    return { category: 'timeout', reason: `runtime_ms=${entry.runtime_ms} exceeds ${TIMEOUT_THRESHOLD_MS}ms threshold` };
+  }
+
+  // Check for drift (trace formalism with divergences)
+  if (entry.formalism === 'trace') {
+    const divMatch = entry.summary && entry.summary.match(/(\d+)\s+divergence/);
+    if (divMatch) {
+      return { category: 'drift', reason: `Trace divergence: ${divMatch[1]} divergence(s) detected` };
+    }
+    return { category: 'drift', reason: 'Trace formalism failure (drift)' };
+  }
+
+  // Degradation: reserved, falls back to logic_violation when no baseline
+  // (no baseline data exists in current codebase)
+
+  // Default: logic_violation (counterexample, assertion failure, etc.)
+  return { category: 'logic_violation', reason: entry.summary
+    ? `Logic violation: ${entry.summary.substring(0, 100)}`
+    : 'Logic violation: check-result failure without specific categorization' };
+}
+
+// ── Main ────────────────────────────────────────────────────────────────────
+
+function main() {
+  if (!fs.existsSync(EVIDENCE_DIR)) {
+    fs.mkdirSync(EVIDENCE_DIR, { recursive: true });
+  }
+
+  // Read check-results.ndjson
+  const failures = [];
+
+  if (fs.existsSync(CHECK_RESULTS_PATH)) {
+    const lines = fs.readFileSync(CHECK_RESULTS_PATH, 'utf8').split('\n').filter(l => l.trim());
+    for (const line of lines) {
+      try {
+        const entry = JSON.parse(line);
+        if (entry.result === 'fail') {
+          failures.push(entry);
+        }
+      } catch (_) {}
+    }
+  }
+
+  // Read debt.json for additional failure sources
+  if (fs.existsSync(DEBT_PATH)) {
+    try {
+      const debt = JSON.parse(fs.readFileSync(DEBT_PATH, 'utf8'));
+      if (Array.isArray(debt.debt_entries)) {
+        for (const entry of debt.debt_entries) {
+          failures.push({
+            ...entry,
+            result: 'fail',
+            formalism: entry.formalism || 'unknown',
+            tool: entry.tool || 'debt-entry',
+            summary: entry.summary || entry.description || 'Debt entry',
+            timestamp: entry.timestamp || debt.last_updated,
+            _source: 'debt.json',
+          });
+        }
+      }
+    } catch (_) {}
+  }
+
+  // Classify each failure
+  const categories = {
+    crash: [],
+    timeout: [],
+    logic_violation: [],
+    drift: [],
+    degradation: [],
+  };
+  const unclassified = [];
+
+  for (const failure of failures) {
+    const { category, reason } = classifyFailure(failure);
+    const classified = { ...failure, category, classification_reason: reason };
+
+    if (category in categories) {
+      categories[category].push(classified);
+    } else {
+      unclassified.push(classified);
+    }
+  }
+
+  // Count per category
+  const categoryCounts = {};
+  for (const [cat, entries] of Object.entries(categories)) {
+    categoryCounts[cat] = entries.length;
+  }
+
+  // Build result
+  const result = {
+    schema_version: '1',
+    generated: new Date().toISOString(),
+    total_failures: failures.length,
+    categories,
+    category_counts: categoryCounts,
+    degradation_fallback_note: 'When no baseline exists, potential degradation entries are classified as logic_violation',
+    unclassified,
+    summary: `${failures.length} failures classified: ` +
+             Object.entries(categoryCounts).map(([k, v]) => `${k}=${v}`).join(', ') +
+             `. ${unclassified.length} unclassified.`,
+  };
+
+  fs.writeFileSync(OUTPUT_PATH, JSON.stringify(result, null, 2) + '\n', 'utf8');
+
+  if (JSON_FLAG) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    console.log('Failure Taxonomy Generated');
+    console.log(`  Total failures: ${failures.length}`);
+    for (const [cat, count] of Object.entries(categoryCounts)) {
+      console.log(`  ${cat}: ${count}`);
+    }
+    if (unclassified.length > 0) {
+      console.log(`  UNCLASSIFIED: ${unclassified.length}`);
+    }
+  }
+
+  // Exit 1 if any unclassified
+  if (unclassified.length > 0) {
+    process.exit(1);
+  }
+}
+
+// Export for testing
+module.exports = { classifyFailure, TIMEOUT_THRESHOLD_MS };
+
+if (require.main === module) {
+  main();
+}

package/bin/formal-scope-scan.cjs

@@ -0,0 +1,179 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SPEC_DIR = path.join(process.cwd(), '.planning', 'formal', 'spec');
+
+function printHelp() {
+  console.log(`Usage: node bin/formal-scope-scan.cjs --description "text" [options]
+
+Options:
+  --description "text"   Description to match against (required)
+  --files file1,file2    Source files to check for overlap (optional)
+  --format json|lines    Output format (default: json)
+  --help                 Show this help message
+
+Matching algorithm (any signal fires):
+  1. Source file overlap: --files matched against module source_files globs
+  2. Concept matching: exact token match against curated concepts
+  3. Module name match: exact token match against module directory name
+
+Examples:
+  node bin/formal-scope-scan.cjs --description "fix quorum deliberation bug"
+  node bin/formal-scope-scan.cjs --description "update breaker" --format lines
+  node bin/formal-scope-scan.cjs --files "hooks/nf-stop.js" --description "something"
+`);
+}
+
+function parseArgs(argv) {
+  const args = { description: '', files: [], format: 'json', help: false };
+  for (let i = 2; i < argv.length; i++) {
+    if (argv[i] === '--help' || argv[i] === '-h') {
+      args.help = true;
+    } else if (argv[i] === '--description' && argv[i + 1]) {
+      args.description = argv[++i];
+    } else if (argv[i] === '--files' && argv[i + 1]) {
+      args.files = argv[++i].split(',').map(f => f.trim()).filter(Boolean);
+    } else if (argv[i] === '--format' && argv[i + 1]) {
+      args.format = argv[++i];
+    }
+  }
+  return args;
+}
+
+function globToRegex(glob) {
+  let regex = '';
+  let i = 0;
+  while (i < glob.length) {
+    if (glob[i] === '*' && glob[i + 1] === '*') {
+      regex += '.*';
+      i += 2;
+      if (glob[i] === '/') i++; // skip trailing slash after **
+    } else if (glob[i] === '*') {
+      regex += '[^/]*';
+      i++;
+    } else if (glob[i] === '?') {
+      regex += '[^/]';
+      i++;
+    } else if ('.+^${}()|[]\\'.includes(glob[i])) {
+      regex += '\\' + glob[i];
+      i++;
+    } else {
+      regex += glob[i];
+      i++;
+    }
+  }
+  return new RegExp('^' + regex + '$');
+}
+
+function matchesSourceFiles(providedFiles, moduleSourceFiles) {
+  for (const pf of providedFiles) {
+    for (const sf of moduleSourceFiles) {
+      const re = globToRegex(sf);
+      if (re.test(pf)) return true;
+    }
+  }
+  return false;
+}
+
+function matchesConcepts(descLower, tokens, concepts) {
+  for (const concept of concepts) {
+    const conceptLower = concept.toLowerCase();
+    // Exact token match
+    if (tokens.includes(conceptLower)) return true;
+    // Multi-word concept substring match against raw description
+    if (conceptLower.includes('-') || conceptLower.includes(' ')) {
+      if (descLower.includes(conceptLower)) return true;
+    }
+  }
+  return false;
+}
+
+function matchesModuleName(tokens, moduleName) {
+  return tokens.includes(moduleName.toLowerCase());
+}
+
+function main() {
+  const args = parseArgs(process.argv);
+
+  if (args.help) {
+    printHelp();
+    process.exit(0);
+  }
+
+  if (!args.description) {
+    console.error('Error: --description is required');
+    process.exit(1);
+  }
+
+  // Fail-open: if spec dir doesn't exist, output empty
+  if (!fs.existsSync(SPEC_DIR)) {
+    if (args.format === 'lines') {
+      // no output
+    } else {
+      console.log('[]');
+    }
+    process.exit(0);
+  }
+
+  const descLower = args.description.toLowerCase();
+  const tokens = descLower.split(/[\s\-_]+/).filter(t => t.length > 0);
+
+  const modules = fs.readdirSync(SPEC_DIR, { withFileTypes: true })
+    .filter(d => d.isDirectory())
+    .map(d => d.name);
+
+  const matches = [];
+
+  for (const mod of modules) {
+    const scopePath = path.join(SPEC_DIR, mod, 'scope.json');
+    if (!fs.existsSync(scopePath)) {
+      process.stderr.write(`Warning: ${scopePath} not found, skipping module ${mod}\n`);
+      continue;
+    }
+
+    let scope;
+    try {
+      scope = JSON.parse(fs.readFileSync(scopePath, 'utf8'));
+    } catch (e) {
+      process.stderr.write(`Warning: Failed to parse ${scopePath}: ${e.message}\n`);
+      continue;
+    }
+
+    const invariantsPath = `.planning/formal/spec/${mod}/invariants.md`;
+    let matchedBy = null;
+
+    // Priority 1: Source file overlap
+    if (args.files.length > 0 && scope.source_files && matchesSourceFiles(args.files, scope.source_files)) {
+      matchedBy = 'source_file';
+    }
+
+    // Priority 2: Concept matching
+    if (!matchedBy && scope.concepts && matchesConcepts(descLower, tokens, scope.concepts)) {
+      matchedBy = 'concept';
+    }
+
+    // Priority 3: Module name match (exact token only)
+    if (!matchedBy && matchesModuleName(tokens, mod)) {
+      matchedBy = 'module_name';
+    }
+
+    if (matchedBy) {
+      matches.push({ module: mod, path: invariantsPath, matched_by: matchedBy });
+    }
+  }
+
+  if (args.format === 'lines') {
+    for (const m of matches) {
+      console.log(`${m.module}\t${m.path}`);
+    }
+  } else {
+    console.log(JSON.stringify(matches, null, 2));
+  }
+
+  process.exit(0);
+}
+
+main();