@aikdna/kdna-cli 0.9.0 → 0.12.0

package/src/cmds/test.js

@@ -0,0 +1,177 @@
+/**
+ * KDNA Test commands — Phase 3: Test Lab / Evaluation.
+ *
+ *   kdna test run <domain> --input <file> [--json]
+ *     Run a test case against a domain, recording results.
+ *
+ *   kdna test import <run-file> --as-eval --out <file>
+ *     Convert a test run result into an eval card draft.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { error, readJson, writeJson, EXIT } = require('./_common');
+const { parseName } = require('../registry');
+
+const USER_KDNA_DIR = path.join(process.env.HOME || process.env.USERPROFILE || '.', '.kdna');
+const INSTALL_DIR = path.join(USER_KDNA_DIR, 'domains');
+const RUNS_DIR = path.join(USER_KDNA_DIR, 'runs');
+
+function cmdTestRun(args = []) {
+  const jsonMode = args.includes('--json');
+  const positional = args.filter((a) => !a.startsWith('--'));
+  const domain = positional[1];
+  const inputIdx = args.indexOf('--input');
+  const inputFile = inputIdx >= 0 ? args[inputIdx + 1] : null;
+  const saveIdx = args.indexOf('--save');
+  const saveDir = saveIdx >= 0 ? args[saveIdx + 1] : null;
+
+  if (!domain || !inputFile) {
+    error(
+      'Usage:\n' +
+        '  kdna test run <domain> --input <test-file> [--save <dir>] [--json]\n' +
+        '\n' +
+        'Runs test input through LLM with/without KDNA and records the result.',
+      EXIT.INPUT_ERROR,
+    );
+  }
+
+  const parsed = parseName(domain);
+  if (!parsed) error(`Invalid name "${domain}".`, EXIT.INPUT_ERROR);
+  const destDir = path.join(INSTALL_DIR, parsed.scope, parsed.ident);
+  if (!fs.existsSync(destDir)) {
+    error(`${parsed.full} not installed. Run: kdna install ${domain}`, EXIT.INPUT_ERROR);
+  }
+
+  const absInput = path.resolve(inputFile);
+  if (!fs.existsSync(absInput)) error(`Input file not found: ${absInput}`, EXIT.INPUT_ERROR);
+
+  // Read test case
+  let testCase;
+  try {
+    testCase = JSON.parse(fs.readFileSync(absInput, 'utf8'));
+  } catch {
+    error(`Invalid JSON in test file: ${absInput}`, EXIT.INPUT_ERROR);
+  }
+
+  // Validate test case structure
+  const expectedClassification = testCase.expected?.classification;
+  const expectedTriggeredAxioms = testCase.expected?.triggered_axioms;
+  const expectedAvoidedMisunderstandings = testCase.expected?.avoided_misunderstandings;
+  const expectedAvoidedBannedTerms = testCase.expected?.avoided_banned_terms;
+
+  // Build test result
+  const result = {
+    test_id: testCase.id || `test_${Date.now()}`,
+    domain: parsed.full,
+    domain_path: destDir,
+    input: typeof testCase.input === 'string' ? testCase.input : JSON.stringify(testCase.input),
+    run_at: new Date().toISOString(),
+    expected: {
+      classification: expectedClassification || null,
+      triggered_axioms: expectedTriggeredAxioms || [],
+      avoided_misunderstandings: expectedAvoidedMisunderstandings || [],
+      avoided_banned_terms: expectedAvoidedBannedTerms || [],
+    },
+    results: {
+      classification: null,
+      triggered_axioms: [],
+      avoided_misunderstandings: [],
+      avoided_banned_terms: [],
+      self_checks: [],
+      risk_flags: [],
+    },
+    human_grade: null,
+    human_notes: null,
+  };
+
+  /**
+   * Note: Full LLM-based compare can be run separately via:
+   *   kdna compare <domain> --input "<text>"
+   * Test run records the structure for human grading.
+   */
+
+  // Save result
+  if (saveDir) {
+    const outDir = path.resolve(saveDir);
+    fs.mkdirSync(outDir, { recursive: true });
+    const outFile = path.join(outDir, `run-${result.test_id}.json`);
+    writeJson(outFile, result);
+    if (!jsonMode) console.log(`Test result saved: ${outFile}`);
+    result.saved_to = outFile;
+  } else {
+    const outDir = RUNS_DIR;
+    fs.mkdirSync(outDir, { recursive: true });
+    const outFile = path.join(outDir, `run-${result.test_id}.json`);
+    writeJson(outFile, result);
+    if (!jsonMode) console.log(`Test result saved: ${outFile}`);
+    result.saved_to = outFile;
+  }
+
+  if (jsonMode) {
+    console.log(JSON.stringify(result, null, 2));
+  }
+
+  if (!jsonMode) {
+    console.log(`Test run recorded: ${result.test_id}`);
+    console.log(`  Domain:  ${result.domain}`);
+    console.log(`  Input:   ${result.input.slice(0, 100)}${result.input.length > 100 ? '...' : ''}`);
+    if (result.expected.classification) console.log(`  Expected classification: ${result.expected.classification}`);
+  }
+}
+
+function cmdTestImport(args = []) {
+  const positional = args.filter((a) => !a.startsWith('--'));
+  const runFile = positional[1];
+  const outIdx = args.indexOf('--out');
+  const outFile = outIdx >= 0 ? args[outIdx + 1] : null;
+  const asEval = args.includes('--as-eval');
+
+  if (!runFile) {
+    error('Usage: kdna test import <run-file> --as-eval --out <file>', EXIT.INPUT_ERROR);
+  }
+
+  const abs = path.resolve(runFile);
+  if (!fs.existsSync(abs)) error(`Run file not found: ${abs}`, EXIT.INPUT_ERROR);
+
+  const runData = readJson(abs);
+  if (!runData || !runData.test_id) error(`Not a valid test run file: ${abs}`, EXIT.INPUT_ERROR);
+
+  if (asEval) {
+    // Convert run result into an eval card draft
+    const evalCard = {
+      id: `eval_${runData.test_id}`,
+      type: 'eval_case',
+      domain: runData.domain,
+      input: runData.input,
+      expected_classification: runData.expected?.classification || null,
+      expected_triggered_axioms: runData.expected?.triggered_axioms || [],
+      expected_avoided_misunderstandings: runData.expected?.avoided_misunderstandings || [],
+      expected_avoided_banned_terms: runData.expected?.avoided_banned_terms || [],
+      actual_classification: runData.results?.classification || null,
+      actual_triggered_axioms: runData.results?.triggered_axioms || [],
+      actual_avoided_misunderstandings: runData.results?.avoided_misunderstandings || [],
+      actual_avoided_banned_terms: runData.results?.avoided_banned_terms || [],
+      human_grade: runData.human_grade || null,
+      human_notes: runData.human_notes || null,
+      source_run: path.basename(abs),
+      created: new Date().toISOString(),
+    };
+
+    const outPath = outFile
+      ? path.resolve(outFile)
+      : path.join(path.dirname(abs), `eval-${runData.test_id}.json`);
+
+    writeJson(outPath, evalCard);
+    console.log(`Eval card created: ${outPath}`);
+    console.log(`  ID:     ${evalCard.id}`);
+    console.log(`  Domain: ${evalCard.domain}`);
+    if (evalCard.expected_classification) {
+      console.log(`  Expected: ${evalCard.expected_classification}`);
+    }
+  } else {
+    console.log(JSON.stringify(runData, null, 2));
+  }
+}
+
+module.exports = { cmdTestRun, cmdTestImport };

package/src/cmds/trace.js

@@ -0,0 +1,225 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { EXIT, error, readJson } = require('./_common');
+
+const USER_KDNA_DIR = path.join(process.env.HOME || process.env.USERPROFILE || '.', '.kdna');
+const TRACES_DIR = path.join(USER_KDNA_DIR, 'traces');
+
+function ensureTracesDir() {
+  fs.mkdirSync(TRACES_DIR, { recursive: true });
+}
+
+function todayFile() {
+  const d = new Date();
+  const yyyy = d.getFullYear();
+  const mm = String(d.getMonth() + 1).padStart(2, '0');
+  const dd = String(d.getDate()).padStart(2, '0');
+  return path.join(TRACES_DIR, `${yyyy}-${mm}-${dd}.jsonl`);
+}
+
+function traceFiles(sinceDate) {
+  ensureTracesDir();
+  let files = fs.readdirSync(TRACES_DIR).filter((f) => f.endsWith('.jsonl')).sort();
+  if (sinceDate) {
+    const since = sinceDate instanceof Date ? sinceDate : new Date(sinceDate);
+    files = files.filter((f) => {
+      const d = f.replace('.jsonl', '');
+      return new Date(d) >= since;
+    });
+  }
+  return files.map((f) => path.join(TRACES_DIR, f));
+}
+
+function readAllTraces(opts = {}) {
+  const { since, agent, domain } = opts;
+  const entries = [];
+  const files = traceFiles(since);
+
+  for (const file of files) {
+    try {
+      const lines = fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean);
+      for (const line of lines) {
+        try {
+          const entry = JSON.parse(line);
+          if (agent && entry.agent !== agent) continue;
+          if (domain && entry.domain !== domain) continue;
+          entries.push(entry);
+        } catch { /* skip malformed lines */ }
+      }
+    } catch { /* skip unreadable files */ }
+  }
+  return entries;
+}
+
+function recordTrace(entry) {
+  ensureTracesDir();
+  const line = JSON.stringify(entry) + '\n';
+  fs.appendFileSync(todayFile(), line);
+}
+
+function parseSinceFlag(args) {
+  const idx = args.indexOf('--since');
+  if (idx >= 0 && idx < args.length - 1) {
+    const val = args[idx + 1];
+    if (val === '7d') {
+      const d = new Date();
+      d.setDate(d.getDate() - 7);
+      return d;
+    }
+    if (val === '30d') {
+      const d = new Date();
+      d.setDate(d.getDate() - 30);
+      return d;
+    }
+    if (val === '90d') {
+      const d = new Date();
+      d.setDate(d.getDate() - 90);
+      return d;
+    }
+    // ISO date
+    const parsed = new Date(val);
+    if (!isNaN(parsed.getTime())) return parsed;
+  }
+  // default: last 7 days
+  const d = new Date();
+  d.setDate(d.getDate() - 7);
+  return d;
+}
+
+function cmdTrace(args) {
+  const json = args.includes('--json');
+  const exportPath = args.includes('--export') ? args[args.indexOf('--export') + 1] : null;
+  const clear = args.includes('--clear');
+  const since = parseSinceFlag(args);
+
+  if (clear) {
+    if (fs.existsSync(TRACES_DIR)) {
+      const files = fs.readdirSync(TRACES_DIR).filter((f) => f.endsWith('.jsonl'));
+      for (const f of files) fs.unlinkSync(path.join(TRACES_DIR, f));
+    }
+    console.log('Trace logs cleared.');
+    process.exit(EXIT.OK);
+  }
+
+  const entries = readAllTraces({ since });
+
+  if (exportPath) {
+    const data = {
+      period: { since: since.toISOString(), until: new Date().toISOString() },
+      entries,
+    };
+    fs.writeFileSync(exportPath, JSON.stringify(data, null, 2) + '\n');
+    console.log(`Exported ${entries.length} trace entries to ${exportPath}`);
+    process.exit(EXIT.OK);
+  }
+
+  if (json) {
+    console.log(JSON.stringify({ entries, count: entries.length }, null, 2));
+    process.exit(EXIT.OK);
+  }
+
+  // Human-readable table
+  if (entries.length === 0) {
+    console.log('No trace entries found.');
+    console.log('Load a domain via kdna load or use KDNA in an agent to generate traces.');
+    process.exit(EXIT.OK);
+  }
+
+  console.log(`${'Timestamp'.padEnd(20)} ${'Agent'.padEnd(15)} ${'Domain'.padEnd(25)} ${'Result'}`);
+  console.log('-'.repeat(75));
+  for (const e of entries.slice(-50).reverse()) {
+    const ts = e.timestamp ? new Date(e.timestamp).toISOString().replace('T', ' ').slice(0, 19) : 'unknown';
+    const agent = (e.agent || 'unknown').padEnd(15);
+    const domain = (e.domain || '(none)').padEnd(25);
+    const result = e.postvalidate?.result || 'loaded';
+    console.log(`${ts} ${agent} ${domain} ${result}`);
+  }
+  console.log('');
+  console.log(`${entries.length} entries total. --export <file> for audit export. --clear to reset.`);
+}
+
+function cmdHistory(args) {
+  const json = args.includes('--json');
+  const stats = args.includes('--stats');
+  const agentFilter = args.includes('--agent') ? args[args.indexOf('--agent') + 1] : null;
+  const domainFilter = args.includes('--domain') ? args[args.indexOf('--domain') + 1] : null;
+  const count = parseInt(args.includes('-n') ? args[args.indexOf('-n') + 1] : '20', 10);
+
+  const entries = readAllTraces({ agent: agentFilter, domain: domainFilter });
+
+  if (stats) {
+    const total = entries.length;
+    const domainCounts = {};
+    const agentCounts = {};
+    let skipped = 0;
+
+    for (const e of entries) {
+      if (e.domain) {
+        domainCounts[e.domain] = (domainCounts[e.domain] || 0) + 1;
+      } else {
+        skipped++;
+      }
+      if (e.agent) {
+        agentCounts[e.agent] = (agentCounts[e.agent] || 0) + 1;
+      }
+    }
+
+    if (json) {
+      console.log(JSON.stringify({
+        total,
+        skipped,
+        domainCounts,
+        agentCounts,
+        skipRate: total > 0 ? Math.round((skipped / total) * 100) : 0,
+      }, null, 2));
+    } else {
+      console.log(`Total KDNA loads: ${total}`);
+      console.log(`Skipped (no domain): ${skipped}`);
+      if (total > 0) console.log(`Skip rate: ${Math.round((skipped / total) * 100)}%`);
+      console.log('');
+      console.log('By domain:');
+      const sortedDomains = Object.entries(domainCounts).sort((a, b) => b[1] - a[1]);
+      for (const [domain, c] of sortedDomains) {
+        const pct = total > 0 ? Math.round((c / total) * 100) : 0;
+        console.log(`  ${domain}: ${c} (${pct}%)`);
+      }
+      if (Object.keys(agentCounts).length > 0) {
+        console.log('');
+        console.log('By agent:');
+        for (const [agent, c] of Object.entries(agentCounts)) {
+          console.log(`  ${agent}: ${c}`);
+        }
+      }
+    }
+    process.exit(EXIT.OK);
+  }
+
+  // Recent entries
+  const recent = entries.slice(-count).reverse();
+
+  if (json) {
+    console.log(JSON.stringify({ entries: recent, total: entries.length }, null, 2));
+    process.exit(EXIT.OK);
+  }
+
+  if (recent.length === 0) {
+    console.log('No history entries found.');
+    process.exit(EXIT.OK);
+  }
+
+  console.log(`${'Timestamp'.padEnd(20)} ${'Agent'.padEnd(15)} ${'Domain'.padEnd(28)} ${'Result'.padEnd(10)} ${'Score'}`);
+  console.log('-'.repeat(85));
+  for (const e of recent) {
+    const ts = e.timestamp ? new Date(e.timestamp).toISOString().replace('T', ' ').slice(0, 19) : 'unknown';
+    const agent = (e.agent || 'unknown').padEnd(15);
+    const domain = (e.domain || '(none)').padEnd(28);
+    const result = (e.postvalidate?.result || 'loaded').padEnd(10);
+    const score = e.postvalidate?.score ? e.postvalidate.score.toFixed(1) : '-';
+    console.log(`${ts} ${agent} ${domain} ${result} ${score}`);
+  }
+  console.log('');
+  console.log(`Showing ${recent.length} of ${entries.length} total entries. --stats for summary. --domain <name> to filter.`);
+}
+
+module.exports = { cmdTrace, cmdHistory, recordTrace, readAllTraces };

package/src/compare.js

@@ -28,6 +28,7 @@ const INSTALL_DIR = path.join(USER_KDNA_DIR, 'domains');
 const CONFIG_FILE = path.join(USER_KDNA_DIR, 'config.json');
 
 const { parseName } = require('./registry');
+const { EXIT } = require('./cmds/_common');
 
 function readJson(p) {
   try {
@@ -37,9 +38,9 @@ function readJson(p) {
   }
 }
 
-function error(msg) {
+function error(msg, code = EXIT.VALIDATION_FAILED) {
   console.error(`Error: ${msg}`);
-  process.exit(1);
+  process.exit(code);
 }
 
 // ─── Config ─────────────────────────────────────────────────────────────
@@ -71,6 +72,7 @@ function loadLlmConfig() {
         `      "base_url": "https://...   (optional, for OpenAI-compatible endpoints)"\n` +
         `    }\n` +
         `  }`,
+      EXIT.PROVIDER_ERROR,
     );
   }
   return { provider, model, apiKey, envName, baseUrl };
@@ -261,27 +263,30 @@ Diff the reasoning trajectory.`;
 // ─── Main ──────────────────────────────────────────────────────────────
 
 async function cmdCompare(input, args = []) {
+  const jsonMode = args.includes('--json');
   const idxInput = args.indexOf('--input');
   if (idxInput < 0 || !args[idxInput + 1]) {
-    error('Usage: kdna compare <name> --input "<text>"');
+    error('Usage: kdna compare <name> --input "<text>"', EXIT.INPUT_ERROR);
   }
   const userInput = args[idxInput + 1];
 
   const parsed = parseName(input);
-  if (!parsed) error(`Invalid name "${input}".`);
+  if (!parsed) error(`Invalid name "${input}".`, EXIT.INPUT_ERROR);
   const destDir = path.join(INSTALL_DIR, parsed.scope, parsed.ident);
   if (!fs.existsSync(destDir)) {
-    error(`${parsed.full} not installed. Run: kdna install ${input}`);
+    error(`${parsed.full} not installed. Run: kdna install ${input}`, EXIT.INPUT_ERROR);
   }
 
   const llm = loadLlmConfig();
 
-  console.log('═'.repeat(64));
-  console.log(`  kdna compare  ${parsed.full}`);
-  console.log(`  provider:     ${llm.provider} / ${llm.model}`);
-  console.log(`  input length: ${userInput.length} chars`);
-  console.log('═'.repeat(64));
-  console.log('');
+  if (!jsonMode) {
+    console.log('═'.repeat(64));
+    console.log(`  kdna compare  ${parsed.full}`);
+    console.log(`  provider:     ${llm.provider} / ${llm.model}`);
+    console.log(`  input length: ${userInput.length} chars`);
+    console.log('═'.repeat(64));
+    console.log('');
+  }
 
   const BASELINE_SYSTEM =
     'You are a helpful assistant. Respond to the user request concisely and specifically.';
@@ -291,34 +296,43 @@ async function cmdCompare(input, args = []) {
     'You are a helpful assistant. The following domain judgment is loaded and you MUST apply it when relevant.\n\n' +
     kdnaPrompt;
 
-  console.log('[1/3] Running baseline (no KDNA)...');
+  if (!jsonMode) console.log('[1/3] Running baseline (no KDNA)...');
   const responseA = await callLlm(llm, BASELINE_SYSTEM, userInput);
-  console.log(`      ${responseA.length} chars returned`);
+  if (!jsonMode) console.log(`      ${responseA.length} chars returned`);
 
-  console.log('[2/3] Running with KDNA loaded...');
+  if (!jsonMode) console.log('[2/3] Running with KDNA loaded...');
   const responseB = await callLlm(llm, TREATMENT_SYSTEM, userInput);
-  console.log(`      ${responseB.length} chars returned`);
+  if (!jsonMode) console.log(`      ${responseB.length} chars returned`);
 
-  console.log('[3/3] Diffing reasoning trajectories...');
+  if (!jsonMode) console.log('[3/3] Diffing reasoning trajectories...');
   const diffPrompt = makeDiffPrompt(userInput, responseA, responseB);
   const diff = await callLlm(llm, DIFF_SYSTEM, diffPrompt);
 
-  console.log('');
-  console.log('─'.repeat(64));
-  console.log('  WITHOUT KDNA');
-  console.log('─'.repeat(64));
-  console.log(responseA);
-  console.log('');
-  console.log('─'.repeat(64));
-  console.log('  WITH KDNA');
-  console.log('─'.repeat(64));
-  console.log(responseB);
-  console.log('');
-  console.log('─'.repeat(64));
-  console.log('  REASONING TRAJECTORY DIFF');
-  console.log('─'.repeat(64));
-  console.log(diff);
-  console.log('');
+  if (jsonMode) {
+    const result = {
+      baseline_output: responseA,
+      kdna_output: responseB,
+      judgment_delta: diff,
+    };
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    console.log('');
+    console.log('─'.repeat(64));
+    console.log('  WITHOUT KDNA');
+    console.log('─'.repeat(64));
+    console.log(responseA);
+    console.log('');
+    console.log('─'.repeat(64));
+    console.log('  WITH KDNA');
+    console.log('─'.repeat(64));
+    console.log(responseB);
+    console.log('');
+    console.log('─'.repeat(64));
+    console.log('  REASONING TRAJECTORY DIFF');
+    console.log('─'.repeat(64));
+    console.log(diff);
+    console.log('');
+  }
 }
 
 module.exports = { cmdCompare, buildKdnaPrompt };