chainlesschain 0.162.148 → 0.162.150

package/src/commands/eval.js

@@ -0,0 +1,255 @@
+/**
+ * `cc eval` — reliability eval harness (Phase 7). Runs a suite of self-checking
+ * coding tasks against the real agent and reports the task-success rate.
+ *
+ * Each task gets a fresh temp workspace; the agent is invoked headlessly
+ * (`cc agent -p` in that cwd with acceptEdits so it can write files) and the
+ * task's objective check() decides pass/fail. `--dry-run` swaps in a no-op agent
+ * so the harness + report can be exercised without a model.
+ */
+
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { spawn } from "child_process";
+import { runEvalSuite } from "../lib/eval/runner.js";
+import { getSuite } from "../lib/eval/tasks.js";
+import {
+  TelemetryRecorder,
+  formatTelemetry,
+} from "../lib/telemetry/span-recorder.js";
+import { computeTrend, formatTrend } from "../lib/eval/trend.js";
+
+/** Append a run summary as one JSONL line to the history file. */
+function appendHistory(file, record) {
+  fs.appendFileSync(file, JSON.stringify(record) + "\n", "utf8");
+}
+
+/** Read a JSONL eval-history file into an array of run records (skips bad lines). */
+function readHistory(file) {
+  if (!fs.existsSync(file)) return [];
+  const out = [];
+  for (const line of fs.readFileSync(file, "utf8").split(/\r?\n/)) {
+    const t = line.trim();
+    if (!t) continue;
+    try {
+      out.push(JSON.parse(t));
+    } catch {
+      /* skip a corrupt history line rather than abort the report */
+    }
+  }
+  return out;
+}
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+// This file is src/commands/eval.js → the CLI entry is bin/chainlesschain.js.
+const BIN = path.resolve(__dirname, "..", "..", "bin", "chainlesschain.js");
+
+/**
+ * Build a runAgent that shells out to a headless `cc agent -p` in the task's
+ * workspace. acceptEdits lets it write files without an interactive prompt.
+ */
+function makeHeadlessRunAgent(opts = {}) {
+  return function runAgent({ prompt, cwd, timeoutMs }) {
+    return new Promise((resolve) => {
+      const args = [
+        BIN,
+        "agent",
+        "-p",
+        prompt,
+        "--permission-mode",
+        "acceptEdits",
+        "--output-format",
+        "text",
+      ];
+      if (opts.model) args.push("--model", opts.model);
+      if (opts.provider) args.push("--provider", opts.provider);
+      const child = spawn(process.execPath, args, {
+        cwd,
+        env: { ...process.env, CLAUDECODE: "1" },
+        windowsHide: true,
+      });
+      let out = "";
+      let err = "";
+      let settled = false;
+      const done = (result) => {
+        if (settled) return;
+        settled = true;
+        resolve(result);
+      };
+      const timer = setTimeout(() => {
+        try {
+          child.kill("SIGTERM");
+        } catch {
+          /* already gone */
+        }
+        done({
+          ok: false,
+          output: out,
+          error: `timed out after ${timeoutMs || 120000}ms`,
+        });
+      }, timeoutMs || 120000);
+      if (typeof timer.unref === "function") timer.unref();
+      child.stdout?.on("data", (d) => (out += d.toString("utf8")));
+      child.stderr?.on("data", (d) => (err += d.toString("utf8")));
+      child.on("error", (e) => {
+        clearTimeout(timer);
+        done({ ok: false, output: out, error: e.message });
+      });
+      child.on("close", (code) => {
+        clearTimeout(timer);
+        done({ ok: code === 0, output: out, error: code === 0 ? null : err });
+      });
+    });
+  };
+}
+
+export function registerEvalCommand(program, { logger } = {}) {
+  const log = logger || console;
+  program
+    .command("eval")
+    .description(
+      "Run the reliability eval suite (self-checking coding tasks) and report the task-success rate",
+    )
+    .option("--suite <name>", "Suite to run", "builtin")
+    .option("--model <model>", "Model for the agent runs")
+    .option("--provider <provider>", "Provider for the agent runs")
+    .option("--json", "Output the summary as JSON")
+    .option("--keep", "Keep the per-task temp workspaces (debug)")
+    .option(
+      "--otlp <file>",
+      "Write OpenTelemetry (OTLP/JSON) spans for the run to a file",
+    )
+    .option(
+      "--dry-run",
+      "Use a no-op agent (exercise the harness/report without a model)",
+    )
+    .option(
+      "--history <file>",
+      "Append this run's summary (JSONL) for trend tracking",
+    )
+    .option(
+      "--label <label>",
+      "Tag this run in the history (e.g. a version/commit)",
+    )
+    .option(
+      "--trend",
+      "Report the pass-rate trend from --history instead of running (CI gate)",
+    )
+    .option(
+      "--regression-threshold <pct>",
+      "With --trend: pass-rate drop (in points) that fails the gate on its own",
+      "0",
+    )
+    .action(async (options) => {
+      // Trend mode: don't run the suite — read the recorded history and report
+      // the pass-rate trend + per-task regressions. This is the release-pipeline
+      // consumer (the runs themselves need a real model; the report is pure).
+      if (options.trend) {
+        if (!options.history) {
+          (log.error || console.error)("--trend requires --history <file>");
+          process.exitCode = 1;
+          return;
+        }
+        const runs = readHistory(options.history);
+        const threshold =
+          Math.max(0, Number(options.regressionThreshold) || 0) / 100;
+        const trend = computeTrend(runs, { regressionThreshold: threshold });
+        if (options.json) console.log(JSON.stringify(trend, null, 2));
+        else (log.log || console.log)(formatTrend(trend));
+        if (trend.regressed) process.exitCode = 1;
+        return;
+      }
+
+      let tasks;
+      try {
+        tasks = getSuite(options.suite);
+      } catch (err) {
+        log.error ? log.error(err.message) : console.error(err.message);
+        process.exitCode = 1;
+        return;
+      }
+      const runAgent = options.dryRun
+        ? async () => ({ ok: true, output: "(dry-run: no agent)" })
+        : makeHeadlessRunAgent({
+            model: options.model,
+            provider: options.provider,
+          });
+
+      // OTel-shaped telemetry for the run (per-task span + failure class).
+      const recorder = new TelemetryRecorder({ serviceName: "cc-eval" });
+
+      const summary = await runEvalSuite(tasks, {
+        runAgent,
+        recorder,
+        keepWorkspaces: options.keep === true,
+        onResult: options.json
+          ? undefined
+          : (r) =>
+              (log.info || console.log)(
+                `  ${r.pass ? "✔" : "✗"} ${r.id} (${r.ms}ms)` +
+                  (r.error
+                    ? ` — ${r.error}`
+                    : r.detail
+                      ? ` — ${r.detail}`
+                      : ""),
+              ),
+      });
+
+      if (options.otlp) {
+        try {
+          fs.writeFileSync(
+            options.otlp,
+            JSON.stringify(recorder.toOtlp(), null, 2),
+            "utf8",
+          );
+        } catch (e) {
+          (log.error || console.error)(`  otlp write failed: ${e.message}`);
+        }
+      }
+
+      // Append to the trend history (a compact record — pass counts + per-task
+      // pass/fail + a timestamp/label — so `cc eval --trend` can chart it).
+      if (options.history) {
+        try {
+          appendHistory(options.history, {
+            ranAt: new Date().toISOString(),
+            label: options.label || null,
+            passed: summary.passed,
+            total: summary.total,
+            passRate: summary.passRate,
+            unrelatedChangeRate: summary.unrelatedChangeRate,
+            results: (summary.results || []).map((r) => ({
+              id: r.id,
+              pass: r.pass,
+            })),
+          });
+        } catch (e) {
+          (log.error || console.error)(`  history write failed: ${e.message}`);
+        }
+      }
+
+      if (options.json) {
+        console.log(JSON.stringify(summary, null, 2));
+      } else {
+        // Per-task lines already streamed via onResult; print the summary +
+        // the telemetry metrics (durations / failure classification).
+        const pct = (summary.passRate * 100).toFixed(1);
+        (log.log || console.log)(
+          `\nEval: ${summary.passed}/${summary.total} passed (${pct}%) in ${summary.ms}ms`,
+        );
+        if (typeof summary.unrelatedChangeRate === "number") {
+          const upct = (summary.unrelatedChangeRate * 100).toFixed(1);
+          (log.log || console.log)(
+            `Unrelated-change rate: ${upct}% (${summary.tasksWithUnrelatedChanges} task(s) edited files outside their expected surface)`,
+          );
+        }
+        if (summary.telemetry) {
+          (log.log || console.log)("\n" + formatTelemetry(summary.telemetry));
+        }
+      }
+      // Non-zero exit when not every task passed — usable as a CI gate.
+      if (summary.passed < summary.total) process.exitCode = 1;
+    });
+  return program;
+}

package/src/commands/evomap.js

@@ -11,6 +11,7 @@ import { EvoMapClient } from "../lib/evomap-client.js";
 import { EvoMapManager } from "../lib/evomap-manager.js";
 import {
   ensureEvoMapFederationTables,
+  loadFromDb as loadFedFromDb,
   listFederatedHubs,
   addFederatedHub,
   syncGenes,
@@ -28,6 +29,7 @@ import {
 } from "../lib/evomap-federation.js";
 import {
   ensureEvoMapGovernanceTables,
+  loadFromDb as loadGovFromDb,
   registerOwnership,
   traceOwnership,
   createGovernanceProposal,
@@ -221,6 +223,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
 
         const hubs = listFederatedHubs(db, {
           status: options.status,
@@ -261,6 +264,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
 
         const hub = addFederatedHub(db, url, options.name, options.region);
         logger.success("Hub added");
@@ -288,6 +292,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
 
         const geneIds = options.geneIds
           ? options.geneIds.split(",").map((s) => s.trim())
@@ -317,6 +322,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
 
         const report = getPressureReport();
         if (options.json) {
@@ -354,6 +360,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
 
         const entry = recombineGenes(db, geneId1, geneId2);
         logger.success("Genes recombined");
@@ -389,6 +396,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
 
         const entries = getLineage(geneId);
         if (options.json) {
@@ -428,6 +436,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
 
         const result = registerOwnership(db, geneId, ownerDid);
         logger.success("Ownership registered");
@@ -455,6 +464,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
 
         const result = traceOwnership(geneId);
         if (options.json) {
@@ -490,6 +500,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
 
         const result = createGovernanceProposal(
           db,
@@ -522,6 +533,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
 
         const result = voteOnGovernanceProposal(
           db,
@@ -555,6 +567,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
 
         const result = getGovernanceDashboard();
         if (options.json) {
@@ -626,6 +639,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
         console.log(JSON.stringify(setHubStatus(db, hubId, status), null, 2));
         await shutdown();
       } catch (err) {
@@ -650,6 +664,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
         const hubs = listHubsV2(db, {
           status: options.status,
           region: options.region,
@@ -676,6 +691,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
         console.log(JSON.stringify(buildFederationContext(), null, 2));
         await shutdown();
       } catch (err) {
@@ -696,6 +712,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapFederationTables(db);
+        loadFedFromDb(db);
         console.log(JSON.stringify(getFederationStatsV2(), null, 2));
         await shutdown();
       } catch (err) {
@@ -748,6 +765,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         const p = createGovernanceProposalV2(db, {
           title: options.title,
           description: options.description,
@@ -781,6 +799,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         const r = castVoteV2(db, {
           proposalId,
           voterDid: options.voter,
@@ -807,6 +826,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(
           JSON.stringify(setProposalStatus(db, proposalId, status), null, 2),
         );
@@ -829,6 +849,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(JSON.stringify(executeProposal(db, proposalId), null, 2));
         await shutdown();
       } catch (err) {
@@ -849,6 +870,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(JSON.stringify(cancelProposal(db, proposalId), null, 2));
         await shutdown();
       } catch (err) {
@@ -869,6 +891,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(JSON.stringify(expireProposalsV2(db), null, 2));
         await shutdown();
       } catch (err) {
@@ -892,6 +915,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(
           JSON.stringify(
             listProposalsV2(db, {
@@ -922,6 +946,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(JSON.stringify(traceContributions(geneId), null, 2));
         await shutdown();
       } catch (err) {
@@ -942,6 +967,7 @@ export function registerEvoMapCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureEvoMapGovernanceTables(db);
+        loadGovFromDb(db);
         console.log(JSON.stringify(getGovernanceStatsV2(), null, 2));
         await shutdown();
       } catch (err) {

package/src/commands/governance.js

@@ -12,6 +12,7 @@ import { parseJsonOption } from "../lib/parse-json-option.js";
 import { bootstrap, shutdown } from "../runtime/bootstrap.js";
 import {
   ensureGovernanceTables,
+  loadFromDb,
   listProposalTypes,
   listProposalStatuses,
   listImpactLevels,
@@ -68,6 +69,7 @@ function _dbFromCtx(ctx) {
   }
   const db = ctx.db.getDatabase();
   ensureGovernanceTables(db);
+  loadFromDb(db);
   return db;
 }
 
@@ -292,7 +294,11 @@ export function registerGovernanceCommand(program) {
     .command("close <proposal-id>")
     .description("Close voting — auto-resolves to passed/rejected")
     .option("-q, --quorum <n>", "Quorum threshold (0-1)", floatArg("--quorum"))
-    .option("-t, --threshold <n>", "Pass threshold (0-1)", floatArg("--threshold"))
+    .option(
+      "-t, --threshold <n>",
+      "Pass threshold (0-1)",
+      floatArg("--threshold"),
+    )
     .option(
       "-n, --total-voters <n>",
       "Total eligible voters (for quorum)",
@@ -404,8 +410,16 @@ export function registerGovernanceCommand(program) {
     .command("tally <proposal-id>")
     .description("Show vote tally with quorum/threshold check")
     .option("-q, --quorum <n>", "Quorum threshold (0-1)", floatArg("--quorum"))
-    .option("-t, --threshold <n>", "Pass threshold (0-1)", floatArg("--threshold"))
-    .option("-n, --total-voters <n>", "Total eligible voters", intArg("--total-voters"))
+    .option(
+      "-t, --threshold <n>",
+      "Pass threshold (0-1)",
+      floatArg("--threshold"),
+    )
+    .option(
+      "-n, --total-voters <n>",
+      "Total eligible voters",
+      intArg("--total-voters"),
+    )
     .option("--json", "Output as JSON")
     .action(async (proposalId, options) => {
       try {

package/src/commands/hardening.js

@@ -10,6 +10,7 @@ import { parseJsonOption } from "../lib/parse-json-option.js";
 import { bootstrap, shutdown } from "../runtime/bootstrap.js";
 import {
   ensureHardeningTables,
+  loadFromDb,
   collectBaseline,
   compareBaseline,
   listBaselines,
@@ -71,6 +72,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const result = collectBaseline(db, name, options.version);
         logger.success("Baseline collected");
@@ -99,6 +101,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const result = compareBaseline(baselineId, options.current);
         if (options.json) {
@@ -134,6 +137,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const baselines = listBaselines();
         if (options.json) {
@@ -171,6 +175,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const result = runAudit(db, name);
         logger.success(`Audit complete: score ${result.score}%`);
@@ -204,6 +209,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const reports = getAuditReports();
         if (options.json) {
@@ -247,6 +253,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const requiredKeys = options.required
           ? options.required
@@ -305,6 +312,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const result = deployCheck();
         if (options.json) {
@@ -345,6 +353,7 @@ export function registerHardeningCommand(program) {
         }
         const db = ctx.db.getDatabase();
         ensureHardeningTables(db);
+        loadFromDb(db);
 
         const report = getAuditReport(auditId);
         if (options.json) {
@@ -520,6 +529,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const metadata = parseJsonOption(opts.metadata, "--metadata");
         const r = registerAuditV2(db, {
           name,
@@ -544,6 +554,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const r = startAudit(db, auditId);
         if (opts.json) console.log(JSON.stringify(r, null, 2));
         else console.log(`${auditId} → ${r.status}`);
@@ -565,6 +576,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const warningThreshold = opts.warningThreshold
           ? Number(opts.warningThreshold)
           : undefined;
@@ -592,6 +604,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const patch = {};
         if (opts.errorMessage !== undefined)
           patch.errorMessage = opts.errorMessage;
@@ -626,6 +639,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const r = autoTimeoutAudits(db);
         if (opts.json) console.log(JSON.stringify(r, null, 2));
         else console.log(`Timed out ${r.length} audit(s)`);
@@ -646,6 +660,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const metadata = parseJsonOption(opts.metadata, "--metadata");
         const r = createBaselineV2(db, {
           name,
@@ -669,6 +684,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const r = activateBaseline(db, baselineId);
         if (opts.json) console.log(JSON.stringify(r, null, 2));
         else console.log(`${baselineId} → ${r.status}`);
@@ -689,6 +705,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const patch = {};
         if (opts.reason !== undefined) patch.reason = opts.reason;
         if (opts.metadata !== undefined)
@@ -722,6 +739,7 @@ export function registerHardeningCommand(program) {
       try {
         const { db } = await bootstrap();
         ensureHardeningTables(db);
+        loadFromDb(db);
         const r = autoArchiveStaleBaselines(db);
         if (opts.json) console.log(JSON.stringify(r, null, 2));
         else console.log(`Archived ${r.length} baseline(s)`);

package/src/commands/incentive.js

@@ -10,6 +10,7 @@ import { parseJsonOption } from "../lib/parse-json-option.js";
 import { bootstrap, shutdown } from "../runtime/bootstrap.js";
 import {
   ensureTokenTables,
+  loadFromDb as loadIncentiveFromDb,
   listContributionTypes,
   listTxTypes,
   getBalance,
@@ -57,6 +58,10 @@ function _dbFromCtx(ctx) {
   }
   const db = ctx.db.getDatabase();
   ensureTokenTables(db);
+  // Rehydrate the ledger Maps from the token_* tables — without this every read
+  // is Map-only-empty in the fresh CLI process, and worse, _ensureAccount would
+  // clobber the persisted balance back to 0 via INSERT OR REPLACE.
+  loadIncentiveFromDb(db);
   return db;
 }