@claude-flow/cli 3.22.0 → 3.24.0

package/dist/src/services/evolve-proof.js

@@ -0,0 +1,341 @@
+/**
+ * single-round proof-of-mechanism  (ADR-176)
+ *
+ * SCOPE — read this before citing any output:
+ *   This is a SINGLE-ROUND PROOF-OF-MECHANISM. It is NOT flywheel proof, NOT
+ *   compounding learning, and NOT production learning. Its only purpose is to
+ *   prove, on ONE deterministic synthetic round:
+ *     (a) gate wiring        — the real versioned accept() decides promotion,
+ *     (b) receipt persistence — a self-contained bundle is written to disk,
+ *     (c) SHADOW registration — a passing candidate is registered in shadow,
+ *     (d) no auto-serve path  — nothing is applied to the active/served policy.
+ *   A synthetic PASS here is NOT evidence of real improvement.
+ *
+ * Independently verifiable: the bundle embeds the holdout, both manifests, the
+ * exact PromotionVerdict inputs, and their hashes. verifyReceiptBundle() rehashes
+ * everything and RE-RUNS the same versioned accept() — so a third party can
+ * confirm *why* the candidate passed/failed without trusting any service log.
+ *
+ * Pure Node, $0, no LLM, no network, no real store. Deterministic.
+ */
+import { createHash } from 'node:crypto';
+import { accept } from './harness-benchmark.js';
+import { bootstrapDeltaCILow } from './harness-improvement-ledger.js';
+import { canonicalManifestBytes } from '../config/proven-config.js';
+/**
+ * The promotion rule is versioned so a receipt pins exactly which semantics
+ * decided it. v1+sig = the accept() conjunction AND a statistical-significance
+ * term: the per-held-out-task deltas must have a positive one-sided 95% bootstrap
+ * lower bound, so a small-N mean gain can't ride on noise. The canary term is a
+ * SEPARATE deployment-safety signal (a distinct slice), not held-out dominance.
+ */
+export const PROMOTION_RULE_VERSION = 'accept/v1+sig';
+export const PROOF_LABEL = 'single-round proof-of-mechanism';
+export const NOT_CLAIMS = ['not flywheel proof', 'not compounding learning', 'not production learning'];
+function sha256(s) { return 'sha256:' + createHash('sha256').update(s).digest('hex'); }
+function canon(v) {
+    const c = (x) => Array.isArray(x) ? x.map(c)
+        : (x && typeof x === 'object') ? Object.fromEntries(Object.keys(x).sort().map((k) => [k, c(x[k])])) : x;
+    return JSON.stringify(c(v));
+}
+function manifestHash(m) { return sha256(canonicalManifestBytes(m).toString('utf-8')); }
+const FAILURE_CAUSE = {
+    held_out_improves: 'holdout', redblue_pass: 'security', drift_within: 'drift',
+    replay_deterministic: 'replay', receipt_coverage_full: 'governance', canary_no_worse: 'canary',
+};
+/** Classify a policy mutation by which fields changed → a repeatable mutation CLASS + a diff summary. */
+export function classifyMutation(baseline, candidate) {
+    const changed = Object.keys(candidate).filter((k) => candidate[k] !== baseline[k]).sort();
+    if (changed.length === 0)
+        return { mutationClass: 'none', mutationSummary: 'no change' };
+    const summary = changed.map((k) => `${k}:${baseline[k]}→${candidate[k]}`).join(', ');
+    const mutationClass = changed.length === 1 ? `retrieval:${changed[0]}` : 'retrieval:multi';
+    return { mutationClass, mutationSummary: summary };
+}
+function mean(xs) { return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0; }
+function mkManifest(policyValue, layer = 'synthetic/proof', corpus = 'synthetic-proof-v1') {
+    const ref = sha256(canon(policyValue));
+    return { schema: 'ruflo.proven-config/v1', policy: { ref, value: policyValue }, layer, benchmark: { corpus, corpusHash: sha256(corpus) } };
+}
+/**
+ * Assemble a receipt bundle from a holdout + configs. This is the SHARED core:
+ * the synthetic proof and a REAL measured round produce byte-identical bundle
+ * structure and run the SAME versioned accept() — so `verifyReceiptBundle`
+ * replays a real bundle exactly as it replays the synthetic fixture.
+ */
+export function assembleBundle(baseline, candidate, holdout, o) {
+    const baselineHeldOut = mean(holdout.map((h) => h.baselineScore));
+    const candidateHeldOut = mean(holdout.map((h) => h.candidateScore));
+    // Canary is a SEPARATE deployment-safety signal (default: strict per-held-out
+    // regression, preserving synthetic behavior; real rounds pass a distinct slice).
+    const canaryRollbackRate = o.canaryRollbackRate ?? holdout.filter((h) => h.candidateScore < h.baselineScore - 1e-9).length / holdout.length;
+    // Statistical-significance term (small-N noise guard): bootstrap lower bound on
+    // the per-task deltas must be > 0.
+    const deltaCILow = bootstrapDeltaCILow(holdout.map((h) => h.candidateScore - h.baselineScore));
+    const significant = deltaCILow > 0;
+    const verdictInputs = {
+        heldOutScore: candidateHeldOut, baselineHeldOutScore: baselineHeldOut,
+        redblue: o.redblue ?? 'PASS', drift: o.drift ?? 0, driftThreshold: 0.05,
+        replayDeterministic: true, receiptCoverage: 1, canaryRollbackRate, baselineRollbackRate: 0,
+    };
+    const result = accept(verdictInputs);
+    const promoted = result.accept && significant;
+    const baselineManifest = mkManifest(baseline, o.layer, o.corpus);
+    const candidateManifest = mkManifest(candidate, o.layer, o.corpus);
+    const baselineManifestHash = manifestHash(baselineManifest);
+    const candidateManifestHash = manifestHash(candidateManifest);
+    const inputHoldoutHash = sha256(canon(holdout));
+    const failed = [...result.failed, ...(!significant ? ['significant'] : [])];
+    const decisionReceipt = {
+        promotionRuleVersion: PROMOTION_RULE_VERSION, verdictInputs, result, significant, deltaCILow, promoted,
+        reason: promoted ? `promoted (all ${PROMOTION_RULE_VERSION} terms held)` : `rejected — ${failed.join(', ')}`,
+    };
+    const shadow = promoted ? {
+        registrationId: sha256(`${candidateManifestHash}|gen${o.generation}|shadow`).replace('sha256:', 'shadow:'),
+        state: 'shadow', served: false, candidateManifestHash, registeredAt: o.now,
+    } : null;
+    const { mutationClass, mutationSummary } = classifyMutation(baseline, candidate);
+    const deltas = { benchmark: candidateHeldOut - baselineHeldOut, security: o.redblue === 'FAIL' ? -1 : 0, cost: 0 };
+    const promotion = promoted ? { parentManifestHash: o.parent, candidateManifestHash, mutationClass, mutationSummary, deltas, decisionReceipt } : null;
+    const regression = promoted ? null : {
+        candidateManifestHash, ancestor: o.parent ?? baselineManifestHash, mutationClass,
+        failureCause: !significant && result.accept ? 'significance' : (FAILURE_CAUSE[result.failed[0]] ?? 'holdout'), failedTerms: failed,
+    };
+    return {
+        label: PROOF_LABEL, disclaimers: NOT_CLAIMS, generation: o.generation, parent: o.parent, branch: o.branch, kind: o.kind, createdAt: o.now,
+        inputHoldoutHash, baselineManifestHash, candidateManifestHash,
+        meetsPromotionRule: { version: PROMOTION_RULE_VERSION, result: promoted },
+        decisionReceipt, shadow,
+        costReceipt: { usd: 0, llmCalls: 0, tier: o.cost.tier, notes: o.cost.notes },
+        mutationClass, mutationSummary, deltas, promotion, regression,
+        holdout, baselineManifest, candidateManifest,
+    };
+}
+/**
+ * Build a REAL evolve-round receipt bundle from MEASURED holdout scores (live
+ * retrieval over a frozen anchor). Same gate, same replayability as the
+ * synthetic proof — but `kind: 'real'` and the scores come from actual runs.
+ * `redblue` should be 'FAIL' if the candidate regressed a frozen security/anchor
+ * slice; drift from real distribution shift. $0 (no LLM/network on this path).
+ */
+export function runRealEvolveRound(opts) {
+    return assembleBundle(opts.baseline, opts.candidate, opts.holdout, {
+        generation: opts.generation, parent: opts.parent, branch: opts.branch ?? 'main', now: opts.now, kind: 'real',
+        cost: { tier: 'real-local', notes: 'measured on the frozen anchor via live retrieval — no LLM, no network' },
+        redblue: opts.redblue, drift: opts.drift, canaryRollbackRate: opts.canaryRollbackRate, layer: 'real/retrieval', corpus: opts.corpus,
+    });
+}
+/**
+ * Run ONE deterministic synthetic evolve round and produce the receipt bundle.
+ * `now` is injected (no Date in the pure path) for reproducible fixtures.
+ * The default scenario is a strict Pareto improvement (candidate ≥ baseline on
+ * every task, > on the mean) so the full PROMOTE→SHADOW path is exercised; pass
+ * `regress: true` to exercise the REJECT path instead.
+ */
+export function runSyntheticProofRound(opts = { now: 0 }) {
+    const generation = opts.generation ?? 0;
+    const parent = opts.parent ?? null;
+    const branch = opts.branch ?? 'main';
+    const baseline = opts.baseline ?? { alpha: 0.5, subjectWeight: 2, mmrLambda: 0.7, bodyWeight: 1, typePenaltyFactor: 1 };
+    const candidate = opts.candidate ?? { alpha: 0.3, subjectWeight: 1, mmrLambda: 0.5, bodyWeight: 1.5, typePenaltyFactor: 0.5 };
+    // Deterministic synthetic holdout. Default: candidate never worse, sometimes
+    // better (Pareto). regress: candidate worse on one task (drives a REJECT).
+    const holdout = [
+        { taskId: 't0', baselineScore: 0.60, candidateScore: 0.70 },
+        { taskId: 't1', baselineScore: 0.80, candidateScore: 0.86 },
+        { taskId: 't2', baselineScore: 0.50, candidateScore: 0.62 },
+        { taskId: 't3', baselineScore: 0.72, candidateScore: 0.80 },
+        { taskId: 't4', baselineScore: 0.66, candidateScore: opts.regress ? 0.50 : 0.78 },
+    ];
+    return assembleBundle(baseline, candidate, holdout, {
+        generation, parent, branch, now: opts.now, kind: 'synthetic',
+        cost: { tier: 'synthetic', notes: 'deterministic synthetic round — no model, no network, no real store' },
+    });
+}
+/**
+ * Independently verify a receipt bundle WITHOUT trusting any service log: rehash
+ * the embedded holdout + manifests, recompute the held-out means + canary rate
+ * from the embedded per-task scores, RE-RUN the same versioned accept(), and
+ * confirm the recomputed decision equals the recorded one. Also confirms the
+ * SHADOW registration is not served (no auto-serve). Pure; never throws.
+ */
+export function verifyReceiptBundle(bundle) {
+    const mismatches = [];
+    const hashChecks = {
+        inputHoldout: sha256(canon(bundle.holdout)) === bundle.inputHoldoutHash,
+        baselineManifest: manifestHash(bundle.baselineManifest) === bundle.baselineManifestHash,
+        candidateManifest: manifestHash(bundle.candidateManifest) === bundle.candidateManifestHash,
+    };
+    if (!hashChecks.inputHoldout)
+        mismatches.push('input holdout hash mismatch');
+    if (!hashChecks.baselineManifest)
+        mismatches.push('baseline manifest hash mismatch');
+    if (!hashChecks.candidateManifest)
+        mismatches.push('candidate manifest hash mismatch');
+    const baselineHeldOut = mean(bundle.holdout.map((h) => h.baselineScore));
+    const candidateHeldOut = mean(bundle.holdout.map((h) => h.candidateScore));
+    // The canary is a SEPARATE slice (not embedded) — trust the recorded rate; the
+    // held-out means + significance ARE independently recomputed from the holdout,
+    // so holdout tampering is still caught (via means/significance/decision).
+    const canaryRollbackRate = bundle.decisionReceipt.verdictInputs.canaryRollbackRate;
+    const deltaCILow = bootstrapDeltaCILow(bundle.holdout.map((h) => h.candidateScore - h.baselineScore));
+    const significant = deltaCILow > 0;
+    // Re-run the SAME versioned rule on independently-recomputed inputs.
+    const ruleVersionMatches = bundle.decisionReceipt.promotionRuleVersion === PROMOTION_RULE_VERSION
+        && bundle.meetsPromotionRule.version === PROMOTION_RULE_VERSION;
+    if (!ruleVersionMatches)
+        mismatches.push(`promotion rule version != ${PROMOTION_RULE_VERSION}`);
+    const decision = accept({
+        ...bundle.decisionReceipt.verdictInputs,
+        heldOutScore: candidateHeldOut, baselineHeldOutScore: baselineHeldOut,
+    });
+    const promotedRecomputed = decision.accept && significant;
+    const decisionMatches = promotedRecomputed === bundle.decisionReceipt.promoted && promotedRecomputed === bundle.meetsPromotionRule.result;
+    if (!decisionMatches)
+        mismatches.push('recomputed decision != recorded decision');
+    // no-auto-serve: a shadow registration must never be marked served.
+    const noAutoServe = bundle.shadow === null || bundle.shadow.served === false;
+    if (!noAutoServe)
+        mismatches.push('candidate was auto-served (served=true) — violates shadow-only');
+    // causal record consistency: a pass carries a promotion record (with a delta
+    // that matches the recompute) and no regression; a reject carries the inverse.
+    const benchmarkDeltaMatches = Math.abs(bundle.deltas.benchmark - (candidateHeldOut - baselineHeldOut)) < 1e-9;
+    const causalConsistent = promotedRecomputed
+        ? (bundle.promotion !== null && bundle.regression === null && benchmarkDeltaMatches)
+        : (bundle.promotion === null && bundle.regression !== null);
+    if (!causalConsistent)
+        mismatches.push('causal record inconsistent with the decision (promotion/regression/delta)');
+    const valid = hashChecks.inputHoldout && hashChecks.baselineManifest && hashChecks.candidateManifest
+        && ruleVersionMatches && decisionMatches && noAutoServe && causalConsistent;
+    const why = promotedRecomputed
+        ? `PASS under ${PROMOTION_RULE_VERSION}: held_out ${candidateHeldOut.toFixed(4)} > ${baselineHeldOut.toFixed(4)} (Δ CI-low ${deltaCILow.toFixed(4)} > 0, significant), canary rollback ${canaryRollbackRate} ≤ 0, all terms held`
+        : `FAIL under ${PROMOTION_RULE_VERSION}: ${[...decision.failed, ...(!significant ? ['significant'] : [])].join(', ')}`;
+    return {
+        valid, hashChecks,
+        recomputed: { baselineHeldOut, candidateHeldOut, canaryRollbackRate, decision },
+        decisionMatches, ruleVersionMatches, noAutoServe, causalConsistent,
+        explanation: `independently recomputed from the bundle (no service logs) → ${why}`,
+        mismatches,
+    };
+}
+/**
+ * Reconstruct + audit a lineage DAG. Independently replays every bundle and
+ * checks the graph invariants back to the immutable root. Pure; never throws.
+ */
+export function reconstructLineage(bundles) {
+    const problems = [];
+    const ordered = [...bundles].sort((a, b) => a.generation - b.generation);
+    const byCandidate = new Map(ordered.map((b) => [b.candidateManifestHash, b]));
+    let allReplayable = true;
+    const nodes = ordered.map((b) => {
+        const rep = verifyReceiptBundle(b);
+        if (!rep.valid) {
+            allReplayable = false;
+            problems.push(`gen ${b.generation}: not independently replayable (${rep.mismatches.join('; ')})`);
+        }
+        return { generation: b.generation, branch: b.branch, promoted: b.decisionReceipt.promoted, parent: b.parent, candidateManifestHash: b.candidateManifestHash, mutationClass: b.mutationClass, delta: b.deltas.benchmark, replayable: rep.valid };
+    });
+    // DAG invariants.
+    let lineageIntact = true;
+    const roots = ordered.filter((b) => b.parent === null);
+    if (roots.length !== 1) {
+        lineageIntact = false;
+        problems.push(`expected exactly one immutable root, found ${roots.length}`);
+    }
+    for (const b of ordered) {
+        if (b.parent === null)
+            continue;
+        const parent = byCandidate.get(b.parent);
+        if (!parent) {
+            lineageIntact = false;
+            problems.push(`gen ${b.generation} (${b.branch}): parent ${b.parent.slice(0, 20)}… not found in graph`);
+            continue;
+        }
+        if (parent.generation >= b.generation) {
+            lineageIntact = false;
+            problems.push(`gen ${b.generation}: parent generation ${parent.generation} not older (cycle risk)`);
+        }
+        if (b.baselineManifestHash !== parent.candidateManifestHash) {
+            lineageIntact = false;
+            problems.push(`gen ${b.generation} (${b.branch}): baseline != parent's promoted candidate (did not inherit the verified policy)`);
+        }
+    }
+    return {
+        generations: ordered.length,
+        candidatesEvaluated: ordered.length,
+        promotions: ordered.filter((b) => b.decisionReceipt.promoted).length,
+        rejections: ordered.filter((b) => !b.decisionReceipt.promoted).length,
+        cumulativeHeldOutImprovement: ordered.reduce((s, b) => s + (b.decisionReceipt.promoted ? b.deltas.benchmark : 0), 0),
+        rootHash: roots[0]?.candidateManifestHash ?? null,
+        branches: [...new Set(ordered.map((b) => b.branch))],
+        lineageIntact: lineageIntact && allReplayable, allReplayable, nodes, problems,
+    };
+}
+/**
+ * Aggregate per-mutation-class effectiveness across a lineage. After enough
+ * generations the optimizer can bias toward classes with higher historical
+ * payoff — meta-learning grounded in evidence, not intuition.
+ */
+export function mutationEffectiveness(bundles) {
+    const by = new Map();
+    for (const b of bundles) {
+        const e = by.get(b.mutationClass) ?? { attempts: 0, promotions: 0, deltas: [] };
+        e.attempts++;
+        if (b.decisionReceipt.promoted) {
+            e.promotions++;
+            e.deltas.push(b.deltas.benchmark);
+        }
+        by.set(b.mutationClass, e);
+    }
+    return [...by.entries()]
+        .map(([mutationClass, e]) => ({ mutationClass, attempts: e.attempts, promotions: e.promotions, meanDelta: mean(e.deltas) }))
+        .sort((a, b) => b.meanDelta - a.meanDelta);
+}
+function median(xs) { if (!xs.length)
+    return 0; const s = [...xs].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; }
+function variance(xs) { if (xs.length < 2)
+    return 0; const mu = mean(xs); return mean(xs.map((x) => (x - mu) ** 2)); }
+/**
+ * Distinguish local optimum vs noisy benchmark vs optimizer failure — rigorously,
+ * not by intuition. Over a rolling window: near-zero median improvement AND low
+ * promotion rate is a plateau; shrinking candidate variance ⇒ converged
+ * (local optimum); high, non-shrinking variance ⇒ the benchmark is noisy; low
+ * variance with no promotions ⇒ the optimizer stopped exploring (failure).
+ */
+export function detectPlateau(bundles, opts = {}) {
+    const window = opts.window ?? 20;
+    const epsilon = opts.epsilon ?? 0.001;
+    const maxPromotionRate = opts.maxPromotionRate ?? 0.1;
+    const ordered = [...bundles].sort((a, b) => a.generation - b.generation);
+    if (ordered.length < window) {
+        return { status: 'insufficient-data', window, medianImprovement: 0, promotionRate: 0, varianceShrinking: false, candidateVariance: 0, rationale: `need ${window} generations, have ${ordered.length}` };
+    }
+    const recent = ordered.slice(-window);
+    const promotedDeltas = recent.filter((b) => b.decisionReceipt.promoted).map((b) => b.deltas.benchmark);
+    const medianImprovement = median(promotedDeltas);
+    const promotionRate = promotedDeltas.length / window;
+    const candScores = recent.map((b) => mean(b.holdout.map((h) => h.candidateScore)));
+    const firstHalfVar = variance(candScores.slice(0, window >> 1));
+    const secondHalfVar = variance(candScores.slice(window >> 1));
+    const candidateVariance = variance(candScores);
+    const varianceShrinking = secondHalfVar < firstHalfVar * 0.5;
+    let status = 'active';
+    let rationale = `median Δ ${medianImprovement.toFixed(4)} ≥ ε or promotion rate ${(promotionRate * 100).toFixed(0)}% ≥ ${(maxPromotionRate * 100).toFixed(0)}% — still improving`;
+    const plateaued = medianImprovement < epsilon && promotionRate < maxPromotionRate;
+    if (plateaued) {
+        if (varianceShrinking) {
+            status = 'local-optimum';
+            rationale = 'no gains + candidate variance shrinking → converged to a local optimum';
+        }
+        else if (candidateVariance > epsilon) {
+            status = 'noisy-benchmark';
+            rationale = 'no gains but candidates vary widely → benchmark noise is masking signal';
+        }
+        else {
+            status = 'optimizer-failure';
+            rationale = 'no gains + candidates barely vary → optimizer stopped exploring';
+        }
+    }
+    return { status, window, medianImprovement, promotionRate, varianceShrinking, candidateVariance, rationale };
+}
+//# sourceMappingURL=evolve-proof.js.map

package/dist/src/services/harness-benchmark.d.ts

@@ -0,0 +1,68 @@
+export interface BenchTask {
+    id: string;
+    input: unknown;
+    expected: unknown;
+    weight?: number;
+}
+export interface HarnessBenchmarkCorpus {
+    version: string;
+    corpusHash: string;
+    tasks: BenchTask[];
+}
+/** Run a candidate policy on a task's input, producing an output. */
+export type EvalFn<C> = (input: unknown, candidate: C) => unknown;
+/** Grade an output against expected → [0,1]. */
+export type GradeFn = (output: unknown, expected: unknown) => number;
+/** Content hash of a corpus's tasks — the tamper-evident pin for the held-out set. */
+export declare function hashCorpus(tasks: BenchTask[]): string;
+/** Verify a corpus's declared hash matches its tasks (integrity of the benchmark). */
+export declare function verifyCorpus(corpus: HarnessBenchmarkCorpus): boolean;
+export interface ScoreResult {
+    fitness: number;
+    passRate: number;
+    n: number;
+}
+/** Score a candidate over a task set, on isolated tasks (no shared state). */
+export declare function scoreOnTasks<C>(tasks: BenchTask[], candidate: C, evalFn: EvalFn<C>, gradeFn: GradeFn): ScoreResult;
+export interface HeldOutSplit {
+    train: BenchTask[];
+    heldOut: BenchTask[];
+}
+/**
+ * Deterministic held-out split. Tasks are stably ordered by id, and the last
+ * `holdoutFrac` become the held-out set — disjoint from train, reproducible
+ * (so two runs converge, per the ADR-176 acceptance test).
+ */
+export declare function computeHeldOutSplit(tasks: BenchTask[], holdoutFrac?: number): HeldOutSplit;
+/** Independently-measured verdicts fed to accept(). Each comes from a different mechanism. */
+export interface PromotionVerdict {
+    heldOutScore: number;
+    baselineHeldOutScore: number;
+    redblue: 'PASS' | 'FAIL' | 'SKIPPED';
+    drift: number;
+    driftThreshold: number;
+    replayDeterministic: boolean;
+    receiptCoverage: number;
+    canaryRollbackRate: number;
+    baselineRollbackRate: number;
+}
+export interface AcceptResult {
+    accept: boolean;
+    terms: Record<string, {
+        value: unknown;
+        pass: boolean;
+    }>;
+    failed: string[];
+}
+/**
+ * accept(candidate) ⟺
+ *   held_out_score  >  baseline
+ *   AND redblue     == PASS
+ *   AND drift       <= threshold
+ *   AND replay      == deterministic
+ *   AND receipt_cov == 100%
+ *   AND canary.rollback_rate <= baseline
+ * Every term is externally measurable; ANY failure → reject.
+ */
+export declare function accept(v: PromotionVerdict): AcceptResult;
+//# sourceMappingURL=harness-benchmark.d.ts.map

package/dist/src/services/harness-benchmark.js

@@ -0,0 +1,94 @@
+/**
+ * Benchmark corpus + held-out scoring + the accept() promotion gate
+ * (ADR-176 phase 2 + the multi-term promotion rule).
+ *
+ * Proof #1 (measured, held-out): a candidate is scored against a VERSIONED,
+ * content-hashed corpus on a held-out split — the "prove-before-ship" pattern
+ * distill-tuning.ts established (train/held-out disjoint by index; held-out
+ * scored once). Fitness is a composite, not a single gameable number.
+ *
+ * Promotion is a CONJUNCTION of externally-measurable predicates (ADR-176) —
+ * never a scalar. accept() takes the independently-measured verdicts (held-out
+ * delta, redblue, drift, deterministic replay, receipt coverage, canary
+ * rollback) and admits only when EVERY term holds. Multi-dimensional +
+ * Goodhart-resistant. Zero deps, $0.
+ */
+import { createHash } from 'crypto';
+function stable(v) {
+    const c = (x) => {
+        if (Array.isArray(x))
+            return x.map(c);
+        if (x && typeof x === 'object') {
+            const o = {};
+            for (const k of Object.keys(x).sort())
+                o[k] = c(x[k]);
+            return o;
+        }
+        return x;
+    };
+    return JSON.stringify(c(v));
+}
+/** Content hash of a corpus's tasks — the tamper-evident pin for the held-out set. */
+export function hashCorpus(tasks) {
+    const canon = tasks.map(t => ({ id: t.id, input: t.input, expected: t.expected, weight: t.weight ?? 1 }))
+        .sort((a, b) => a.id.localeCompare(b.id));
+    return 'sha256:' + createHash('sha256').update(stable(canon)).digest('hex');
+}
+/** Verify a corpus's declared hash matches its tasks (integrity of the benchmark). */
+export function verifyCorpus(corpus) {
+    return hashCorpus(corpus.tasks) === corpus.corpusHash;
+}
+/** Score a candidate over a task set, on isolated tasks (no shared state). */
+export function scoreOnTasks(tasks, candidate, evalFn, gradeFn) {
+    if (tasks.length === 0)
+        return { fitness: 0, passRate: 0, n: 0 };
+    let weighted = 0, totalW = 0, passes = 0;
+    for (const t of tasks) {
+        const w = t.weight ?? 1;
+        let g = 0;
+        try {
+            g = Math.max(0, Math.min(1, gradeFn(evalFn(t.input, candidate), t.expected)));
+        }
+        catch {
+            g = 0;
+        }
+        weighted += g * w;
+        totalW += w;
+        if (g >= 0.999)
+            passes++;
+    }
+    return { fitness: totalW > 0 ? weighted / totalW : 0, passRate: passes / tasks.length, n: tasks.length };
+}
+/**
+ * Deterministic held-out split. Tasks are stably ordered by id, and the last
+ * `holdoutFrac` become the held-out set — disjoint from train, reproducible
+ * (so two runs converge, per the ADR-176 acceptance test).
+ */
+export function computeHeldOutSplit(tasks, holdoutFrac = 0.2) {
+    const ordered = [...tasks].sort((a, b) => a.id.localeCompare(b.id));
+    const cut = Math.max(0, ordered.length - Math.max(1, Math.round(ordered.length * holdoutFrac)));
+    return { train: ordered.slice(0, cut), heldOut: ordered.slice(cut) };
+}
+/**
+ * accept(candidate) ⟺
+ *   held_out_score  >  baseline
+ *   AND redblue     == PASS
+ *   AND drift       <= threshold
+ *   AND replay      == deterministic
+ *   AND receipt_cov == 100%
+ *   AND canary.rollback_rate <= baseline
+ * Every term is externally measurable; ANY failure → reject.
+ */
+export function accept(v) {
+    const terms = {
+        held_out_improves: { value: `${v.heldOutScore.toFixed(4)} > ${v.baselineHeldOutScore.toFixed(4)}`, pass: v.heldOutScore > v.baselineHeldOutScore },
+        redblue_pass: { value: v.redblue, pass: v.redblue === 'PASS' },
+        drift_within: { value: `${v.drift} <= ${v.driftThreshold}`, pass: v.drift <= v.driftThreshold },
+        replay_deterministic: { value: v.replayDeterministic, pass: v.replayDeterministic === true },
+        receipt_coverage_full: { value: v.receiptCoverage, pass: v.receiptCoverage >= 1 },
+        canary_no_worse: { value: `${v.canaryRollbackRate} <= ${v.baselineRollbackRate}`, pass: v.canaryRollbackRate <= v.baselineRollbackRate },
+    };
+    const failed = Object.entries(terms).filter(([, t]) => !t.pass).map(([k]) => k);
+    return { accept: failed.length === 0, terms, failed };
+}
+//# sourceMappingURL=harness-benchmark.js.map

package/dist/src/services/harness-canary.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Canary — separate promotion from deployment (ADR-176 phase 5).
+ *
+ * Held-out evaluation proves a candidate on FROZEN data; it has not observed
+ * real-world behavior. The canary runs the candidate on a bounded, reversible
+ * SLICE of live work and measures rollback rate, latency, cost, failure
+ * frequency, and acceptance — the telemetry the accept() conjunction needs
+ * (canary.rollback_rate <= baseline). Only after canary evidence does PROMOTE
+ * fire. This is what stops benchmark-specific evolution from reaching global
+ * rollout. Zero deps, $0 (the runner is injected; a real one meters real work).
+ */
+/** One canary run's observed outcome. */
+export interface CanaryOutcome {
+    ok: boolean;
+    rolledBack: boolean;
+    latencyMs: number;
+    costUsd: number;
+    accepted: boolean;
+}
+/** Executes a candidate on one task input under real (or simulated) conditions. */
+export type CanaryRunner<C> = (input: unknown, candidate: C) => CanaryOutcome;
+export interface CanaryTelemetry {
+    n: number;
+    rollbackRate: number;
+    failureRate: number;
+    acceptanceRate: number;
+    latencyP95: number;
+    latencyMean: number;
+    costPerTask: number;
+    costTotalUsd: number;
+}
+export interface CanaryOptions {
+    sampleFraction?: number;
+    maxSamples?: number;
+}
+/**
+ * Run the candidate on a bounded, deterministic sample of the slice and
+ * aggregate telemetry. Deterministic sampling (stride over id-sorted inputs) so
+ * two runs converge (ADR-176 acceptance test).
+ */
+export declare function runCanary<C>(candidate: C, slice: Array<{
+    id: string;
+    input: unknown;
+}>, runner: CanaryRunner<C>, opts?: CanaryOptions): CanaryTelemetry;
+export interface CanaryComparison {
+    pass: boolean;
+    checks: Record<string, {
+        candidate: number;
+        baseline: number;
+        pass: boolean;
+    }>;
+    failed: string[];
+}
+/**
+ * Canary gate: the candidate must be NO WORSE than the baseline on rollback,
+ * latency, and cost (the ADR-176 metrics table constraints). Feeds accept()'s
+ * `canary.rollback_rate <= baseline` term plus the latency/cost guards.
+ */
+export declare function compareCanary(candidate: CanaryTelemetry, baseline: CanaryTelemetry, tolerance?: number): CanaryComparison;
+//# sourceMappingURL=harness-canary.d.ts.map

package/dist/src/services/harness-canary.js

@@ -0,0 +1,69 @@
+/**
+ * Canary — separate promotion from deployment (ADR-176 phase 5).
+ *
+ * Held-out evaluation proves a candidate on FROZEN data; it has not observed
+ * real-world behavior. The canary runs the candidate on a bounded, reversible
+ * SLICE of live work and measures rollback rate, latency, cost, failure
+ * frequency, and acceptance — the telemetry the accept() conjunction needs
+ * (canary.rollback_rate <= baseline). Only after canary evidence does PROMOTE
+ * fire. This is what stops benchmark-specific evolution from reaching global
+ * rollout. Zero deps, $0 (the runner is injected; a real one meters real work).
+ */
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
+    return sorted[idx];
+}
+/**
+ * Run the candidate on a bounded, deterministic sample of the slice and
+ * aggregate telemetry. Deterministic sampling (stride over id-sorted inputs) so
+ * two runs converge (ADR-176 acceptance test).
+ */
+export function runCanary(candidate, slice, runner, opts = {}) {
+    const frac = opts.sampleFraction ?? 0.1;
+    const cap = opts.maxSamples ?? 100;
+    const ordered = [...slice].sort((a, b) => a.id.localeCompare(b.id));
+    const want = Math.min(cap, Math.max(1, Math.round(ordered.length * frac)));
+    // Even stride so the sample spans the slice rather than a contiguous head.
+    const stride = Math.max(1, Math.floor(ordered.length / want));
+    const sample = [];
+    for (let i = 0; i < ordered.length && sample.length < want; i += stride)
+        sample.push(ordered[i]);
+    const outcomes = sample.map(s => {
+        try {
+            return runner(s.input, candidate);
+        }
+        catch {
+            return { ok: false, rolledBack: true, latencyMs: 0, costUsd: 0, accepted: false };
+        }
+    });
+    const n = outcomes.length || 1;
+    const lat = outcomes.map(o => o.latencyMs).sort((a, b) => a - b);
+    const costTotal = outcomes.reduce((s, o) => s + o.costUsd, 0);
+    return {
+        n: outcomes.length,
+        rollbackRate: outcomes.filter(o => o.rolledBack).length / n,
+        failureRate: outcomes.filter(o => !o.ok).length / n,
+        acceptanceRate: outcomes.filter(o => o.accepted).length / n,
+        latencyP95: percentile(lat, 0.95),
+        latencyMean: lat.reduce((s, v) => s + v, 0) / n,
+        costPerTask: costTotal / n,
+        costTotalUsd: costTotal,
+    };
+}
+/**
+ * Canary gate: the candidate must be NO WORSE than the baseline on rollback,
+ * latency, and cost (the ADR-176 metrics table constraints). Feeds accept()'s
+ * `canary.rollback_rate <= baseline` term plus the latency/cost guards.
+ */
+export function compareCanary(candidate, baseline, tolerance = 1e-9) {
+    const checks = {
+        rollback_no_worse: { candidate: candidate.rollbackRate, baseline: baseline.rollbackRate, pass: candidate.rollbackRate <= baseline.rollbackRate + tolerance },
+        latency_no_worse: { candidate: candidate.latencyP95, baseline: baseline.latencyP95, pass: candidate.latencyP95 <= baseline.latencyP95 * (1 + 0.01) },
+        cost_no_worse: { candidate: candidate.costPerTask, baseline: baseline.costPerTask, pass: candidate.costPerTask <= baseline.costPerTask * (1 + 0.01) },
+    };
+    const failed = Object.entries(checks).filter(([, c]) => !c.pass).map(([k]) => k);
+    return { pass: failed.length === 0, checks, failed };
+}
+//# sourceMappingURL=harness-canary.js.map