claude-flow 3.23.0 → 3.24.0

package/v3/@claude-flow/cli/dist/src/services/harness-flywheel.js

@@ -0,0 +1,207 @@
+/**
+ * Self-optimizing flywheel (ADR-176) — closes the loop so an install gets
+ * smarter AS IT RUNS, with proof.
+ *
+ * Each tick:
+ *   1. HARVEST a benchmark corpus from the install's REAL store (self-supervised
+ *      self-retrieval), blended with the human-labeled ADR-081 anchor.
+ *   2. BASELINE = the currently-active champion (or shipped defaults).
+ *   3. PROPOSE neighbor configs; pick the best on the TRAIN split (local
+ *      hill-climb — deterministic, selection uses train only).
+ *   4. GATE the winner through the shipped runHarnessLoop on the HELD-OUT split:
+ *      held_out_improves AND redblue(anchor-no-regress) AND drift<=thr AND
+ *      replay-deterministic AND receipt_coverage AND canary-no-worse.
+ *   5. On accept → APPLY locally (the install self-optimizes on its own data; no
+ *      signing needed because nothing is propagated), CHAIN to the previous
+ *      champion, and record the attempt in the improvement ledger. Also STAGE
+ *      the unsigned champion for optional promotion to the signed global channel.
+ *
+ * Trust split: LOCAL self-optimization is unsigned (an install trusting its own
+ * measured gate); CROSS-install propagation still requires the config-signed
+ * champion (ADR-177). Deterministic, $0, never throws. Injectable deps → testable
+ * without ONNX/network.
+ */
+import { createHash } from 'node:crypto';
+import { runHarnessLoop } from './harness-loop.js';
+import { hashCorpus } from './harness-benchmark.js';
+import { harvestSelfSupervisedTasks, blendCorpus } from './harness-corpus-harvester.js';
+import { applyChampionParams } from '../config/harness-feedback-applier.js';
+import { appendLedger, bootstrapDeltaCILow } from './harness-improvement-ledger.js';
+export const DEFAULT_CONFIG = { alpha: 0.5, subjectWeight: 2.0, mmrLambda: 0.7, bodyWeight: 1.0, typePenaltyFactor: 1.0 };
+const EPS = 1e-3;
+const cfgCanon = (c) => JSON.stringify(Object.fromEntries(Object.keys(c).sort().map((k) => [k, c[k]])));
+const refOf = (c) => 'sha256:' + createHash('sha256').update(cfgCanon(c)).digest('hex');
+const cfgKey = (c) => cfgCanon(c);
+function ndcg3(names, labels) {
+    const rel = names.slice(0, 3).map((n) => !!n && labels.some((s) => n.toLowerCase().includes(s.toLowerCase())));
+    const dcg = rel.reduce((a, r, i) => a + (r ? 1 / Math.log2(i + 2) : 0), 0);
+    const num = rel.filter(Boolean).length;
+    if (num === 0)
+        return 0;
+    let idcg = 0;
+    for (let i = 0; i < num; i++)
+        idcg += 1 / Math.log2(i + 2);
+    return idcg > 0 ? dcg / idcg : 0;
+}
+/** grade dispatch: anchor tasks (labels[]) → nDCG@3; harvested tasks (doc id) → reciprocal rank. */
+function grade(ranked, expected) {
+    if (Array.isArray(expected))
+        return ndcg3(ranked.map((r) => r.name), expected);
+    const idx = ranked.findIndex((r) => r.id === expected);
+    return idx >= 0 ? 1 / (idx + 1) : 0;
+}
+function neighbors(base) {
+    const steps = { alpha: 0.1, subjectWeight: 0.5, mmrLambda: 0.1, bodyWeight: 0.5, typePenaltyFactor: 0.25 };
+    const out = [];
+    const seen = new Set();
+    // Per-axis moves at 1 AND 2 steps in each direction — enough to escape a flat
+    // single step and reach a multi-step optimum over successive ticks (the
+    // single-step search got stuck one hop short of the known champion).
+    for (const ax of Object.keys(steps)) {
+        for (const mult of [1, 2]) {
+            for (const dir of [-1, 1]) {
+                const v = +(base[ax] + dir * mult * steps[ax]).toFixed(3);
+                if (ax === 'alpha' && (v <= 0 || v >= 1))
+                    continue;
+                if (ax === 'mmrLambda' && (v < 0 || v > 1))
+                    continue;
+                if (v <= 0)
+                    continue;
+                const cand = { ...base, [ax]: v };
+                const k = cfgKey(cand);
+                if (seen.has(k))
+                    continue;
+                seen.add(k);
+                out.push(cand);
+            }
+        }
+    }
+    return out;
+}
+/** Deterministic id-sort split (matches computeHeldOutSplit @ frac). */
+function split(tasks, frac) {
+    const ordered = [...tasks].sort((a, b) => a.id.localeCompare(b.id));
+    const cut = Math.max(0, ordered.length - Math.max(1, Math.round(ordered.length * frac)));
+    return { train: ordered.slice(0, cut), held: ordered.slice(cut) };
+}
+/**
+ * Run one flywheel tick against `projectRoot`. Best-effort; never throws.
+ * Returns a rich result AND (as a side effect) appends to the improvement ledger
+ * and — on accept — applies the champion locally + chains it.
+ */
+export async function runFlywheelTick(projectRoot, deps) {
+    try {
+        const patterns = await deps.getPatterns();
+        if (!patterns || patterns.length < 8)
+            return { ran: false, reason: 'store too small to harvest a corpus' };
+        const harvested = harvestSelfSupervisedTasks(patterns, { sample: deps.sample ?? 40 });
+        if (harvested.length < 4)
+            return { ran: false, reason: 'not enough harvestable tasks' };
+        const blended = blendCorpus(deps.anchorTasks, harvested);
+        const anchorIdSet = new Set(blended.anchorIds);
+        const baseline = { ...DEFAULT_CONFIG, ...(deps.activeParams?.() ?? {}) };
+        const candidates = neighbors(baseline);
+        // OBJECTIVE = the human-labeled anchor (the relevance we actually care about,
+        // where headroom is known to exist). GUARD = the large, growing harvested set
+        // (don't wreck broad retrieval while tuning the objective). Optimize the
+        // trusted signal; guard breadth with the cheap one.
+        const objective = blended.tasks.filter((t) => anchorIdSet.has(t.id));
+        const guard = blended.tasks.filter((t) => !anchorIdSet.has(t.id));
+        if (objective.length < 4)
+            return { ran: false, reason: 'objective (anchor) too small to gate' };
+        // Precompute retrieval for baseline + all candidates over every task (async
+        // I/O up front → the harness scoring stays pure/sync).
+        const cache = new Map();
+        const configs = [baseline, ...candidates];
+        for (const cfg of configs) {
+            for (const t of blended.tasks) {
+                const q = t.input.q;
+                cache.set(`${t.id}::${cfgKey(cfg)}`, (await deps.search(q, cfg)) || []);
+            }
+        }
+        const evalFn = (input, cfg) => cache.get(`${input.id}::${cfgKey(cfg)}`) ?? [];
+        const gradeFn = (output, expected) => grade(output, expected);
+        const heldScoreFor = (cfg, t) => gradeFn(evalFn(t.input, cfg), t.expected);
+        // Local hill-climb on the OBJECTIVE train split (selection uses train only).
+        const { train, held } = split(objective, 0.5);
+        const trainScore = (cfg) => train.reduce((s, t) => s + heldScoreFor(cfg, t), 0) / train.length;
+        const baseTrain = trainScore(baseline);
+        let candidate = baseline, candTrain = baseTrain;
+        for (const c of candidates) {
+            const s = trainScore(c);
+            if (s > candTrain + 1e-9) {
+                candTrain = s;
+                candidate = c;
+            }
+        }
+        // Generalization guard: the candidate must not regress the broad harvested
+        // set (bound to the adversarial redblue verdict). This replaces the earlier
+        // (inverted) design where the cheap metric was the objective.
+        const guardScore = (cfg) => guard.length ? guard.reduce((s, t) => s + heldScoreFor(cfg, t), 0) / guard.length : 1;
+        const guardRegressed = guardScore(candidate) < guardScore(baseline) - EPS;
+        // Qualified trajectories — one per objective train task. Deterministic,
+        // executable checks with unambiguous ground truth → oracle:test-exec.
+        const trajectories = train.map((t) => ({
+            id: `fw-${t.id}`, steps: [{ action: 'retrieve', tier: 'oracle:test-exec' }],
+            outcome: 'success', benchmarkTaskId: `${blended.version}/${t.id}`,
+            inputs: { q: t.input.q }, recordedOutputs: { ranked: cache.get(`${t.id}::${cfgKey(candidate)}`) },
+        }));
+        const replay = (tr) => tr.recordedOutputs;
+        const anchorRegressed = guardRegressed; // ledger field: did the broad guard set regress?
+        const corpus = { version: blended.version, tasks: objective, corpusHash: hashCorpus(objective) };
+        const result = await runHarnessLoop({
+            trajectories, corpus, baseline, candidate, evalFn, gradeFn, replay,
+            verify: {
+                redblue: async () => (guardRegressed ? 'FAIL' : 'PASS'),
+                drift: async () => guard.length ? guard.filter((t) => heldScoreFor(candidate, t) < heldScoreFor(baseline, t) - EPS).length / guard.length : 0,
+            },
+            canaryRunner: (input, cfg) => {
+                const t = objective.find((x) => x.id === input.id);
+                const worse = heldScoreFor(cfg, t) < heldScoreFor(baseline, t) - EPS;
+                return { ok: !worse, rolledBack: worse, latencyMs: 0, costUsd: 0, accepted: !worse };
+            },
+            holdoutFrac: 0.5, driftThreshold: 0.2, layer: 'repo/local', policyRefOf: refOf, now: deps.now,
+        });
+        const baselineScore = result.baselineScore ?? 0;
+        const candidateScore = result.candidateScore ?? 0;
+        // Significance gate (SOTA noise guard): the per-held-out-task deltas must have
+        // a positive one-sided 95% bootstrap lower bound — the gain has to survive
+        // resampling, not ride on one lucky task. FINAL accept = loop-accept AND
+        // significant, so the ledger's accepted subsequence stays monotonic + real.
+        const heldDeltas = held.map((t) => heldScoreFor(candidate, t) - heldScoreFor(baseline, t));
+        const deltaCILow = bootstrapDeltaCILow(heldDeltas);
+        const significant = deltaCILow > 0;
+        const finalAccept = result.accepted && significant;
+        const entry = {
+            ts: deps.now ?? Date.now(),
+            corpusVersion: blended.version, corpusHash: blended.corpusHash,
+            corpusSize: blended.tasks.length, anchorSize: blended.anchorIds.length,
+            baselineRef: refOf(baseline), candidateRef: refOf(candidate),
+            baselineScore, candidateScore, delta: candidateScore - baselineScore,
+            deltaCILow, significant, loopAccepted: result.accepted,
+            anchorRegressed, accepted: finalAccept,
+            gates: Object.fromEntries(Object.entries(result.verdict?.terms ?? {}).map(([k, v]) => [k, v.pass])),
+            reason: finalAccept ? result.reason : (result.accepted ? `held back — improvement not significant (CI low ${deltaCILow.toFixed(4)})` : result.reason),
+        };
+        let applied = false;
+        if (finalAccept && result.manifest) {
+            entry.championRef = refOf(candidate);
+            // Apply locally (self-optimization) + chain to the previous champion.
+            const ap = applyChampionParams(projectRoot, {
+                championId: refOf(candidate), params: candidate,
+                layer: 'repo/local', previous: refOf(baseline), now: deps.now,
+            });
+            applied = ap.applied;
+        }
+        appendLedger(`${projectRoot}/.claude-flow/metrics`, entry);
+        return {
+            ran: true, reason: entry.reason, accepted: finalAccept, applied,
+            baselineScore, candidateScore, delta: candidateScore - baselineScore,
+            anchorRegressed, championRef: entry.championRef, corpusVersion: blended.version,
+        };
+    }
+    catch (e) {
+        return { ran: false, reason: `error: ${e?.message ?? e}` };
+    }
+}
+//# sourceMappingURL=harness-flywheel.js.map

package/v3/@claude-flow/cli/dist/src/services/harness-hosts.d.ts

@@ -0,0 +1,40 @@
+export interface HostAdapter {
+    id: string;
+    label: string;
+    /** True when this host is usable in the current environment. */
+    detect: () => boolean;
+}
+export declare class HostRegistry {
+    private hosts;
+    register(a: HostAdapter): this;
+    get(id: string): HostAdapter | undefined;
+    all(): HostAdapter[];
+    /** Hosts present in this environment (detect() true, swallowing errors). */
+    available(): HostAdapter[];
+}
+/** True if `bin --version` runs. Used by the built-in adapters' detect(). */
+export declare function commandExists(bin: string): boolean;
+/** The built-in hosts. Detection is injectable per-adapter for tests. */
+export declare function defaultHostRegistry(): HostRegistry;
+/** Run `fn` for each host (sequentially, isolated), collecting per-host results. */
+export declare function fanOutHosts<T>(hosts: HostAdapter[], fn: (h: HostAdapter) => Promise<T> | T): Promise<Array<{
+    host: string;
+    result: T | null;
+    error?: string;
+}>>;
+export declare const LAYER_LEVELS: readonly ["global", "language", "framework", "repo"];
+/** All ancestor prefixes of a layer path, shallowest first. */
+export declare function ancestorsOf(layer: string): string[];
+/** True if `a` is an ancestor of (or equal to) `b` at a path boundary. */
+export declare function isAncestorOrEqual(a: string, b: string): boolean;
+export declare function layerDepth(layer: string): number;
+/**
+ * Pick the most-specific champion applicable to an install's layer: the deepest
+ * manifest whose layer is an ancestor-or-equal of `installLayer`. Layers the
+ * install can't clear fall back to the parent (return the next-deepest). Null if
+ * none applies.
+ */
+export declare function selectChampionForLayer<M extends {
+    layer?: string;
+}>(manifests: M[], installLayer: string): M | null;
+//# sourceMappingURL=harness-hosts.d.ts.map

package/v3/@claude-flow/cli/dist/src/services/harness-hosts.js

@@ -0,0 +1,88 @@
+/**
+ * Host registry + hierarchical layers (ADR-176 phase 7).
+ *
+ * "All available hosts": a small registry (claude-code, codex, extensible) that
+ * the optimize+verify+canary pass fans out across, so a champion is proven
+ * per-host rather than via an unvalidated `--host` passthrough.
+ *
+ * Hierarchical evolution: repository-specific optima emerge, so evolution is
+ * layered — global → language → framework → repo — each layer inheriting upward
+ * but independently benchmarked. An install adopts the most-specific champion
+ * whose layer is an ancestor-or-equal of its own. Zero deps, $0.
+ */
+import { execFileSync } from 'child_process';
+export class HostRegistry {
+    hosts = new Map();
+    register(a) { this.hosts.set(a.id, a); return this; }
+    get(id) { return this.hosts.get(id); }
+    all() { return [...this.hosts.values()]; }
+    /** Hosts present in this environment (detect() true, swallowing errors). */
+    available() {
+        return this.all().filter(h => { try {
+            return h.detect();
+        }
+        catch {
+            return false;
+        } });
+    }
+}
+/** True if `bin --version` runs. Used by the built-in adapters' detect(). */
+export function commandExists(bin) {
+    try {
+        execFileSync(bin, ['--version'], { stdio: 'ignore', timeout: 3000 });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** The built-in hosts. Detection is injectable per-adapter for tests. */
+export function defaultHostRegistry() {
+    return new HostRegistry()
+        .register({ id: 'claude-code', label: 'Claude Code', detect: () => commandExists('claude') })
+        .register({ id: 'codex', label: 'OpenAI Codex', detect: () => commandExists('codex') });
+}
+/** Run `fn` for each host (sequentially, isolated), collecting per-host results. */
+export async function fanOutHosts(hosts, fn) {
+    const out = [];
+    for (const h of hosts) {
+        try {
+            out.push({ host: h.id, result: await fn(h) });
+        }
+        catch (e) {
+            out.push({ host: h.id, result: null, error: e?.message ?? String(e) });
+        }
+    }
+    return out;
+}
+// ── Hierarchical layers ─────────────────────────────────────────────────────
+export const LAYER_LEVELS = ['global', 'language', 'framework', 'repo'];
+/** All ancestor prefixes of a layer path, shallowest first. */
+export function ancestorsOf(layer) {
+    const parts = layer.split('/').filter(Boolean);
+    const out = [];
+    for (let i = 1; i <= parts.length; i++)
+        out.push(parts.slice(0, i).join('/'));
+    return out;
+}
+/** True if `a` is an ancestor of (or equal to) `b` at a path boundary. */
+export function isAncestorOrEqual(a, b) {
+    return a === b || b.startsWith(a + '/');
+}
+export function layerDepth(layer) {
+    return layer.split('/').filter(Boolean).length;
+}
+/**
+ * Pick the most-specific champion applicable to an install's layer: the deepest
+ * manifest whose layer is an ancestor-or-equal of `installLayer`. Layers the
+ * install can't clear fall back to the parent (return the next-deepest). Null if
+ * none applies.
+ */
+export function selectChampionForLayer(manifests, installLayer) {
+    const applicable = manifests.filter(m => !!m.layer && isAncestorOrEqual(m.layer, installLayer));
+    if (applicable.length === 0)
+        return null;
+    applicable.sort((x, y) => layerDepth(y.layer) - layerDepth(x.layer));
+    return applicable[0];
+}
+//# sourceMappingURL=harness-hosts.js.map

package/v3/@claude-flow/cli/dist/src/services/harness-improvement-ledger.d.ts

@@ -0,0 +1,63 @@
+export declare const LEDGER_FILE = "harness-improvement.jsonl";
+export interface LedgerEntry {
+    ts: number;
+    corpusVersion: string;
+    corpusHash: string;
+    corpusSize: number;
+    anchorSize: number;
+    baselineRef: string;
+    candidateRef: string;
+    baselineScore: number;
+    candidateScore: number;
+    delta: number;
+    deltaCILow?: number;
+    significant?: boolean;
+    loopAccepted?: boolean;
+    anchorRegressed: boolean;
+    accepted: boolean;
+    gates: Record<string, boolean>;
+    championRef?: string;
+    reason: string;
+}
+/**
+ * One-sided 95% bootstrap lower bound on the mean of paired per-task deltas.
+ * Deterministic (seeded LCG) so the confidence bound is reproducible — the same
+ * evidence yields the same verdict. A positive lower bound means the improvement
+ * survives resampling: it is not an artifact of a lucky held-out task (small-N
+ * noise guard, "measured not marketing").
+ */
+export declare function bootstrapDeltaCILow(deltas: number[], opts?: {
+    iters?: number;
+    alpha?: number;
+    seed?: number;
+}): number;
+export interface ImprovementSummary {
+    attempts: number;
+    accepted: number;
+    rejected: number;
+    cumulativeDelta: number;
+    currentScore: number | null;
+    firstScore: number | null;
+    monotonic: boolean;
+    chainIntact: boolean;
+    trajectory: Array<{
+        ts: number;
+        corpusVersion: string;
+        baseline: number;
+        candidate: number;
+        delta: number;
+        accepted: boolean;
+    }>;
+}
+/** Append one attempt. Best-effort, never throws; rotates when over the cap. */
+export declare function appendLedger(dir: string, entry: LedgerEntry): void;
+/** Read all ledger entries (oldest → newest). Never throws. */
+export declare function readLedger(dir: string): LedgerEntry[];
+/**
+ * Fold the ledger into an auditable improvement claim. Monotonicity + chain
+ * integrity are the PROOF: they hold iff every shipped champion strictly beat
+ * its predecessor and referenced it. A single violation flips the flag — the
+ * summary cannot silently launder a regression.
+ */
+export declare function summarizeImprovement(dir: string): ImprovementSummary;
+//# sourceMappingURL=harness-improvement-ledger.d.ts.map

package/v3/@claude-flow/cli/dist/src/services/harness-improvement-ledger.js

@@ -0,0 +1,101 @@
+/**
+ * Improvement ledger — the proof-of-improvement surface for the self-optimizing
+ * flywheel (ADR-176). Every mint ATTEMPT (accepted or rejected) is appended to a
+ * bounded, append-only JSONL. Because runHarnessLoop only ACCEPTS a strict
+ * improvement (held_out_improves AND canary-no-worse AND anchor-no-regress), the
+ * subsequence of accepted champions is monotonically non-decreasing in held-out
+ * score BY CONSTRUCTION — and each accepted entry chains to its predecessor via
+ * baselineRef == previous championRef. summarizeImprovement() turns that into an
+ * auditable claim: "N champions shipped, each strictly beat the last, cumulative
+ * +X; M attempts refused (the gate is not a rubber stamp)."
+ *
+ * Pure Node, $0. Rotation-capped so an always-on daemon can't grow it unbounded.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+export const LEDGER_FILE = 'harness-improvement.jsonl';
+const MAX_ENTRIES = 5000; // rotation cap (keeps the most-recent tail)
+/**
+ * One-sided 95% bootstrap lower bound on the mean of paired per-task deltas.
+ * Deterministic (seeded LCG) so the confidence bound is reproducible — the same
+ * evidence yields the same verdict. A positive lower bound means the improvement
+ * survives resampling: it is not an artifact of a lucky held-out task (small-N
+ * noise guard, "measured not marketing").
+ */
+export function bootstrapDeltaCILow(deltas, opts = {}) {
+    const n = deltas.length;
+    if (n === 0)
+        return 0;
+    const iters = opts.iters ?? 2000;
+    const alpha = opts.alpha ?? 0.05;
+    let seed = (opts.seed ?? 0x9e3779b1) >>> 0;
+    const rnd = () => { seed = (1664525 * seed + 1013904223) >>> 0; return seed / 4294967296; };
+    const means = new Array(iters);
+    for (let b = 0; b < iters; b++) {
+        let s = 0;
+        for (let i = 0; i < n; i++)
+            s += deltas[Math.floor(rnd() * n)];
+        means[b] = s / n;
+    }
+    means.sort((a, b) => a - b);
+    return means[Math.floor(alpha * iters)];
+}
+function ledgerPath(dir) {
+    return path.join(dir, LEDGER_FILE);
+}
+/** Append one attempt. Best-effort, never throws; rotates when over the cap. */
+export function appendLedger(dir, entry) {
+    try {
+        fs.mkdirSync(dir, { recursive: true });
+        const p = ledgerPath(dir);
+        fs.appendFileSync(p, JSON.stringify(entry) + '\n', 'utf-8');
+        // Rotate: if too many lines, keep the most-recent MAX_ENTRIES.
+        const lines = fs.readFileSync(p, 'utf-8').split('\n').filter(Boolean);
+        if (lines.length > MAX_ENTRIES) {
+            fs.writeFileSync(p, lines.slice(lines.length - MAX_ENTRIES).join('\n') + '\n', 'utf-8');
+        }
+    }
+    catch { /* non-fatal */ }
+}
+/** Read all ledger entries (oldest → newest). Never throws. */
+export function readLedger(dir) {
+    try {
+        const raw = fs.readFileSync(ledgerPath(dir), 'utf-8');
+        return raw.split('\n').filter(Boolean).map((l) => JSON.parse(l));
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Fold the ledger into an auditable improvement claim. Monotonicity + chain
+ * integrity are the PROOF: they hold iff every shipped champion strictly beat
+ * its predecessor and referenced it. A single violation flips the flag — the
+ * summary cannot silently launder a regression.
+ */
+export function summarizeImprovement(dir) {
+    const all = readLedger(dir);
+    const accepted = all.filter((e) => e.accepted);
+    let monotonic = true;
+    let chainIntact = true;
+    for (let i = 0; i < accepted.length; i++) {
+        if (!(accepted[i].candidateScore > accepted[i].baselineScore))
+            monotonic = false;
+        if (accepted[i].anchorRegressed)
+            monotonic = false;
+        if (i > 0 && accepted[i].baselineRef !== accepted[i - 1].championRef)
+            chainIntact = false;
+    }
+    return {
+        attempts: all.length,
+        accepted: accepted.length,
+        rejected: all.length - accepted.length,
+        cumulativeDelta: accepted.reduce((s, e) => s + e.delta, 0),
+        currentScore: accepted.length ? accepted[accepted.length - 1].candidateScore : null,
+        firstScore: accepted.length ? accepted[0].baselineScore : null,
+        monotonic,
+        chainIntact,
+        trajectory: all.map((e) => ({ ts: e.ts, corpusVersion: e.corpusVersion, baseline: e.baselineScore, candidate: e.candidateScore, delta: e.delta, accepted: e.accepted })),
+    };
+}
+//# sourceMappingURL=harness-improvement-ledger.js.map

package/v3/@claude-flow/cli/dist/src/services/harness-loop.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Loop orchestrator (ADR-176 phase 8).
+ *
+ * Composes every gate into one pass:
+ *   OBSERVE → QUALIFY → BENCHMARK(held-out) → VERIFY(adversarial+drift)
+ *           → CANARY → ACCEPT(conjunction) → emit champion manifest.
+ *
+ * $0 + fail-closed by default: with no optimizer/verifier/canary wired, nothing
+ * is promoted and the current signed champion stands. A champion manifest is
+ * emitted ONLY when every accept() term holds — and it is emitted UNSIGNED here;
+ * signing is a separate publish step (scripts/sign-proven-config.mjs, GCP key)
+ * so key material never touches the loop. The daemon worker runs this bounded.
+ */
+import { AntiPatternArchive, type Trajectory, type ReplayFn } from './harness-qualification.js';
+import { type HarnessBenchmarkCorpus, type EvalFn, type GradeFn, type AcceptResult } from './harness-benchmark.js';
+import { type VerifyOptions, type VerifyResult } from './harness-verify.js';
+import { type CanaryRunner, type CanaryTelemetry } from './harness-canary.js';
+import type { ProvenConfigManifest } from '../config/proven-config.js';
+export interface HarnessLoopOptions<C> {
+    trajectories: Trajectory[];
+    corpus: HarnessBenchmarkCorpus;
+    baseline: C;
+    candidate?: C;
+    evalFn: EvalFn<C>;
+    gradeFn: GradeFn;
+    replay?: ReplayFn;
+    verify?: VerifyOptions;
+    canaryRunner?: CanaryRunner<C>;
+    archive?: AntiPatternArchive;
+    holdoutFrac?: number;
+    driftThreshold?: number;
+    layer?: string;
+    policyRefOf?: (candidate: C) => string;
+    now?: number;
+}
+export interface HarnessLoopResult {
+    admitted: number;
+    rejected: number;
+    baselineScore?: number;
+    candidateScore?: number;
+    verify?: VerifyResult;
+    canary?: {
+        candidate: CanaryTelemetry;
+        baseline: CanaryTelemetry;
+        pass: boolean;
+    };
+    verdict?: AcceptResult;
+    accepted: boolean;
+    manifest?: ProvenConfigManifest;
+    reason: string;
+}
+export declare function runHarnessLoop<C>(opts: HarnessLoopOptions<C>): Promise<HarnessLoopResult>;
+//# sourceMappingURL=harness-loop.d.ts.map

package/v3/@claude-flow/cli/dist/src/services/harness-loop.js

@@ -0,0 +1,85 @@
+/**
+ * Loop orchestrator (ADR-176 phase 8).
+ *
+ * Composes every gate into one pass:
+ *   OBSERVE → QUALIFY → BENCHMARK(held-out) → VERIFY(adversarial+drift)
+ *           → CANARY → ACCEPT(conjunction) → emit champion manifest.
+ *
+ * $0 + fail-closed by default: with no optimizer/verifier/canary wired, nothing
+ * is promoted and the current signed champion stands. A champion manifest is
+ * emitted ONLY when every accept() term holds — and it is emitted UNSIGNED here;
+ * signing is a separate publish step (scripts/sign-proven-config.mjs, GCP key)
+ * so key material never touches the loop. The daemon worker runs this bounded.
+ */
+import { admitTrajectories } from './harness-qualification.js';
+import { computeHeldOutSplit, scoreOnTasks, accept, } from './harness-benchmark.js';
+import { runVerify } from './harness-verify.js';
+import { runCanary, compareCanary } from './harness-canary.js';
+export async function runHarnessLoop(opts) {
+    const driftThreshold = opts.driftThreshold ?? 0.05;
+    // 1. QUALIFY — only receipt-backed, deterministic, benchmark-attributed
+    //    trajectories become training data; rejects → anti-pattern archive.
+    const admission = admitTrajectories(opts.trajectories, { replay: opts.replay, archive: opts.archive, ts: opts.now });
+    const base = { admitted: admission.admittedCount, rejected: admission.rejectedCount, accepted: false, reason: '' };
+    if (admission.admittedCount === 0)
+        return { ...base, reason: 'no qualified trajectories (receipt_coverage=0)' };
+    if (opts.candidate === undefined)
+        return { ...base, reason: 'no candidate to evaluate' };
+    // 2. BENCHMARK — held-out scoring (isolated, reproducible split).
+    const { heldOut } = computeHeldOutSplit(opts.corpus.tasks, opts.holdoutFrac ?? 0.2);
+    const baselineScore = scoreOnTasks(heldOut, opts.baseline, opts.evalFn, opts.gradeFn).fitness;
+    const candidateScore = scoreOnTasks(heldOut, opts.candidate, opts.evalFn, opts.gradeFn).fitness;
+    // 3. VERIFY — adversarial + drift (fail-closed: SKIPPED cannot pass).
+    const verify = await runVerify({ ...opts.verify, driftThreshold });
+    // 4. CANARY — required (separate promotion from deployment); absent => no promote.
+    if (!opts.canaryRunner) {
+        return { ...base, baselineScore, candidateScore, verify, accepted: false, reason: 'canary required (no runner) — cannot promote' };
+    }
+    const slice = opts.corpus.tasks.map(t => ({ id: t.id, input: t.input }));
+    const canaryCand = runCanary(opts.candidate, slice, opts.canaryRunner);
+    const canaryBase = runCanary(opts.baseline, slice, opts.canaryRunner);
+    const canaryCmp = compareCanary(canaryCand, canaryBase);
+    // 5. ACCEPT — the full conjunction. receipt_coverage=1 by construction (only
+    //    qualified trajectories are used; rejects were excluded).
+    const verdict = {
+        heldOutScore: candidateScore,
+        baselineHeldOutScore: baselineScore,
+        redblue: verify.redblue,
+        drift: verify.drift < 0 ? Number.POSITIVE_INFINITY : verify.drift,
+        driftThreshold,
+        replayDeterministic: !!opts.replay, // qualification already enforced replay determinism on admitted set
+        receiptCoverage: 1,
+        canaryRollbackRate: canaryCand.rollbackRate,
+        baselineRollbackRate: canaryBase.rollbackRate,
+    };
+    const decision = accept(verdict);
+    const result = {
+        ...base, baselineScore, candidateScore, verify,
+        canary: { candidate: canaryCand, baseline: canaryBase, pass: canaryCmp.pass },
+        verdict: decision, accepted: decision.accept,
+        reason: decision.accept ? 'accepted — all gates hold' : `rejected — ${decision.failed.join(', ')}`,
+    };
+    // 6. EMIT the champion manifest (UNSIGNED) on acceptance.
+    if (decision.accept && canaryCmp.pass) {
+        result.manifest = {
+            schema: 'ruflo.proven-config/v1',
+            policy: { ref: opts.policyRefOf ? opts.policyRefOf(opts.candidate) : 'sha256:unknown' },
+            layer: opts.layer,
+            compatibility: { ruflo: '>=3.24.0' },
+            benchmark: { corpus: opts.corpus.version, corpusHash: opts.corpus.corpusHash },
+            receipt: {
+                heldOutDelta: candidateScore - baselineScore,
+                redblue: verify.redblue,
+                drift: verify.drift,
+                canary: { rollbackRate: canaryCand.rollbackRate, latencyP95: canaryCand.latencyP95, costPerTask: canaryCand.costPerTask },
+                receiptCoverage: 1,
+            },
+        };
+    }
+    else if (decision.accept && !canaryCmp.pass) {
+        result.accepted = false;
+        result.reason = `rejected — canary regressed (${canaryCmp.failed.join(', ')})`;
+    }
+    return result;
+}
+//# sourceMappingURL=harness-loop.js.map