claude-flow 3.23.0 → 3.24.0

package/v3/@claude-flow/cli/dist/src/services/harness-benchmark.js

@@ -0,0 +1,94 @@
+/**
+ * Benchmark corpus + held-out scoring + the accept() promotion gate
+ * (ADR-176 phase 2 + the multi-term promotion rule).
+ *
+ * Proof #1 (measured, held-out): a candidate is scored against a VERSIONED,
+ * content-hashed corpus on a held-out split — the "prove-before-ship" pattern
+ * distill-tuning.ts established (train/held-out disjoint by index; held-out
+ * scored once). Fitness is a composite, not a single gameable number.
+ *
+ * Promotion is a CONJUNCTION of externally-measurable predicates (ADR-176) —
+ * never a scalar. accept() takes the independently-measured verdicts (held-out
+ * delta, redblue, drift, deterministic replay, receipt coverage, canary
+ * rollback) and admits only when EVERY term holds. Multi-dimensional +
+ * Goodhart-resistant. Zero deps, $0.
+ */
+import { createHash } from 'crypto';
+function stable(v) {
+    const c = (x) => {
+        if (Array.isArray(x))
+            return x.map(c);
+        if (x && typeof x === 'object') {
+            const o = {};
+            for (const k of Object.keys(x).sort())
+                o[k] = c(x[k]);
+            return o;
+        }
+        return x;
+    };
+    return JSON.stringify(c(v));
+}
+/** Content hash of a corpus's tasks — the tamper-evident pin for the held-out set. */
+export function hashCorpus(tasks) {
+    const canon = tasks.map(t => ({ id: t.id, input: t.input, expected: t.expected, weight: t.weight ?? 1 }))
+        .sort((a, b) => a.id.localeCompare(b.id));
+    return 'sha256:' + createHash('sha256').update(stable(canon)).digest('hex');
+}
+/** Verify a corpus's declared hash matches its tasks (integrity of the benchmark). */
+export function verifyCorpus(corpus) {
+    return hashCorpus(corpus.tasks) === corpus.corpusHash;
+}
+/** Score a candidate over a task set, on isolated tasks (no shared state). */
+export function scoreOnTasks(tasks, candidate, evalFn, gradeFn) {
+    if (tasks.length === 0)
+        return { fitness: 0, passRate: 0, n: 0 };
+    let weighted = 0, totalW = 0, passes = 0;
+    for (const t of tasks) {
+        const w = t.weight ?? 1;
+        let g = 0;
+        try {
+            g = Math.max(0, Math.min(1, gradeFn(evalFn(t.input, candidate), t.expected)));
+        }
+        catch {
+            g = 0;
+        }
+        weighted += g * w;
+        totalW += w;
+        if (g >= 0.999)
+            passes++;
+    }
+    return { fitness: totalW > 0 ? weighted / totalW : 0, passRate: passes / tasks.length, n: tasks.length };
+}
+/**
+ * Deterministic held-out split. Tasks are stably ordered by id, and the last
+ * `holdoutFrac` become the held-out set — disjoint from train, reproducible
+ * (so two runs converge, per the ADR-176 acceptance test).
+ */
+export function computeHeldOutSplit(tasks, holdoutFrac = 0.2) {
+    const ordered = [...tasks].sort((a, b) => a.id.localeCompare(b.id));
+    const cut = Math.max(0, ordered.length - Math.max(1, Math.round(ordered.length * holdoutFrac)));
+    return { train: ordered.slice(0, cut), heldOut: ordered.slice(cut) };
+}
+/**
+ * accept(candidate) ⟺
+ *   held_out_score  >  baseline
+ *   AND redblue     == PASS
+ *   AND drift       <= threshold
+ *   AND replay      == deterministic
+ *   AND receipt_cov == 100%
+ *   AND canary.rollback_rate <= baseline
+ * Every term is externally measurable; ANY failure → reject.
+ */
+export function accept(v) {
+    const terms = {
+        held_out_improves: { value: `${v.heldOutScore.toFixed(4)} > ${v.baselineHeldOutScore.toFixed(4)}`, pass: v.heldOutScore > v.baselineHeldOutScore },
+        redblue_pass: { value: v.redblue, pass: v.redblue === 'PASS' },
+        drift_within: { value: `${v.drift} <= ${v.driftThreshold}`, pass: v.drift <= v.driftThreshold },
+        replay_deterministic: { value: v.replayDeterministic, pass: v.replayDeterministic === true },
+        receipt_coverage_full: { value: v.receiptCoverage, pass: v.receiptCoverage >= 1 },
+        canary_no_worse: { value: `${v.canaryRollbackRate} <= ${v.baselineRollbackRate}`, pass: v.canaryRollbackRate <= v.baselineRollbackRate },
+    };
+    const failed = Object.entries(terms).filter(([, t]) => !t.pass).map(([k]) => k);
+    return { accept: failed.length === 0, terms, failed };
+}
+//# sourceMappingURL=harness-benchmark.js.map

package/v3/@claude-flow/cli/dist/src/services/harness-canary.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Canary — separate promotion from deployment (ADR-176 phase 5).
+ *
+ * Held-out evaluation proves a candidate on FROZEN data; it has not observed
+ * real-world behavior. The canary runs the candidate on a bounded, reversible
+ * SLICE of live work and measures rollback rate, latency, cost, failure
+ * frequency, and acceptance — the telemetry the accept() conjunction needs
+ * (canary.rollback_rate <= baseline). Only after canary evidence does PROMOTE
+ * fire. This is what stops benchmark-specific evolution from reaching global
+ * rollout. Zero deps, $0 (the runner is injected; a real one meters real work).
+ */
+/** One canary run's observed outcome. */
+export interface CanaryOutcome {
+    ok: boolean;
+    rolledBack: boolean;
+    latencyMs: number;
+    costUsd: number;
+    accepted: boolean;
+}
+/** Executes a candidate on one task input under real (or simulated) conditions. */
+export type CanaryRunner<C> = (input: unknown, candidate: C) => CanaryOutcome;
+export interface CanaryTelemetry {
+    n: number;
+    rollbackRate: number;
+    failureRate: number;
+    acceptanceRate: number;
+    latencyP95: number;
+    latencyMean: number;
+    costPerTask: number;
+    costTotalUsd: number;
+}
+export interface CanaryOptions {
+    sampleFraction?: number;
+    maxSamples?: number;
+}
+/**
+ * Run the candidate on a bounded, deterministic sample of the slice and
+ * aggregate telemetry. Deterministic sampling (stride over id-sorted inputs) so
+ * two runs converge (ADR-176 acceptance test).
+ */
+export declare function runCanary<C>(candidate: C, slice: Array<{
+    id: string;
+    input: unknown;
+}>, runner: CanaryRunner<C>, opts?: CanaryOptions): CanaryTelemetry;
+export interface CanaryComparison {
+    pass: boolean;
+    checks: Record<string, {
+        candidate: number;
+        baseline: number;
+        pass: boolean;
+    }>;
+    failed: string[];
+}
+/**
+ * Canary gate: the candidate must be NO WORSE than the baseline on rollback,
+ * latency, and cost (the ADR-176 metrics table constraints). Feeds accept()'s
+ * `canary.rollback_rate <= baseline` term plus the latency/cost guards.
+ */
+export declare function compareCanary(candidate: CanaryTelemetry, baseline: CanaryTelemetry, tolerance?: number): CanaryComparison;
+//# sourceMappingURL=harness-canary.d.ts.map

package/v3/@claude-flow/cli/dist/src/services/harness-canary.js

@@ -0,0 +1,69 @@
+/**
+ * Canary — separate promotion from deployment (ADR-176 phase 5).
+ *
+ * Held-out evaluation proves a candidate on FROZEN data; it has not observed
+ * real-world behavior. The canary runs the candidate on a bounded, reversible
+ * SLICE of live work and measures rollback rate, latency, cost, failure
+ * frequency, and acceptance — the telemetry the accept() conjunction needs
+ * (canary.rollback_rate <= baseline). Only after canary evidence does PROMOTE
+ * fire. This is what stops benchmark-specific evolution from reaching global
+ * rollout. Zero deps, $0 (the runner is injected; a real one meters real work).
+ */
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
+    return sorted[idx];
+}
+/**
+ * Run the candidate on a bounded, deterministic sample of the slice and
+ * aggregate telemetry. Deterministic sampling (stride over id-sorted inputs) so
+ * two runs converge (ADR-176 acceptance test).
+ */
+export function runCanary(candidate, slice, runner, opts = {}) {
+    const frac = opts.sampleFraction ?? 0.1;
+    const cap = opts.maxSamples ?? 100;
+    const ordered = [...slice].sort((a, b) => a.id.localeCompare(b.id));
+    const want = Math.min(cap, Math.max(1, Math.round(ordered.length * frac)));
+    // Even stride so the sample spans the slice rather than a contiguous head.
+    const stride = Math.max(1, Math.floor(ordered.length / want));
+    const sample = [];
+    for (let i = 0; i < ordered.length && sample.length < want; i += stride)
+        sample.push(ordered[i]);
+    const outcomes = sample.map(s => {
+        try {
+            return runner(s.input, candidate);
+        }
+        catch {
+            return { ok: false, rolledBack: true, latencyMs: 0, costUsd: 0, accepted: false };
+        }
+    });
+    const n = outcomes.length || 1;
+    const lat = outcomes.map(o => o.latencyMs).sort((a, b) => a - b);
+    const costTotal = outcomes.reduce((s, o) => s + o.costUsd, 0);
+    return {
+        n: outcomes.length,
+        rollbackRate: outcomes.filter(o => o.rolledBack).length / n,
+        failureRate: outcomes.filter(o => !o.ok).length / n,
+        acceptanceRate: outcomes.filter(o => o.accepted).length / n,
+        latencyP95: percentile(lat, 0.95),
+        latencyMean: lat.reduce((s, v) => s + v, 0) / n,
+        costPerTask: costTotal / n,
+        costTotalUsd: costTotal,
+    };
+}
+/**
+ * Canary gate: the candidate must be NO WORSE than the baseline on rollback,
+ * latency, and cost (the ADR-176 metrics table constraints). Feeds accept()'s
+ * `canary.rollback_rate <= baseline` term plus the latency/cost guards.
+ */
+export function compareCanary(candidate, baseline, tolerance = 1e-9) {
+    const checks = {
+        rollback_no_worse: { candidate: candidate.rollbackRate, baseline: baseline.rollbackRate, pass: candidate.rollbackRate <= baseline.rollbackRate + tolerance },
+        latency_no_worse: { candidate: candidate.latencyP95, baseline: baseline.latencyP95, pass: candidate.latencyP95 <= baseline.latencyP95 * (1 + 0.01) },
+        cost_no_worse: { candidate: candidate.costPerTask, baseline: baseline.costPerTask, pass: candidate.costPerTask <= baseline.costPerTask * (1 + 0.01) },
+    };
+    const failed = Object.entries(checks).filter(([, c]) => !c.pass).map(([k]) => k);
+    return { pass: failed.length === 0, checks, failed };
+}
+//# sourceMappingURL=harness-canary.js.map

package/v3/@claude-flow/cli/dist/src/services/harness-corpus-harvester.d.ts

@@ -0,0 +1,51 @@
+export interface HarvestPattern {
+    id: string;
+    name?: string;
+    content?: string;
+}
+export interface HarvestedTask {
+    id: string;
+    input: {
+        id: string;
+        q: string;
+        targetId: string;
+    };
+    expected: string;
+    provenanceTier: 'oracle:self-identity';
+}
+export interface BlendedCorpus {
+    version: string;
+    corpusHash: string;
+    anchorIds: string[];
+    tasks: Array<{
+        id: string;
+        input: unknown;
+        expected: unknown;
+    }>;
+}
+/**
+ * Harvest up to `sample` self-retrieval tasks from real patterns. Deterministic
+ * stride sampling (stable across runs); skips docs whose body is too thin to
+ * form a discriminative query.
+ */
+export declare function harvestSelfSupervisedTasks(patterns: HarvestPattern[], opts?: {
+    sample?: number;
+    minQueryTerms?: number;
+}): HarvestedTask[];
+/** Content hash over the task ids+queries — changes iff the corpus changes. */
+export declare function hashBlend(tasks: Array<{
+    id: string;
+    input: unknown;
+    expected: unknown;
+}>): string;
+/**
+ * Blend a human-labeled anchor set with harvested tasks into one versioned,
+ * hashed corpus. The version encodes the sizes + hash so it visibly grows as the
+ * store does; `anchorIds` marks the never-regress subset.
+ */
+export declare function blendCorpus(anchor: Array<{
+    id: string;
+    input: unknown;
+    expected: unknown;
+}>, harvested: HarvestedTask[]): BlendedCorpus;
+//# sourceMappingURL=harness-corpus-harvester.d.ts.map

package/v3/@claude-flow/cli/dist/src/services/harness-corpus-harvester.js

@@ -0,0 +1,74 @@
+/**
+ * Corpus harvester — grows the benchmark yardstick from REAL store data so the
+ * flywheel gets a bigger, fresher test set as ruflo is used (ADR-176).
+ *
+ * Self-supervised self-retrieval: a stored doc is unambiguous ground truth for a
+ * query derived from its OWN body. To make it discriminative (so different
+ * retrieval configs actually score differently — a usable gradient) the query is
+ * built from body keywords with the doc's SUBJECT tokens removed: "can retrieval
+ * find this doc from its content when the obvious title words are withheld?"
+ * The label is the doc's identity — oracle-grade, not a proxy guess.
+ *
+ * This grows the auto signal; callers keep the human-labeled seed as a separate
+ * NON-REGRESSION anchor (Goodhart guard) so optimizing self-retrieval can never
+ * silently drift away from human relevance. Pure + deterministic (no RNG): the
+ * sample is a fixed stride, so the same store yields the same corpus.
+ */
+import { createHash } from 'node:crypto';
+const STOP = new Set(['the', 'a', 'an', 'of', 'to', 'in', 'on', 'for', 'and', 'or', 'is', 'was', 'with', 'by', 'at', 'as', 'it', 'this', 'that', 'from', 'be', 'are', 'so', 'if', 'we', 'i', 'you', 'not', 'no', 'do', 'via', 'per']);
+function tokenize(s) {
+    return (s.toLowerCase().match(/[a-z0-9][a-z0-9_-]{1,}/g) ?? []).filter((t) => !STOP.has(t));
+}
+/** Top-K body keywords with the subject tokens removed, most-frequent first. */
+function deriveQuery(name, content, k = 6) {
+    const nameToks = new Set(tokenize(name));
+    const freq = new Map();
+    for (const t of tokenize(content)) {
+        if (nameToks.has(t))
+            continue; // withhold the obvious title words
+        freq.set(t, (freq.get(t) ?? 0) + 1);
+    }
+    return [...freq.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, k).map(([t]) => t).join(' ');
+}
+/**
+ * Harvest up to `sample` self-retrieval tasks from real patterns. Deterministic
+ * stride sampling (stable across runs); skips docs whose body is too thin to
+ * form a discriminative query.
+ */
+export function harvestSelfSupervisedTasks(patterns, opts = {}) {
+    const sample = Math.max(1, opts.sample ?? 40);
+    const minTerms = opts.minQueryTerms ?? 3;
+    // Deterministic order: sort by id, then take an even stride to cover the store.
+    const ordered = [...patterns].sort((a, b) => a.id.localeCompare(b.id));
+    const stride = Math.max(1, Math.floor(ordered.length / sample));
+    const out = [];
+    for (let i = 0; i < ordered.length && out.length < sample; i += stride) {
+        const p = ordered[i];
+        const q = deriveQuery(p.name ?? '', p.content ?? '', 6);
+        if (q.split(' ').filter(Boolean).length < minTerms)
+            continue; // too thin → skip
+        out.push({ id: `hv-${p.id}`, input: { id: `hv-${p.id}`, q, targetId: p.id }, expected: p.id, provenanceTier: 'oracle:self-identity' });
+    }
+    return out;
+}
+/** Content hash over the task ids+queries — changes iff the corpus changes. */
+export function hashBlend(tasks) {
+    const canon = tasks.map((t) => `${t.id}|${JSON.stringify(t.input)}|${JSON.stringify(t.expected)}`).sort().join('\n');
+    return 'sha256:' + createHash('sha256').update(canon).digest('hex');
+}
+/**
+ * Blend a human-labeled anchor set with harvested tasks into one versioned,
+ * hashed corpus. The version encodes the sizes + hash so it visibly grows as the
+ * store does; `anchorIds` marks the never-regress subset.
+ */
+export function blendCorpus(anchor, harvested) {
+    const tasks = [...anchor, ...harvested.map((h) => ({ id: h.id, input: h.input, expected: h.expected }))];
+    const corpusHash = hashBlend(tasks);
+    return {
+        version: `flywheel-a${anchor.length}-h${harvested.length}-${corpusHash.slice(7, 19)}`,
+        corpusHash,
+        anchorIds: anchor.map((a) => a.id),
+        tasks,
+    };
+}
+//# sourceMappingURL=harness-corpus-harvester.js.map

package/v3/@claude-flow/cli/dist/src/services/harness-flywheel-generations.d.ts

@@ -0,0 +1,111 @@
+import { type EvolveReceiptBundle, type LineageTelemetry, type PlateauReport, type MutationStat } from './evolve-proof.js';
+import { type HarvestPattern } from './harness-corpus-harvester.js';
+import { type RetrievalConfig, type RankedItem } from './harness-flywheel.js';
+export declare const FLYWHEEL_DIR: string[];
+export declare const FROZEN_CORPUS = "harvested-selfsup-frozen-v1";
+export interface AnchorTask {
+    id: string;
+    q: string;
+    labels: string[];
+}
+export interface GenerationDeps {
+    getPatterns: () => HarvestPattern[] | Promise<HarvestPattern[]>;
+    search: (q: string, cfg: RetrievalConfig) => Promise<RankedItem[]> | RankedItem[];
+    anchorTasks: AnchorTask[];
+    sample?: number;
+    now: number;
+    applyFn?: (cfg: Record<string, number>, hash: string, generation: number) => void;
+}
+export interface GenerationResult {
+    ran: boolean;
+    reason: string;
+    generation: number;
+    promoted?: boolean;
+    delta?: number;
+    significant?: boolean;
+    championConfig?: Record<string, number>;
+    servedChampion?: string | null;
+    anchorRegressed?: boolean;
+}
+/** Promotions only — the champion chain (generation-N.json), sorted by generation. */
+export declare function loadPromotions(root: string): EvolveReceiptBundle[];
+/** Every attempt (promoted + rejected) — for telemetry + mutation-effectiveness. */
+export declare function loadAttempts(root: string): EvolveReceiptBundle[];
+/** The current operating champion (last promotion's config), or defaults. */
+export declare function currentChampion(root: string): {
+    config: Record<string, number>;
+    hash: string | null;
+    generation: number;
+};
+export interface ServedState {
+    championHash: string | null;
+    config: Record<string, number> | null;
+    servedAt: number | null;
+    fromGeneration: number | null;
+}
+export declare function servedChampion(root: string): ServedState;
+/**
+ * Shadow→serve gate: apply the latest promoted champion to the ACTIVE policy iff
+ * it is newer than what is currently served. Called at tick START, so a champion
+ * promoted last tick is served this tick (a 1-generation shadow delay) — never
+ * auto-served the instant it is promoted.
+ */
+export declare function serveCurrentChampionIfPending(root: string, now: number, applyFn?: GenerationDeps['applyFn']): string | null;
+export interface AxisStat {
+    axis: string;
+    promotions: number;
+    meanDelta: number;
+}
+/**
+ * Which policy AXES have historically paid off — attribute each promotion's
+ * held-out Δ to the axes that changed in it. Turns the lineage into a
+ * knowledge base the optimizer can act on (not just audit).
+ */
+export declare function axisEffectiveness(promotions: EvolveReceiptBundle[]): AxisStat[];
+/**
+ * Candidate set biased by measured axis payoff (meta-learning). Every axis keeps
+ * a ±1 exploration floor (never abandon a dimension), but axes with a positive
+ * historical Δ get EXPANDED range (±2, ±3) and PAIRWISE joint moves with other
+ * productive axes — so compute concentrates on the dimensions that have actually
+ * produced gains, instead of a uniform grid. Deterministic; bounded.
+ */
+export declare function biasedGrid(c: RetrievalConfig, ranking: AxisStat[]): RetrievalConfig[];
+/**
+ * Run ONE generation against `root`, compounding on the persisted champion.
+ * Serves the prior champion first (shadow delay), then evaluates a new candidate.
+ */
+export declare function runFlywheelGeneration(root: string, deps: GenerationDeps): Promise<GenerationResult>;
+export interface DriftCheck {
+    checked: boolean;
+    rolledBack: boolean;
+    reason: string;
+    servedScore?: number;
+    predecessorScore?: number;
+}
+/**
+ * A real deployment-safety check on real (not fabricated) data: the store keeps
+ * changing as ruflo is used, so a champion benchmarked at promotion time can
+ * DRIFT. Each tick, re-score the currently-SERVED champion against its
+ * predecessor on a FRESH harvest of the current store; if it now regresses
+ * (self-retrieval OR the human anchor), automatically ROLL BACK the active policy
+ * to the predecessor. This is the honest analogue of a live-traffic canary —
+ * genuine ongoing measurement + genuine rollback — without fabricating traffic.
+ * $0; never throws.
+ */
+export declare function checkServedChampionDrift(root: string, deps: GenerationDeps): Promise<DriftCheck>;
+export interface FlywheelStatus {
+    generations: number;
+    attempts: number;
+    lineage: LineageTelemetry;
+    plateau: PlateauReport;
+    mutation: MutationStat[];
+    axisEffectiveness: AxisStat[];
+    served: ServedState;
+    champion: {
+        config: Record<string, number>;
+        hash: string | null;
+    };
+}
+/** Reconstruct the persisted lineage + telemetry for a status endpoint / CLI. */
+export declare function flywheelStatus(root: string): FlywheelStatus;
+//# sourceMappingURL=harness-flywheel-generations.d.ts.map