claude-flow 3.20.0 → 3.21.1

package/v3/@claude-flow/cli/dist/src/services/distill-oracle.js

@@ -0,0 +1,349 @@
+/**
+ * distill-oracle.ts — Tiered `resolved` oracle for distill / weight-EFT SFT data.
+ *
+ * THE PROBLEM: weight-EFT's SFT data wants a gold `resolved: boolean` per
+ * trajectory. Ruflo has no SWE-bench oracle, so today `resolved` is
+ * structural-confidence — a proxy that risks distilling plausible-but-wrong
+ * completions. This module replaces that single proxy with a TIERED labeler,
+ * where every label carries HONEST PROVENANCE (ADR-169 reporting integrity — a
+ * proxy is never presented as ground truth).
+ *
+ * ── TIERS (tried in order, per trajectory) ──────────────────────────────────
+ *   TIER 1 — MECHANICAL ORACLE  (provenance `oracle:test-exec`, real ground truth)
+ *     When a trajectory carries a test spec (SWE-bench FAIL_TO_PASS shape) or
+ *     maps to a metaharness/darwin bench-suite case, EXECUTE the real eval and
+ *     set resolved = tests-pass. This needs Docker/compute, so it runs on a
+ *     REMOTE host over SSH (host parameterized via `remote` / env
+ *     RUFLO_DISTILL_REMOTE — never hard-coded). DRY-RUN by default: it prints
+ *     the ssh/darwin-bench/eval commands + a preflight plan and does NOT touch
+ *     the network; only `execute: true` runs the real eval (with a wrapped,
+ *     non-fatal preflight probe first).
+ *
+ *   TIER 2 — FABLE JUDGE  (provenance `judge:fable`, smarter proxy)
+ *     For trajectories with no mechanical spec, judge via the cost-disciplined
+ *     headless Fable harness (fable-harness.ts). Opt-in and OFF by default;
+ *     spends nothing unless `fableJudge: true` AND a `maxBudgetUsd` cap is set.
+ *
+ *   TIER 3 — STRUCTURAL PROXY  (provenance `proxy:structural`, weakest)
+ *     The existing output-verifier structural confidence. Always available,
+ *     $0, no external calls. Explicitly the weakest signal, clearly labeled.
+ *
+ * DEFAULT (no opts): dry-run oracle preflight + structural-proxy fallback.
+ * ZERO spend, no SSH exec, no Fable call. Everything degrades gracefully
+ * (ADR-150) — a missing remote or a failed probe is reported, never fatal.
+ *
+ * @module services/distill-oracle
+ */
+import { spawn } from 'child_process';
+import { verifyAndEscalate } from '../ruvector/output-verifier.js';
+import { FableHarness, } from './fable-harness.js';
+// ── Public API ───────────────────────────────────────────────────────────
+/**
+ * Label each trajectory with `resolved` + honest provenance, trying the tiers
+ * in order per trajectory. See the module header for tier semantics and the
+ * zero-spend default guarantee.
+ */
+export async function labelResolved(trajectories, opts = {}) {
+    const results = new Array(trajectories.length);
+    const preflights = new Map();
+    const remote = resolveRemote(opts.remote);
+    const fableActive = Boolean(opts.fableJudge) && Number(opts.maxBudgetUsd) > 0;
+    // ── Tier 1: mechanical oracle (real exec only under execute:true) ──────────
+    const fableQueue = [];
+    for (let i = 0; i < trajectories.length; i++) {
+        const t = trajectories[i];
+        if (hasMechanicalSpec(t)) {
+            const plan = buildOraclePlan(t.testSpec, remote);
+            if (opts.execute) {
+                const runner = opts.runner ?? createSshOracleRunner();
+                // Wrapped probe: a failure is REPORTED, not thrown.
+                const probe = await safeProbe(runner, plan);
+                if (probe.ok) {
+                    const outcome = await safeRunEval(runner, plan, t.testSpec);
+                    results[i] = {
+                        ...t,
+                        resolved: outcome.resolved,
+                        resolvedBy: 'oracle:test-exec',
+                        resolvedConfidence: 1,
+                        resolvedReason: outcome.reason,
+                        oraclePreflight: { dryRun: false, plan, probe },
+                    };
+                    continue;
+                }
+                // Probe failed → report and fall through to next tier.
+                preflights.set(i, {
+                    dryRun: false,
+                    plan,
+                    probe,
+                    fellThroughBecause: `preflight failed: ${probe.reason}`,
+                });
+            }
+            else {
+                // DRY-RUN: emit the plan/commands, touch nothing, fall through.
+                preflights.set(i, {
+                    dryRun: true,
+                    plan,
+                    fellThroughBecause: remote
+                        ? 'dry-run (pass execute:true to run the real eval over SSH)'
+                        : 'dry-run and no remote configured (set remote / RUFLO_DISTILL_REMOTE)',
+                });
+            }
+        }
+        // ── Route to Tier 2 (batched later) or Tier 3 now ──
+        if (fableActive) {
+            fableQueue.push({ index: i, item: toJudgeItem(t) });
+        }
+        else {
+            results[i] = await proxyLabel(t, opts, preflights.get(i));
+        }
+    }
+    // ── Tier 2: batched Fable judge for queued items ──────────────────────────
+    if (fableActive && fableQueue.length > 0) {
+        const harness = opts.harness ??
+            new FableHarness({ maxBudgetUsd: opts.maxBudgetUsd, batchSize: opts.fableBatchSize });
+        let verdicts = [];
+        try {
+            verdicts = await harness.judgeBatch(fableQueue.map((q) => q.item));
+        }
+        catch {
+            verdicts = []; // graceful: any harness failure ⇒ proxy fallback for all
+        }
+        const byId = new Map(verdicts.map((v) => [v.id, v]));
+        for (const { index, item } of fableQueue) {
+            const v = byId.get(item.id);
+            const t = trajectories[index];
+            if (v) {
+                results[index] = {
+                    ...t,
+                    resolved: v.resolved,
+                    resolvedBy: 'judge:fable',
+                    resolvedConfidence: v.confidence,
+                    resolvedReason: v.reason,
+                    ...(preflights.has(index) ? { oraclePreflight: preflights.get(index) } : {}),
+                };
+            }
+            else {
+                // Model omitted this id / budget exhausted → Tier-3 proxy fallback.
+                results[index] = await proxyLabel(t, opts, preflights.get(index));
+            }
+        }
+    }
+    // ── Safety net: nothing should be undefined, but fall back to proxy if so ──
+    for (let i = 0; i < results.length; i++) {
+        if (results[i] === undefined) {
+            results[i] = await proxyLabel(trajectories[i], opts, preflights.get(i));
+        }
+    }
+    return results;
+}
+/**
+ * Reflective failure analysis (GEPA/evolve mutation input). Thin, cost-
+ * disciplined pass-through to the Fable harness. Opt-in: returns [] unless a
+ * harness with a budget cap is provided/configured.
+ */
+export async function reflectFailures(items, opts = {}) {
+    const harness = opts.harness ??
+        new FableHarness({ maxBudgetUsd: opts.maxBudgetUsd, batchSize: opts.fableBatchSize });
+    if (!harness.isEnabled())
+        return [];
+    return harness.reflectFailures(items);
+}
+// ── Tier-3 structural proxy ──────────────────────────────────────────────
+async function proxyLabel(t, opts, preflight) {
+    const verdict = await verifyAndEscalate({
+        task: t.task ?? '',
+        output: t.output ?? '',
+        taskKind: t.taskKind,
+    });
+    const threshold = opts.minStructuralConfidence;
+    const resolved = typeof threshold === 'number' ? verdict.score >= threshold : verdict.confident;
+    return {
+        ...t,
+        resolved,
+        resolvedBy: 'proxy:structural',
+        resolvedConfidence: verdict.score,
+        resolvedReason: verdict.confident
+            ? 'structural signals confident'
+            : `structural signals weak: ${verdict.reasons.slice(0, 3).join('; ')}`,
+        ...(preflight ? { oraclePreflight: preflight } : {}),
+    };
+}
+// ── Tier-1 planning ──────────────────────────────────────────────────────
+/** A trajectory can be mechanically evaluated iff it carries an actionable spec. */
+export function hasMechanicalSpec(t) {
+    const s = t.testSpec;
+    if (!s)
+        return false;
+    return Boolean((s.failToPass && s.failToPass.length > 0) ||
+        s.evalCommand ||
+        s.benchSuite ||
+        s.benchCase);
+}
+/** Resolve the remote host from opts → env, never a hard-coded host. */
+export function resolveRemote(remote) {
+    const r = (remote ?? process.env.RUFLO_DISTILL_REMOTE ?? '').trim();
+    return r.length > 0 ? r : null;
+}
+/**
+ * Build the concrete command plan for a spec. Pure — constructs command strings
+ * only; executes nothing. `remote` is always substituted from the parameter,
+ * never hard-coded.
+ */
+export function buildOraclePlan(spec, remote) {
+    const host = remote; // may be null; commands render `<remote>` placeholder then
+    const r = host ?? '<remote>';
+    const sshLocal = host ? `ssh ${host} ` : `ssh <remote> `;
+    const workdir = spec.workdir ?? (spec.repo ? `~/ruflo-distill/${sanitize(spec.repo)}` : '~/ruflo-distill/work');
+    // Preflight probes — reachability + the tools Tier-1 needs on the remote.
+    const probeCommands = [
+        `ssh -o BatchMode=yes -o ConnectTimeout=8 ${r} 'echo ok'`,
+        `${sshLocal}'command -v docker >/dev/null 2>&1 && echo docker-present || echo docker-missing'`,
+        `${sshLocal}'command -v darwin >/dev/null 2>&1 || npx --yes @metaharness/darwin --version'`,
+    ];
+    let kind;
+    const evalCommands = [];
+    let parseHint;
+    if (spec.benchSuite || spec.benchCase) {
+        kind = 'ssh-darwin-bench';
+        const suite = spec.benchSuite ? `--suite ${shq(spec.benchSuite)}` : '';
+        const kase = spec.benchCase ? `--case ${shq(spec.benchCase)}` : '';
+        evalCommands.push(`${sshLocal}'cd ${workdir} && npx --yes @metaharness/darwin bench run ${suite} ${kase} --json'`.replace(/\s+/g, ' ').trim());
+        parseHint = 'resolved = darwin bench run reports the case scored PASS (exit 0, json.passed=true)';
+    }
+    else if (spec.evalCommand) {
+        kind = 'ssh-eval';
+        evalCommands.push(`${sshLocal}'cd ${workdir} && ${spec.evalCommand}'`);
+        parseHint = 'resolved = evalCommand exits 0';
+    }
+    else {
+        kind = 'ssh-swebench-eval';
+        if (spec.patch) {
+            evalCommands.push(`${sshLocal}'cd ${workdir} && git apply - <<"PATCH"\n${spec.patch}\nPATCH'`);
+        }
+        const f2p = (spec.failToPass ?? []).map(shq).join(' ');
+        const p2p = (spec.passToPass ?? []).map(shq).join(' ');
+        evalCommands.push(`${sshLocal}'cd ${workdir} && npx --yes @metaharness/darwin swebench-eval` +
+            `${spec.repo ? ` --repo ${shq(spec.repo)}` : ''}` +
+            `${spec.baseCommit ? ` --base ${shq(spec.baseCommit)}` : ''}` +
+            `${f2p ? ` --fail-to-pass ${f2p}` : ''}` +
+            `${p2p ? ` --pass-to-pass ${p2p}` : ''} --json'`);
+        parseHint = 'resolved = all FAIL_TO_PASS now pass AND all PASS_TO_PASS still pass';
+    }
+    return { kind, remote: host, probeCommands, evalCommands, parseHint };
+}
+// ── Default SSH runner (real exec; injectable for tests) ─────────────────────
+/**
+ * Default command executor: shells out, piping any heredoc/stdin via argv only
+ * (commands are self-contained strings run through `sh -c`). Never invoked on
+ * the default (dry-run) path.
+ */
+export const defaultCommandExec = (cmd, args, opts) => new Promise((resolve) => {
+    let child;
+    try {
+        child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true });
+    }
+    catch (err) {
+        resolve({ stdout: '', stderr: err instanceof Error ? err.message : String(err), code: null });
+        return;
+    }
+    let stdout = '';
+    let stderr = '';
+    let done = false;
+    const finish = (r) => { if (!done) {
+        done = true;
+        clearTimeout(timer);
+        resolve(r);
+    } };
+    const timer = setTimeout(() => {
+        try {
+            child.kill('SIGTERM');
+        }
+        catch { /* dead */ }
+        finish({ stdout, stderr: stderr || `timed out after ${opts.timeoutMs}ms`, code: null });
+    }, opts.timeoutMs);
+    timer.unref?.();
+    child.stdout?.on('data', (d) => { stdout += d.toString(); });
+    child.stderr?.on('data', (d) => { stderr += d.toString(); });
+    child.on('error', (e) => finish({ stdout, stderr: e.message, code: null }));
+    child.on('close', (code) => finish({ stdout, stderr, code }));
+});
+/**
+ * Build a real SSH-backed oracle runner. `exec` is injectable so tests assert
+ * command construction without ever touching a network. Each plan command is
+ * run via `sh -c <command-string>` so the fully-rendered ssh line executes as-is.
+ */
+export function createSshOracleRunner(exec = defaultCommandExec, timeoutMs = 15 * 60 * 1000) {
+    const run = (command) => exec('sh', ['-c', command], { timeoutMs });
+    return {
+        async preflight(plan) {
+            if (!plan.remote)
+                return { ok: false, reason: 'no remote configured' };
+            for (const c of plan.probeCommands) {
+                const res = await run(c);
+                if (res.code !== 0) {
+                    return { ok: false, reason: `probe failed (${c.slice(0, 60)}…): ${(res.stderr || '').slice(0, 120)}` };
+                }
+            }
+            return { ok: true, reason: 'remote reachable; docker/darwin present' };
+        },
+        async runEval(plan) {
+            let lastReason = 'no eval commands';
+            for (const c of plan.evalCommands) {
+                const res = await run(c);
+                if (res.code !== 0) {
+                    return { resolved: false, reason: `eval failed (exit ${res.code}): ${(res.stderr || res.stdout || '').slice(0, 160)}` };
+                }
+                lastReason = interpretEvalStdout(res.stdout);
+                if (lastReason.startsWith('NOT_RESOLVED')) {
+                    return { resolved: false, reason: lastReason };
+                }
+            }
+            return { resolved: true, reason: `${plan.parseHint} — ${lastReason}` };
+        },
+    };
+}
+/** Interpret an eval command's JSON stdout for an explicit pass/fail signal. */
+function interpretEvalStdout(stdout) {
+    const trimmed = (stdout ?? '').trim();
+    const m = trimmed.match(/\{[\s\S]*\}/);
+    if (m) {
+        try {
+            const o = JSON.parse(m[0]);
+            if (o.passed === false || o.resolved === false)
+                return 'NOT_RESOLVED (json.passed=false)';
+            if (o.passed === true || o.resolved === true)
+                return 'json.passed=true';
+        }
+        catch { /* fall through to exit-code semantics */ }
+    }
+    return 'exit 0';
+}
+// ── small helpers ────────────────────────────────────────────────────────
+async function safeProbe(runner, plan) {
+    try {
+        return await runner.preflight(plan);
+    }
+    catch (err) {
+        return { ok: false, reason: `probe threw: ${err instanceof Error ? err.message : String(err)}` };
+    }
+}
+async function safeRunEval(runner, plan, spec) {
+    try {
+        return await runner.runEval(plan, spec);
+    }
+    catch (err) {
+        return { resolved: false, reason: `eval threw: ${err instanceof Error ? err.message : String(err)}` };
+    }
+}
+function toJudgeItem(t) {
+    return { id: t.id, task: t.task ?? '', output: t.output ?? '' };
+}
+function sanitize(s) {
+    return s.replace(/[^A-Za-z0-9._-]/g, '_');
+}
+/** Single-quote a token for safe interpolation into a shell command string. */
+function shq(s) {
+    return `'${String(s).replace(/'/g, `'\\''`)}'`;
+}
+export default labelResolved;
+//# sourceMappingURL=distill-oracle.js.map

package/v3/@claude-flow/cli/dist/src/services/fable-harness.d.ts

@@ -0,0 +1,168 @@
+/**
+ * fable-harness.ts — Cost-disciplined headless Fable LLM-as-judge harness.
+ *
+ * This is TIER 2 of the tiered `resolved` oracle (see distill-oracle.ts). When
+ * a trajectory has NO mechanical test spec to execute, we judge whether its
+ * completion actually resolved the task with a headless Fable model — a smarter
+ * proxy than the structural verifier, but still a proxy (provenance tag
+ * `judge:fable`, never presented as ground truth per ADR-169).
+ *
+ * ── MEASURED COST DATA (load-bearing — this file is built around it) ────────
+ * `claude -p --model claude-fable-5 --output-format json` costs, per call:
+ *   • ~$1.56  when launched FROM THE PROJECT DIR — it auto-loads CLAUDE.md and
+ *             ~56k cache tokens of project context we do NOT want for judging.
+ *   • ~$0.34  from a CLEAN empty cwd with `--append-system-prompt` for the role.
+ *   • ~$0.02/item when we BATCH ~20 items into a single call (context amortizes
+ *             ~free across the batch).
+ *
+ * Therefore this harness MUST, and does:
+ *   1. run `claude -p` from a FRESH EMPTY TEMP cwd — never the project dir, so
+ *      no CLAUDE.md / project context is loaded (the 5x cost driver);
+ *   2. carry the judge/reflect role via `--append-system-prompt`, not a project
+ *      system prompt;
+ *   3. BATCH N items per call (default 20) so per-item cost collapses to ~$0.02;
+ *   4. pass `--max-budget-usd` as a hard per-call cap and stop launching batches
+ *      once the cumulative measured spend reaches the caller's budget;
+ *   5. be OPT-IN and OFF BY DEFAULT — nothing here runs unless a caller
+ *      explicitly constructs the harness AND provides a budget cap.
+ *
+ * SAFETY: constructing this class spends nothing. Only `judgeBatch` /
+ * `reflectFailures` spawn `claude`, and only when `maxBudgetUsd > 0`. The
+ * `spawnClaude` implementation is injectable so tests never touch the real CLI.
+ *
+ * @module services/fable-harness
+ */
+export declare const FABLE_COST_MODEL: {
+    /** Measured $/call when launched from the project dir (loads CLAUDE.md). Anti-pattern. */
+    readonly perCallProjectCwdUsd: 1.56;
+    /** Measured $/call from a clean cwd with --append-system-prompt. */
+    readonly perCallCleanCwdUsd: 0.34;
+    /** Measured amortized $/item when batching ~20 items per call. */
+    readonly perItemBatchedUsd: 0.02;
+    /** Default items per `claude -p` call. */
+    readonly defaultBatchSize: 20;
+    /** The judge model. */
+    readonly model: "claude-fable-5";
+};
+/**
+ * Estimate the USD cost of judging `itemCount` items in batches of `batchSize`.
+ * Uses the measured amortized per-item figure; callers use this to size a
+ * budget cap before opting in.
+ */
+export declare function estimateFableCostUsd(itemCount: number, batchSize?: number): number;
+/** One item to judge: did `output` actually resolve `task`? */
+export interface JudgeItem {
+    id: string;
+    task: string;
+    output: string;
+}
+/** Verdict for a single judged item. */
+export interface JudgeResult {
+    id: string;
+    resolved: boolean;
+    /** 0..1 self-reported judge confidence. */
+    confidence: number;
+    reason: string;
+}
+/** One item for reflective failure analysis (GEPA/evolve mutation input). */
+export interface ReflectItem {
+    id: string;
+    task: string;
+    output: string;
+    /** Optional signal that this trajectory is believed to have failed. */
+    failureHint?: string;
+}
+/** Reflective diagnosis for a single item (the reflective-mutation SOTA trick). */
+export interface ReflectResult {
+    id: string;
+    failureClass: string;
+    diagnosis: string;
+    mutationHint: string;
+}
+/** Result of a raw claude spawn. */
+export interface ClaudeSpawnResult {
+    stdout: string;
+    stderr: string;
+    code: number | null;
+    /** Measured spend for this call, parsed from the JSON envelope when present. */
+    costUsd?: number;
+}
+/**
+ * Injectable spawner. Receives the argv (after `claude`), the prompt to pipe to
+ * stdin, and the cwd (a fresh empty temp dir). Default implementation shells out
+ * to the real `claude` CLI; tests inject a fake.
+ */
+export type ClaudeSpawnFn = (argv: string[], stdinPrompt: string, cwd: string, opts: {
+    timeoutMs: number;
+}) => Promise<ClaudeSpawnResult>;
+export interface FableHarnessOptions {
+    /** Model id (default claude-fable-5). */
+    model?: string;
+    /** Items per `claude -p` call (default 20). */
+    batchSize?: number;
+    /**
+     * Hard budget cap in USD across all calls this harness makes. REQUIRED to be
+     * > 0 for any spend to happen — 0/undefined means the harness refuses to
+     * spawn (safe default).
+     */
+    maxBudgetUsd?: number;
+    /** Per-call timeout (default 5 min). */
+    timeoutMs?: number;
+    /** Injected spawner for tests; defaults to the real `claude` CLI. */
+    spawnClaude?: ClaudeSpawnFn;
+}
+/**
+ * Default `claude -p` spawner. Pipes the prompt via stdin (never as an argv
+ * positional — mirrors the #1852 fix so shell metachars in prompts are never
+ * re-tokenized), runs in the provided (temp) cwd, and returns stdout/stderr.
+ * Parses `total_cost_usd` from the `--output-format json` envelope when present.
+ */
+export declare const defaultSpawnClaude: ClaudeSpawnFn;
+/** Pull `total_cost_usd`/`cost_usd` out of a claude `--output-format json` envelope. */
+export declare function parseCostFromEnvelope(stdout: string): number | undefined;
+export declare class FableHarness {
+    private readonly model;
+    private readonly batchSize;
+    private readonly maxBudgetUsd;
+    private readonly timeoutMs;
+    private readonly spawnClaude;
+    private spentUsd;
+    constructor(opts?: FableHarnessOptions);
+    /** Cumulative measured spend across all calls this harness has made. */
+    getSpentUsd(): number;
+    /** True when a positive budget cap is configured (a precondition for any spend). */
+    isEnabled(): boolean;
+    /**
+     * Judge a set of items in batches. Returns one JudgeResult per input id that
+     * the model returned. Items that fall outside the budget, or that the model
+     * omits, are simply absent from the result — the caller (distill-oracle) is
+     * responsible for falling back to the structural proxy for those.
+     *
+     * Spends $0 and returns [] when no budget cap is configured.
+     */
+    judgeBatch(items: JudgeItem[]): Promise<JudgeResult[]>;
+    /**
+     * Reflective failure analysis over items — the second cost-disciplined entry
+     * point, used by GEPA/evolve for mutation hints. Same batching + budget
+     * discipline as judgeBatch. Returns [] when no budget cap is configured.
+     */
+    reflectFailures(items: ReflectItem[]): Promise<ReflectResult[]>;
+    /** Build the argv for a `claude -p` call. Exposed shape for testability. */
+    buildArgv(systemPrompt: string): string[];
+    /**
+     * Run one batch: create a FRESH EMPTY temp cwd (critical — no project
+     * context), spawn `claude -p` there with the role via --append-system-prompt,
+     * pipe the batch JSON to stdin, parse the verdict array out of the envelope,
+     * and account the measured spend.
+     */
+    private runBatch;
+}
+/**
+ * Extract the model's verdict array. `claude -p --output-format json` wraps the
+ * assistant text in an envelope `{ result: "<text>", ... }`; the text is itself
+ * the JSON array we asked for. Handle both the enveloped and bare forms, and
+ * arrays fenced in ```json blocks.
+ */
+export declare function extractVerdictArray(stdout: string): unknown[];
+export default FableHarness;
+//# sourceMappingURL=fable-harness.d.ts.map