@ggui-ai/negotiator 0.2.0-alpha.3 → 0.2.0-alpha.4

package/dist/synth-bench/run-bench-cli.js

@@ -18,84 +18,9 @@
  *
  * Bench-only — not exported from the package index.
  */
-import { readFileSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { resolve as pathResolve } from 'node:path';
 import { evaluateAgainstCorpus, formatBenchReport } from './run-bench.js';
 import { contractShape } from './corpus.js';
-const ANTHROPIC_API = 'https://api.anthropic.com/v1/messages';
-const DEFAULT_MODEL = 'claude-haiku-4-5';
-function resolveAnthropicKey() {
-    const envKey = process.env['ANTHROPIC_API_KEY'];
-    if (envKey && envKey.length > 0)
-        return envKey;
-    const credsPath = pathResolve(homedir(), '.ggui', 'credentials.json');
-    let parsed;
-    try {
-        parsed = JSON.parse(readFileSync(credsPath, 'utf8'));
-    }
-    catch (err) {
-        throw new Error(`bench-synth: could not read ${credsPath} (${err instanceof Error ? err.message : String(err)}). Set ANTHROPIC_API_KEY env var or run \`ggui auth set anthropic\`.`);
-    }
-    const key = parsed.apps?.global?.anthropic;
-    if (typeof key !== 'string' || key.length === 0) {
-        throw new Error(`bench-synth: no anthropic key found at apps.global.anthropic in ${credsPath}.`);
-    }
-    return key;
-}
-let totalInputTokens = 0;
-let totalOutputTokens = 0;
-function buildAnthropicLlmCaller(apiKey, model) {
-    return {
-        async call() {
-            throw new Error('bench-synth: text-mode not exercised — synth uses callStructured');
-        },
-        async callStructured(systemPrompt, userMessage, tool, maxTokens) {
-            // `temperature` deprecated on Haiku 4.5+ — Anthropic rejects with
-            // HTTP 400. `tool_choice: { type: 'tool', name }` below already
-            // binds output to the input_schema; residual stochasticity stays
-            // bounded via canonical-key normalization downstream.
-            const body = {
-                model,
-                max_tokens: maxTokens ?? 1024,
-                system: systemPrompt,
-                messages: [{ role: 'user', content: userMessage }],
-                tools: [
-                    {
-                        name: tool.name,
-                        description: tool.description,
-                        input_schema: tool.input_schema,
-                    },
-                ],
-                tool_choice: { type: 'tool', name: tool.name },
-            };
-            const res = await fetch(ANTHROPIC_API, {
-                method: 'POST',
-                headers: {
-                    'content-type': 'application/json',
-                    'x-api-key': apiKey,
-                    'anthropic-version': '2023-06-01',
-                },
-                body: JSON.stringify(body),
-            });
-            const json = (await res.json());
-            if (!res.ok) {
-                const errType = json.error?.type ?? 'unknown';
-                const errMsg = json.error?.message ?? `HTTP ${res.status}`;
-                throw new Error(`anthropic ${errType}: ${errMsg}`);
-            }
-            if (json.usage) {
-                totalInputTokens += json.usage.input_tokens ?? 0;
-                totalOutputTokens += json.usage.output_tokens ?? 0;
-            }
-            const toolBlock = json.content?.find((b) => b.type === 'tool_use');
-            if (!toolBlock || toolBlock.input === undefined) {
-                throw new Error(`anthropic: no tool_use block in response (stop_reason=${json.stop_reason ?? 'unknown'})`);
-            }
-            return toolBlock.input;
-        },
-    };
-}
+import { DEFAULT_MODEL, HAIKU_4_5_PRICE_INPUT_PER_TOKEN, HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN, buildAnthropicLlmCaller, getTokenUsage, resolveAnthropicKey, } from './cli-llm.js';
 const SHAPE_FILTERS = [
     'props-only',
     'context-only',
@@ -137,11 +62,9 @@ function parseArgs(argv) {
         result.shapeFilter = shapeFilter;
     return result;
 }
-const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
-const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
 async function main() {
     const args = parseArgs(process.argv.slice(2));
-    const apiKey = resolveAnthropicKey();
+    const apiKey = resolveAnthropicKey('bench-synth');
     const llm = buildAnthropicLlmCaller(apiKey, args.model);
     if (!args.json) {
         process.stdout.write(`bench-synth: model=${args.model}\n\n`);
@@ -168,11 +91,12 @@ async function main() {
     process.stdout.write('\n');
     process.stdout.write(formatBenchReport(report));
     process.stdout.write('\n\n');
-    const totalCost = totalInputTokens * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
-        totalOutputTokens * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
+    const usage = getTokenUsage();
+    const totalCost = usage.input * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
+        usage.output * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
     const callsMade = report.totals.all - report.totals.synthDeclined;
     const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
-    process.stdout.write(`Tokens: input=${totalInputTokens} output=${totalOutputTokens}\n`);
+    process.stdout.write(`Tokens: input=${usage.input} output=${usage.output}\n`);
     process.stdout.write(`Cost:   total=$${totalCost.toFixed(4)} per-call=$${costPerCall.toFixed(4)}\n`);
 }
 main().catch((err) => {

package/dist/synth-bench/run-repair-bench-cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=run-repair-bench-cli.d.ts.map

package/dist/synth-bench/run-repair-bench-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-repair-bench-cli.d.ts","sourceRoot":"","sources":["../../src/synth-bench/run-repair-bench-cli.ts"],"names":[],"mappings":""}

package/dist/synth-bench/run-repair-bench-cli.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+/**
+ * Repair bench CLI — live round-trip-quality probe.
+ *
+ * Runs the production forgiving-handshake create-path
+ * (`ensureConformingContract`) over REPAIR_CORPUS and reports the
+ * round-trip-usable rate — the contract-quality number the shape bench
+ * (bench-synth) cannot see. Reads ~/.ggui/credentials.json (or
+ * ANTHROPIC_API_KEY) for the key.
+ *
+ * Costs ~$0.001 per repaired entry on Haiku 4.5 (clean drafts hit the
+ * lint-clean fast path and make NO LLM call). NOT run in CI; opt-in.
+ *
+ * Usage:
+ *   pnpm -F @ggui-ai/negotiator bench-repair
+ *   pnpm -F @ggui-ai/negotiator bench-repair -- --limit 1
+ *   pnpm -F @ggui-ai/negotiator bench-repair -- --json > repair-report.json
+ *   ANTHROPIC_API_KEY=sk-... pnpm -F @ggui-ai/negotiator bench-repair
+ *
+ * Bench-only — not exported from the package index.
+ */
+import { evaluateRepairCorpus, formatRepairBenchReport, } from './run-repair-bench.js';
+import { DEFAULT_MODEL, HAIKU_4_5_PRICE_INPUT_PER_TOKEN, HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN, buildAnthropicLlmCaller, getTokenUsage, resolveAnthropicKey, } from './cli-llm.js';
+function parseArgs(argv) {
+    let limit;
+    let model = DEFAULT_MODEL;
+    let json = false;
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === '--limit' && argv[i + 1]) {
+            limit = Number(argv[++i]);
+        }
+        else if (a === '--model' && argv[i + 1]) {
+            model = argv[++i];
+        }
+        else if (a === '--json') {
+            json = true;
+        }
+    }
+    const result = { model, json };
+    if (limit !== undefined)
+        result.limit = limit;
+    return result;
+}
+async function main() {
+    const args = parseArgs(process.argv.slice(2));
+    const apiKey = resolveAnthropicKey('bench-repair');
+    const llm = buildAnthropicLlmCaller(apiKey, args.model);
+    if (!args.json) {
+        process.stdout.write(`bench-repair: model=${args.model}\n\n`);
+    }
+    const report = await evaluateRepairCorpus({ llm }, {
+        ...(args.limit !== undefined ? { limit: args.limit } : {}),
+        onProgress: args.json
+            ? undefined
+            : (outcome, idx, total) => {
+                const pass = outcome.roundTrip !== null
+                    ? outcome.roundTrip.pass
+                    : outcome.shape.pass;
+                const status = pass ? 'OK ' : 'NO ';
+                const id = outcome.entry.id.padEnd(22);
+                const origin = `origin:${outcome.origin}`.padEnd(13);
+                process.stdout.write(`[${status}] ${(idx + 1).toString().padStart(2)}/${total}  ${origin}  ${id}  ${outcome.latencyMs}ms\n`);
+            },
+    });
+    if (args.json) {
+        process.stdout.write(JSON.stringify(report, null, 2));
+        process.stdout.write('\n');
+        return;
+    }
+    process.stdout.write('\n');
+    process.stdout.write(formatRepairBenchReport(report));
+    process.stdout.write('\n\n');
+    const usage = getTokenUsage();
+    const totalCost = usage.input * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
+        usage.output * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
+    // Clean drafts (origin agent) make no LLM call — cost is per repaired entry.
+    const callsMade = report.totals.originSynth;
+    const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
+    process.stdout.write(`Tokens: input=${usage.input} output=${usage.output}\n`);
+    process.stdout.write(`Cost:   total=$${totalCost.toFixed(4)} per-repair=$${costPerCall.toFixed(4)}\n`);
+}
+main().catch((err) => {
+    process.stderr.write(`bench-repair failed: ${err instanceof Error ? err.message : String(err)}\n`);
+    process.exit(1);
+});

package/dist/synth-bench/run-repair-bench.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Repair-path bench runner — the round-trip QUALITY probe.
+ *
+ * Where {@link evaluateAgainstCorpus} runs synthesize-from-intent and
+ * scores SHAPE, this runs the production forgiving-handshake create-path
+ * — `ensureConformingContract(draft, intent)` — over {@link
+ * REPAIR_CORPUS} and scores whether the produced contract is round-trip
+ * USABLE, not merely valid.
+ *
+ * Per entry it records:
+ *   - `origin` — `agent` (draft was clean, returned verbatim via the
+ *     fast path) vs `synth` (repaired in-place). The repair discriminator.
+ *   - `shape` — {@link scoreSynthesizedContract} ride-along (which specs).
+ *   - `roundTrip` — {@link scoreContractRoundTrip}: the headline signal.
+ *     A reshape that breaks the agent's seed round-trip fails here even
+ *     though `lintContract` passes the contract.
+ *
+ * The bench's pass verdict is the ROUND-TRIP score when an entry carries
+ * a round-trip expectation, falling back to shape otherwise. So the
+ * top-line precision answers "how often does the negotiator produce a
+ * round-trip-usable contract from an agent draft?" — the contract-
+ * quality number the shape bench cannot see.
+ *
+ * Live LLM probe — opt-in CLI (run-repair-bench-cli.ts), NOT in CI. The
+ * deterministic scorer pinning lives in round-trip-score.test.ts.
+ */
+import type { DataContract, SuggestionFinding } from '@ggui-ai/protocol';
+import type { LLMCaller } from '../llm-caller.js';
+import { type ScoreResult } from './run-bench.js';
+import { type RoundTripScore } from './round-trip-score.js';
+import { type BenchEntry } from './corpus.js';
+export interface RepairBenchOutcome {
+    readonly entry: BenchEntry;
+    /** ensureConformingContract always returns a contract (possibly `{}`). */
+    readonly contract: DataContract;
+    /** `agent` = clean draft returned verbatim; `synth` = repaired in-place. */
+    readonly origin: 'agent' | 'synth';
+    /** How the contract was produced (the efficiency tier): verbatim /
+     *  normalized (deterministic, no LLM) / llm-repair / fallback-empty. */
+    readonly method: 'verbatim' | 'normalized' | 'llm-repair' | 'fallback-empty';
+    /** Structural shape score (ride-along secondary signal). */
+    readonly shape: ScoreResult;
+    /** Round-trip usability — null when the entry declares no round-trip
+     *  expectation (then `shape` carries the verdict). */
+    readonly roundTrip: RoundTripScore | null;
+    /** The error/warn findings the negotiator surfaced back to the agent. */
+    readonly findings: readonly SuggestionFinding[];
+    readonly reasoning: string;
+    readonly latencyMs: number;
+}
+/**
+ * An outcome passes on its ROUND-TRIP score when one exists (the sharper
+ * gate), else on its shape score. Round-trip is the point of this bench.
+ */
+export declare function repairOutcomePass(outcome: RepairBenchOutcome): boolean;
+export interface RepairBenchReport {
+    readonly outcomes: readonly RepairBenchOutcome[];
+    readonly totals: {
+        readonly all: number;
+        readonly pass: number;
+        readonly fail: number;
+        /** Drafts returned verbatim by the lint-clean fast path. */
+        readonly originAgent: number;
+        /** Drafts repaired in-place by the synth loop. */
+        readonly originSynth: number;
+        /** Entries carrying a round-trip expectation. */
+        readonly roundTripScored: number;
+        /** Of those, how many round-trip cleanly. */
+        readonly roundTripPass: number;
+        /** roundTripPass / roundTripScored — the contract-quality headline. */
+        readonly roundTripPrecision: number;
+        /** pass / all across all entries. */
+        readonly precision: number;
+    };
+    /** Histogram of round-trip failure kinds across the run. */
+    readonly byFailureKind: Readonly<Record<string, number>>;
+    readonly latency: {
+        readonly p50Ms: number;
+        readonly p95Ms: number;
+    };
+}
+export interface RunRepairBenchOptions {
+    readonly limit?: number;
+    readonly onProgress?: (outcome: RepairBenchOutcome, index: number, total: number) => void;
+}
+export declare function evaluateRepairCorpus(deps: {
+    readonly llm: LLMCaller;
+}, options?: RunRepairBenchOptions, corpus?: readonly BenchEntry[]): Promise<RepairBenchReport>;
+export declare function summarizeRepair(outcomes: readonly RepairBenchOutcome[]): RepairBenchReport;
+export declare function formatRepairBenchReport(report: RepairBenchReport): string;
+export declare function runRepairBench(deps: {
+    readonly llm: LLMCaller;
+}, options?: RunRepairBenchOptions): Promise<RepairBenchReport>;
+//# sourceMappingURL=run-repair-bench.d.ts.map

package/dist/synth-bench/run-repair-bench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-repair-bench.d.ts","sourceRoot":"","sources":["../../src/synth-bench/run-repair-bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEzE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAA4B,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC5E,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAiB,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAE7D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAChC,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC;IACnC;4EACwE;IACxE,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAC;IAC7E,4DAA4D;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B;0DACsD;IACtD,QAAQ,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI,CAAC;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAItE;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACjD,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,4DAA4D;QAC5D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,kDAAkD;QAClD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,iDAAiD;QACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;QACjC,6CAA6C;QAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;QAC/B,uEAAuE;QACvE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,qCAAqC;QACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,4DAA4D;IAC5D,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACtE;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,CACpB,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,KACV,IAAI,CAAC;CACX;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,OAAO,GAAE,qBAA0B,EACnC,MAAM,GAAE,SAAS,UAAU,EAAkB,GAC5C,OAAO,CAAC,iBAAiB,CAAC,CA6C5B;AAED,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,SAAS,kBAAkB,EAAE,GACtC,iBAAiB,CAwCnB;AAQD,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAyDzE;AAED,wBAAgB,cAAc,CAC5B,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,iBAAiB,CAAC,CAE5B"}

package/dist/synth-bench/run-repair-bench.js

@@ -0,0 +1,172 @@
+/**
+ * Repair-path bench runner — the round-trip QUALITY probe.
+ *
+ * Where {@link evaluateAgainstCorpus} runs synthesize-from-intent and
+ * scores SHAPE, this runs the production forgiving-handshake create-path
+ * — `ensureConformingContract(draft, intent)` — over {@link
+ * REPAIR_CORPUS} and scores whether the produced contract is round-trip
+ * USABLE, not merely valid.
+ *
+ * Per entry it records:
+ *   - `origin` — `agent` (draft was clean, returned verbatim via the
+ *     fast path) vs `synth` (repaired in-place). The repair discriminator.
+ *   - `shape` — {@link scoreSynthesizedContract} ride-along (which specs).
+ *   - `roundTrip` — {@link scoreContractRoundTrip}: the headline signal.
+ *     A reshape that breaks the agent's seed round-trip fails here even
+ *     though `lintContract` passes the contract.
+ *
+ * The bench's pass verdict is the ROUND-TRIP score when an entry carries
+ * a round-trip expectation, falling back to shape otherwise. So the
+ * top-line precision answers "how often does the negotiator produce a
+ * round-trip-usable contract from an agent draft?" — the contract-
+ * quality number the shape bench cannot see.
+ *
+ * Live LLM probe — opt-in CLI (run-repair-bench-cli.ts), NOT in CI. The
+ * deterministic scorer pinning lives in round-trip-score.test.ts.
+ */
+import { ensureConformingContract } from '../ensure-conforming-contract.js';
+import { scoreSynthesizedContract } from './run-bench.js';
+import { scoreContractRoundTrip, } from './round-trip-score.js';
+import { REPAIR_CORPUS } from './corpus.js';
+/**
+ * An outcome passes on its ROUND-TRIP score when one exists (the sharper
+ * gate), else on its shape score. Round-trip is the point of this bench.
+ */
+export function repairOutcomePass(outcome) {
+    return outcome.roundTrip !== null
+        ? outcome.roundTrip.pass
+        : outcome.shape.pass;
+}
+export async function evaluateRepairCorpus(deps, options = {}, corpus = REPAIR_CORPUS) {
+    let subset = corpus;
+    if (options.limit !== undefined) {
+        subset = subset.slice(0, options.limit);
+    }
+    const outcomes = [];
+    for (let i = 0; i < subset.length; i++) {
+        const entry = subset[i];
+        const startedAt = Date.now();
+        // The real production create-path: lint the draft → verbatim if clean
+        // (origin agent), repair-in-place otherwise (origin synth). NEVER
+        // throws; an unrepairable draft yields the empty `{}` contract.
+        const result = await ensureConformingContract({ llm: deps.llm }, {
+            draft: entry.draft,
+            intent: entry.intent,
+            ...(entry.appGadgets !== undefined
+                ? { appGadgets: entry.appGadgets }
+                : {}),
+        });
+        const latencyMs = Date.now() - startedAt;
+        const shape = scoreSynthesizedContract(result.contract, entry.expected);
+        const roundTrip = entry.roundTrip !== undefined
+            ? scoreContractRoundTrip(result.contract, entry.roundTrip)
+            : null;
+        const outcome = {
+            entry,
+            contract: result.contract,
+            origin: result.origin,
+            method: result.method,
+            shape,
+            roundTrip,
+            findings: result.findings,
+            reasoning: result.reasoning,
+            latencyMs,
+        };
+        outcomes.push(outcome);
+        options.onProgress?.(outcome, i, subset.length);
+    }
+    return summarizeRepair(outcomes);
+}
+export function summarizeRepair(outcomes) {
+    const all = outcomes.length;
+    const pass = outcomes.filter(repairOutcomePass).length;
+    const fail = all - pass;
+    const originAgent = outcomes.filter((o) => o.origin === 'agent').length;
+    const originSynth = outcomes.filter((o) => o.origin === 'synth').length;
+    const scored = outcomes.filter((o) => o.roundTrip !== null);
+    const roundTripScored = scored.length;
+    const roundTripPass = scored.filter((o) => o.roundTrip?.pass === true).length;
+    const roundTripPrecision = roundTripScored === 0 ? 0 : roundTripPass / roundTripScored;
+    const byFailureKind = {};
+    for (const o of outcomes) {
+        for (const f of o.roundTrip?.failures ?? []) {
+            byFailureKind[f.kind] = (byFailureKind[f.kind] ?? 0) + 1;
+        }
+    }
+    const latencies = outcomes.map((o) => o.latencyMs).sort((a, b) => a - b);
+    return {
+        outcomes,
+        totals: {
+            all,
+            pass,
+            fail,
+            originAgent,
+            originSynth,
+            roundTripScored,
+            roundTripPass,
+            roundTripPrecision,
+            precision: all === 0 ? 0 : pass / all,
+        },
+        byFailureKind,
+        latency: {
+            p50Ms: percentile(latencies, 0.5),
+            p95Ms: percentile(latencies, 0.95),
+        },
+    };
+}
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
+    return sorted[idx] ?? 0;
+}
+export function formatRepairBenchReport(report) {
+    const lines = [];
+    const t = report.totals;
+    lines.push('=== repair bench report (round-trip quality) ===');
+    lines.push('');
+    lines.push(`Round-trip usable:   ${t.roundTripPass}/${t.roundTripScored} (${(t.roundTripPrecision * 100).toFixed(1)}%)`);
+    lines.push(`Repair origin:       agent×${t.originAgent} synth×${t.originSynth}`);
+    // Efficiency tiers — verbatim + normalized are FREE (no LLM); only
+    // llm-repair pays a model call. A high normalized count = the cheap
+    // deterministic tier doing the work the LLM loop used to.
+    const byMethod = new Map();
+    for (const o of report.outcomes) {
+        byMethod.set(o.method, (byMethod.get(o.method) ?? 0) + 1);
+    }
+    const methodStr = ['verbatim', 'normalized', 'llm-repair', 'fallback-empty']
+        .filter((m) => byMethod.has(m))
+        .map((m) => `${m}×${byMethod.get(m)}`)
+        .join(' ');
+    lines.push(`Method (LLM cost):   ${methodStr}`);
+    lines.push(`Overall pass:        ${t.pass}/${t.all} (${(t.precision * 100).toFixed(1)}%)`);
+    lines.push(`Latency:             p50=${report.latency.p50Ms}ms p95=${report.latency.p95Ms}ms`);
+    const kinds = Object.entries(report.byFailureKind);
+    if (kinds.length > 0) {
+        lines.push('');
+        lines.push('Round-trip failures by kind:');
+        for (const [kind, count] of kinds.sort((a, b) => b[1] - a[1])) {
+            lines.push(`  ${kind.padEnd(20)} ×${count}`);
+        }
+    }
+    const failed = report.outcomes.filter((o) => !repairOutcomePass(o));
+    if (failed.length > 0) {
+        lines.push('');
+        lines.push('Failures:');
+        for (const o of failed) {
+            lines.push(`  [origin ${o.origin}] ${o.entry.id}: ${o.entry.intent.slice(0, 56)}`);
+            // Shape mismatches (gating only — advisory name checks omitted here).
+            for (const f of o.shape.failures) {
+                lines.push(`    shape ${f.kind}: ${f.hint.slice(0, 120)}`);
+            }
+            for (const f of o.roundTrip?.failures ?? []) {
+                lines.push(`    round-trip ${f.kind}: ${f.hint.slice(0, 160)}`);
+            }
+            lines.push(`    reasoning: ${o.reasoning.slice(0, 120)}`);
+        }
+    }
+    return lines.join('\n');
+}
+export function runRepairBench(deps, options = {}) {
+    return evaluateRepairCorpus(deps, options);
+}

package/dist/synthesize-contract.d.ts

@@ -24,10 +24,12 @@
  * a richer surface should author the contract themselves on the
  * handshake input; synthesis is a fallback, not a replacement.
  *
- * **Failure modes collapse to null.** LLM throws, parse fails,
- * provider doesn't support `callStructured` → return `null`. Caller
- * falls back to an empty stub; behavior regresses to pre-synth but
- * doesn't crash.
+ * **Failure modes collapse to null.** LLM throws on every attempt or
+ * the bounded repair budget is exhausted → return `null`. Caller falls
+ * back to an empty stub; behavior regresses to pre-synth but doesn't
+ * crash. Providers without `callStructured` (gemini / openai /
+ * openrouter) use a text-JSON fallback rather than skipping synthesis,
+ * so repair works on every provider.
  *
  * **Cost.** ~$0.0005-0.001 per call (Haiku 4.5, ~500 input + ~300
  * output tokens). Latency ~1.5s. Fires only on cold-path Tier 3
@@ -74,8 +76,14 @@ export declare const SYNTHESIZE_TOOL: ToolSchema;
  * Empty / whitespace intent short-circuits to null with a reason —
  * no contract can be inferred from nothing.
  *
- * Provider lacking `callStructured` (test stubs, providers without
- * tool-use) collapses to null.
+ * Providers without `callStructured` (gemini / openai / openrouter) use
+ * a text-JSON fallback (the validate-and-repair loop catches malformed
+ * output and retries) rather than skipping synthesis.
+ *
+ * When `options.draft` is supplied, the loop REPAIRS that draft in
+ * place (seeded with the agent's contract + the deterministic findings)
+ * instead of synthesizing from `intent` alone — the forgiving-handshake
+ * path.
  *
  * Each attempt is self-checked against the validation gate; a failure
  * feeds the precise error back for up to {@link MAX_SYNTH_ATTEMPTS}
@@ -101,6 +109,30 @@ export declare function synthesizeContract(deps: {
      * no-app-registry path).
      */
     readonly appGadgets?: readonly GadgetDescriptor[];
+    /**
+     * Repair-in-place seed. When provided, the synthesizer does NOT
+     * synthesize from `intent` alone — it starts from the agent's
+     * proposed `draft` and the deterministic findings that rejected it,
+     * and the validate-and-repair loop corrects exactly those problems
+     * while preserving the agent's intent + structure. This is the
+     * forgiving-handshake path: an invalid agent draft is the loop's
+     * SEED rather than a thrown error. Absent ⇒ classic
+     * synthesize-from-intent (cold path). Typed `unknown` because the
+     * agent's draft is untrusted — it may not be a valid DataContract
+     * (that's the whole point of repairing it).
+     */
+    readonly draft?: unknown;
+    /**
+     * Deterministic validation findings that rejected {@link draft}
+     * (from `lintContract(draft).errors`). Fed into the first repair
+     * note so the model corrects the precise problems. Ignored when
+     * `draft` is absent.
+     */
+    readonly draftFindings?: readonly {
+        readonly code: string;
+        readonly path: string;
+        readonly message: string;
+    }[];
 }): Promise<SynthesizeContractResult>;
 /**
  * Compose the "AVAILABLE GADGETS" section appended to synth's user

package/dist/synthesize-contract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"synthesize-contract.d.ts","sourceRoot":"","sources":["../src/synthesize-contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACtB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAKL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AAsBlC,uCAAuC;AACvC,MAAM,WAAW,wBAAwB;IACvC,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,CAAC;IACvC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,yBAAyB,EAAE,CAAC;CACzD;AA+ND;;;;;GAKG;AACH,eAAO,MAAM,eAAe,EAAE,UA+I7B,CAAC;AAsGF;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IACR;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACnD,GACA,OAAO,CAAC,wBAAwB,CAAC,CAiKnC;AAgUD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,SAAS,gBAAgB,EAAE,GAAG,SAAS,GAC/C,MAAM,GAAG,SAAS,CA+CpB"}
+{"version":3,"file":"synthesize-contract.d.ts","sourceRoot":"","sources":["../src/synthesize-contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACtB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAM7D,OAAO,EAKL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AAsBlC,uCAAuC;AACvC,MAAM,WAAW,wBAAwB;IACvC,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,CAAC;IACvC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,yBAAyB,EAAE,CAAC;CACzD;AAuOD;;;;;GAKG;AACH,eAAO,MAAM,eAAe,EAAE,UA+I7B,CAAC;AAwRF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IACR;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAClD;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS;QAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,EAAE,CAAC;CACL,GACA,OAAO,CAAC,wBAAwB,CAAC,CAyMnC;AAgUD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,SAAS,gBAAgB,EAAE,GAAG,SAAS,GAC/C,MAAM,GAAG,SAAS,CA+CpB"}