@ggui-ai/negotiator 0.1.0-rc.1

package/dist/synth-bench/run-bench-cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=run-bench-cli.d.ts.map

package/dist/synth-bench/run-bench-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-bench-cli.d.ts","sourceRoot":"","sources":["../../src/synth-bench/run-bench-cli.ts"],"names":[],"mappings":""}

package/dist/synth-bench/run-bench-cli.js

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+/**
+ * Synthesizer bench CLI — live LLM probe.
+ *
+ * Reads ~/.ggui/credentials.json for the Anthropic API key, runs
+ * synthesizeContract over BENCH_CORPUS, prints per-shape precision
+ * + the redundant-action firing count + p50/p95 latency.
+ *
+ * Costs ~$0.001 per entry on Haiku 4.5 → roughly $0.05 for the full
+ * 50+ corpus run. NOT run in CI; opt-in only.
+ *
+ * Usage:
+ *   pnpm -F @ggui-ai/negotiator bench-synth
+ *   pnpm -F @ggui-ai/negotiator bench-synth -- --limit 10
+ *   pnpm -F @ggui-ai/negotiator bench-synth -- --shape context+action
+ *   ANTHROPIC_API_KEY=sk-... pnpm -F @ggui-ai/negotiator bench-synth
+ *   pnpm -F @ggui-ai/negotiator bench-synth -- --json > report.json
+ *
+ * Bench-only — not exported from the package index.
+ */
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { resolve as pathResolve } from 'node:path';
+import { evaluateAgainstCorpus, formatBenchReport } from './run-bench.js';
+import { contractShape } from './corpus.js';
+const ANTHROPIC_API = 'https://api.anthropic.com/v1/messages';
+const DEFAULT_MODEL = 'claude-haiku-4-5';
+function resolveAnthropicKey() {
+    const envKey = process.env['ANTHROPIC_API_KEY'];
+    if (envKey && envKey.length > 0)
+        return envKey;
+    const credsPath = pathResolve(homedir(), '.ggui', 'credentials.json');
+    let parsed;
+    try {
+        parsed = JSON.parse(readFileSync(credsPath, 'utf8'));
+    }
+    catch (err) {
+        throw new Error(`bench-synth: could not read ${credsPath} (${err instanceof Error ? err.message : String(err)}). Set ANTHROPIC_API_KEY env var or run \`ggui auth set anthropic\`.`);
+    }
+    const key = parsed.apps?.global?.anthropic;
+    if (typeof key !== 'string' || key.length === 0) {
+        throw new Error(`bench-synth: no anthropic key found at apps.global.anthropic in ${credsPath}.`);
+    }
+    return key;
+}
+let totalInputTokens = 0;
+let totalOutputTokens = 0;
+function buildAnthropicLlmCaller(apiKey, model) {
+    return {
+        async call() {
+            throw new Error('bench-synth: text-mode not exercised — synth uses callStructured');
+        },
+        async callStructured(systemPrompt, userMessage, tool, maxTokens) {
+            // `temperature` deprecated on Haiku 4.5+ — Anthropic rejects with
+            // HTTP 400. `tool_choice: { type: 'tool', name }` below already
+            // binds output to the input_schema; residual stochasticity stays
+            // bounded via canonical-key normalization downstream.
+            const body = {
+                model,
+                max_tokens: maxTokens ?? 1024,
+                system: systemPrompt,
+                messages: [{ role: 'user', content: userMessage }],
+                tools: [
+                    {
+                        name: tool.name,
+                        description: tool.description,
+                        input_schema: tool.input_schema,
+                    },
+                ],
+                tool_choice: { type: 'tool', name: tool.name },
+            };
+            const res = await fetch(ANTHROPIC_API, {
+                method: 'POST',
+                headers: {
+                    'content-type': 'application/json',
+                    'x-api-key': apiKey,
+                    'anthropic-version': '2023-06-01',
+                },
+                body: JSON.stringify(body),
+            });
+            const json = (await res.json());
+            if (!res.ok) {
+                const errType = json.error?.type ?? 'unknown';
+                const errMsg = json.error?.message ?? `HTTP ${res.status}`;
+                throw new Error(`anthropic ${errType}: ${errMsg}`);
+            }
+            if (json.usage) {
+                totalInputTokens += json.usage.input_tokens ?? 0;
+                totalOutputTokens += json.usage.output_tokens ?? 0;
+            }
+            const toolBlock = json.content?.find((b) => b.type === 'tool_use');
+            if (!toolBlock || toolBlock.input === undefined) {
+                throw new Error(`anthropic: no tool_use block in response (stop_reason=${json.stop_reason ?? 'unknown'})`);
+            }
+            return toolBlock.input;
+        },
+    };
+}
+const SHAPE_FILTERS = [
+    'props-only',
+    'context-only',
+    'context+action',
+    'stream',
+    'with-gadgets',
+    'empty',
+];
+function parseArgs(argv) {
+    let limit;
+    let shapeFilter;
+    let model = DEFAULT_MODEL;
+    let json = false;
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === '--limit' && argv[i + 1]) {
+            limit = Number(argv[++i]);
+        }
+        else if (a === '--shape' && argv[i + 1]) {
+            const v = argv[++i];
+            if (SHAPE_FILTERS.includes(v)) {
+                shapeFilter = v;
+            }
+            else {
+                throw new Error(`unknown shape: ${v} (expected one of ${SHAPE_FILTERS.join(', ')})`);
+            }
+        }
+        else if (a === '--model' && argv[i + 1]) {
+            model = argv[++i];
+        }
+        else if (a === '--json') {
+            json = true;
+        }
+    }
+    const result = { model, json };
+    if (limit !== undefined)
+        result.limit = limit;
+    if (shapeFilter !== undefined)
+        result.shapeFilter = shapeFilter;
+    return result;
+}
+const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
+const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
+async function main() {
+    const args = parseArgs(process.argv.slice(2));
+    const apiKey = resolveAnthropicKey();
+    const llm = buildAnthropicLlmCaller(apiKey, args.model);
+    if (!args.json) {
+        process.stdout.write(`bench-synth: model=${args.model}\n\n`);
+    }
+    const report = await evaluateAgainstCorpus({ llm }, {
+        ...(args.limit !== undefined ? { limit: args.limit } : {}),
+        ...(args.shapeFilter !== undefined
+            ? { shapeFilter: args.shapeFilter }
+            : {}),
+        onProgress: args.json
+            ? undefined
+            : (outcome, idx, total) => {
+                const status = outcome.score.pass ? 'OK ' : 'NO ';
+                const id = outcome.entry.id.padEnd(28);
+                const shape = contractShape(outcome.entry.expected).padEnd(16);
+                process.stdout.write(`[${status}] ${(idx + 1).toString().padStart(2)}/${total}  ${shape}  ${id}  ${outcome.latencyMs}ms\n`);
+            },
+    });
+    if (args.json) {
+        process.stdout.write(JSON.stringify(report, null, 2));
+        process.stdout.write('\n');
+        return;
+    }
+    process.stdout.write('\n');
+    process.stdout.write(formatBenchReport(report));
+    process.stdout.write('\n\n');
+    const totalCost = totalInputTokens * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
+        totalOutputTokens * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
+    const callsMade = report.totals.all - report.totals.synthDeclined;
+    const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
+    process.stdout.write(`Tokens: input=${totalInputTokens} output=${totalOutputTokens}\n`);
+    process.stdout.write(`Cost:   total=$${totalCost.toFixed(4)} per-call=$${costPerCall.toFixed(4)}\n`);
+}
+main().catch((err) => {
+    process.stderr.write(`bench-synth failed: ${err instanceof Error ? err.message : String(err)}\n`);
+    process.exit(1);
+});

package/dist/synth-bench/run-bench.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Synthesizer bench runner.
+ *
+ * Two evaluators:
+ *
+ *   - {@link evaluateAgainstCorpus} — given an `LLMCaller`, runs
+ *     `synthesizeContract` over the corpus and compares each output to
+ *     the expected structural shape. Used by {@link runSynthBench} for
+ *     a real LLM probe (opt-in CLI, costs ~$0.001 per entry on Haiku).
+ *
+ *   - {@link scoreSynthesizedContract} — pure structural compare. Used
+ *     by the deterministic structure-bench test against fixtures and
+ *     by the LLM bench to grade live outputs.
+ *
+ * Pass criteria:
+ *   - `hasActionSpec` / `hasContextSpec` / `hasStreamSpec` / `hasProps`
+ *     match, OR the entry sets `tolerateEitherShape: true`.
+ *   - Gadget / agent-tool *identity* (`capabilityHooks`,
+ *     `forbiddenCapabilityHooks`) — registry-bounded names, so a
+ *     mismatch is a real retrieval/discrimination bug and DOES fail.
+ *
+ * Advisory (reported, NOT pass-gating) — see {@link ADVISORY_FAILURE_KINDS}:
+ *   - `actionNames` / `contextSlots` / `agentToolNames` are
+ *     free-vocabulary: the synth invents the action/slot/tool name and
+ *     a checkout-completion action (`finish` / `complete` /
+ *     `completePurchase` / `placeOrder` / …) has an unbounded valid
+ *     name space. An exact-name allow-list only generates false
+ *     negatives, so a name mismatch is surfaced but does not fail the
+ *     bench — spec-presence + placement already verify the shape.
+ *
+ * Validator findings ride along on the report so an operator inspecting
+ * a regression can see WHY a synthesized contract was flagged
+ * (redundant-action vs novel-shape vs nothing).
+ */
+import type { DataContract } from '@ggui-ai/protocol';
+import { type ContractValidationFinding } from '../contract-validators.js';
+import type { LLMCaller } from '../llm-caller.js';
+import { type BenchEntry, type BenchExpectation } from './corpus.js';
+export interface ScoreFailure {
+    readonly kind: 'synth-declined' | 'has-action-mismatch' | 'has-context-mismatch' | 'has-stream-mismatch' | 'has-props-mismatch' | 'has-client-capabilities-mismatch' | 'has-agent-tools-mismatch' | 'action-names-disjoint' | 'context-slots-disjoint' | 'capability-hooks-disjoint' | 'forbidden-capability-hooks-present' | 'agent-tool-names-disjoint';
+    readonly hint: string;
+}
+export interface ScoreResult {
+    readonly pass: boolean;
+    /** True when expectation tolerated either shape and structural match
+     *  was loose. Used to distinguish strict pass from tolerated pass in
+     *  reporting. */
+    readonly tolerated: boolean;
+    readonly failures: readonly ScoreFailure[];
+}
+/**
+ * Compare a synthesized contract against the expected shape. Pure /
+ * deterministic — used both by the deterministic structure-bench and
+ * by the live-LLM probe.
+ */
+export declare function scoreSynthesizedContract(contract: DataContract, expected: BenchExpectation): ScoreResult;
+export interface BenchOutcome {
+    readonly entry: BenchEntry;
+    readonly contract: DataContract | null;
+    readonly score: ScoreResult;
+    readonly findings: readonly ContractValidationFinding[];
+    readonly latencyMs: number;
+    /** LLM attempts the synthesizer made (1 = no repair retry needed). */
+    readonly attempts: number;
+    readonly synthReason: string;
+}
+export interface BenchReport {
+    readonly outcomes: readonly BenchOutcome[];
+    readonly totals: {
+        readonly all: number;
+        readonly pass: number;
+        readonly fail: number;
+        readonly synthDeclined: number;
+        readonly precision: number;
+    };
+    readonly byShape: Readonly<Record<string, {
+        all: number;
+        pass: number;
+        precision: number;
+    }>>;
+    readonly redundantActionFindings: number;
+    readonly latency: {
+        readonly p50Ms: number;
+        readonly p95Ms: number;
+    };
+}
+export interface RunSynthBenchOptions {
+    readonly limit?: number;
+    /** Filter the corpus to entries of one `contractShape` bucket. */
+    readonly shapeFilter?: string;
+    readonly onProgress?: (outcome: BenchOutcome, index: number, total: number) => void;
+}
+export declare function evaluateAgainstCorpus(deps: {
+    readonly llm: LLMCaller;
+}, options?: RunSynthBenchOptions, corpus?: readonly BenchEntry[]): Promise<BenchReport>;
+export declare function summarize(outcomes: readonly BenchOutcome[]): BenchReport;
+export declare function formatBenchReport(report: BenchReport): string;
+export declare function runSynthBench(deps: {
+    readonly llm: LLMCaller;
+}, options?: RunSynthBenchOptions): Promise<BenchReport>;
+//# sourceMappingURL=run-bench.d.ts.map

package/dist/synth-bench/run-bench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-bench.d.ts","sourceRoot":"","sources":["../../src/synth-bench/run-bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,OAAO,EAEL,KAAK,yBAAyB,EAC/B,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,gBAAgB,EACtB,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EACT,gBAAgB,GAChB,qBAAqB,GACrB,sBAAsB,GACtB,qBAAqB,GACrB,oBAAoB,GACpB,kCAAkC,GAClC,0BAA0B,GAC1B,uBAAuB,GACvB,wBAAwB,GACxB,2BAA2B,GAC3B,oCAAoC,GACpC,2BAA2B,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB;;qBAEiB;IACjB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;CAC5C;AAiBD;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,YAAY,EACtB,QAAQ,EAAE,gBAAgB,GACzB,WAAW,CA8Kb;AAUD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS,yBAAyB,EAAE,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;QAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,QAAQ,CACxB,MAAM,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CACjE,CAAC;IACF,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACtE;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,CACpB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,KACV,IAAI,CAAC;CACX;AAED,wBAAsB,qBAAqB,CACzC,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,OAAO,GAAE,oBAAyB,EAClC,MAAM,GAAE,SAAS,UAAU,EAAiB,GAC3C,OAAO,CAAC,WAAW,CAAC,CA6DtB;AAED,wBAAgB,SAAS,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,GAAG,WAAW,CAwDxE;AAQD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAiE7D;AAED,wBAAgB,aAAa,CAC3B,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,WAAW,CAAC,CAEtB"}

package/dist/synth-bench/run-bench.js

@@ -0,0 +1,374 @@
+/**
+ * Synthesizer bench runner.
+ *
+ * Two evaluators:
+ *
+ *   - {@link evaluateAgainstCorpus} — given an `LLMCaller`, runs
+ *     `synthesizeContract` over the corpus and compares each output to
+ *     the expected structural shape. Used by {@link runSynthBench} for
+ *     a real LLM probe (opt-in CLI, costs ~$0.001 per entry on Haiku).
+ *
+ *   - {@link scoreSynthesizedContract} — pure structural compare. Used
+ *     by the deterministic structure-bench test against fixtures and
+ *     by the LLM bench to grade live outputs.
+ *
+ * Pass criteria:
+ *   - `hasActionSpec` / `hasContextSpec` / `hasStreamSpec` / `hasProps`
+ *     match, OR the entry sets `tolerateEitherShape: true`.
+ *   - Gadget / agent-tool *identity* (`capabilityHooks`,
+ *     `forbiddenCapabilityHooks`) — registry-bounded names, so a
+ *     mismatch is a real retrieval/discrimination bug and DOES fail.
+ *
+ * Advisory (reported, NOT pass-gating) — see {@link ADVISORY_FAILURE_KINDS}:
+ *   - `actionNames` / `contextSlots` / `agentToolNames` are
+ *     free-vocabulary: the synth invents the action/slot/tool name and
+ *     a checkout-completion action (`finish` / `complete` /
+ *     `completePurchase` / `placeOrder` / …) has an unbounded valid
+ *     name space. An exact-name allow-list only generates false
+ *     negatives, so a name mismatch is surfaced but does not fail the
+ *     bench — spec-presence + placement already verify the shape.
+ *
+ * Validator findings ride along on the report so an operator inspecting
+ * a regression can see WHY a synthesized contract was flagged
+ * (redundant-action vs novel-shape vs nothing).
+ */
+import { listContractGadgets } from '@ggui-ai/protocol';
+import { synthesizeContract } from '../synthesize-contract.js';
+import { validateContractStructure, } from '../contract-validators.js';
+import { BENCH_CORPUS, contractShape, } from './corpus.js';
+/**
+ * Failure kinds that are ADVISORY — surfaced in the report but do NOT
+ * gate pass/fail. These check free-vocabulary names (action / slot /
+ * agent-tool keys the synth invents); an exact-name allow-list cannot
+ * enumerate the unbounded valid space, so a mismatch is a bench
+ * false-negative, not a synth bug. Spec-presence + placement carry the
+ * real verdict. Gadget-identity kinds are deliberately NOT here —
+ * those names are registry-bounded.
+ */
+const ADVISORY_FAILURE_KINDS = new Set([
+    'action-names-disjoint',
+    'context-slots-disjoint',
+    'agent-tool-names-disjoint',
+]);
+/**
+ * Compare a synthesized contract against the expected shape. Pure /
+ * deterministic — used both by the deterministic structure-bench and
+ * by the live-LLM probe.
+ */
+export function scoreSynthesizedContract(contract, expected) {
+    const failures = [];
+    const tolerate = expected.tolerateEitherShape === true;
+    const hasActionSpec = contract.actionSpec !== undefined &&
+        Object.keys(contract.actionSpec).length > 0;
+    const hasContextSpec = contract.contextSpec !== undefined &&
+        Object.keys(contract.contextSpec).length > 0;
+    const hasStreamSpec = contract.streamSpec !== undefined &&
+        Object.keys(contract.streamSpec).length > 0;
+    const hasProps = contract.propsSpec !== undefined &&
+        contract.propsSpec.properties !== undefined &&
+        Object.keys(contract.propsSpec.properties).length > 0;
+    // The wire `clientCapabilities.gadgets` map is package-keyed two-level —
+    // `listContractGadgets` flattens it to `(package, name)` use
+    // records. The export NAME is the discriminating identifier (a
+    // `use`-prefixed hook or a PascalCase component). Default empty when
+    // the catalog is absent.
+    const gadgetUses = listContractGadgets(contract);
+    const hasClientCapabilities = gadgetUses.length > 0;
+    const agentToolMap = contract.agentCapabilities?.tools ?? {};
+    const hasAgentTools = Object.keys(agentToolMap).length > 0;
+    if (!tolerate) {
+        if (hasActionSpec !== expected.hasActionSpec) {
+            failures.push({
+                kind: 'has-action-mismatch',
+                hint: `expected hasActionSpec=${expected.hasActionSpec}, got ${hasActionSpec}${hasActionSpec ? ` (actions: ${Object.keys(contract.actionSpec ?? {}).join(', ')})` : ''}`,
+            });
+        }
+        if (hasContextSpec !== expected.hasContextSpec) {
+            failures.push({
+                kind: 'has-context-mismatch',
+                hint: `expected hasContextSpec=${expected.hasContextSpec}, got ${hasContextSpec}${hasContextSpec ? ` (slots: ${Object.keys(contract.contextSpec ?? {}).join(', ')})` : ''}`,
+            });
+        }
+        if (hasStreamSpec !== expected.hasStreamSpec) {
+            failures.push({
+                kind: 'has-stream-mismatch',
+                hint: `expected hasStreamSpec=${expected.hasStreamSpec}, got ${hasStreamSpec}`,
+            });
+        }
+        if (hasProps !== expected.hasProps) {
+            failures.push({
+                kind: 'has-props-mismatch',
+                hint: `expected hasProps=${expected.hasProps}, got ${hasProps}`,
+            });
+        }
+        // EE+ surfaces — only enforced when the corpus entry opts in
+        // (`hasClientCapabilities`/`hasAgentTools` field present). Legacy
+        // entries leave them undefined and skip the check.
+        if (expected.hasClientCapabilities !== undefined) {
+            if (hasClientCapabilities !== expected.hasClientCapabilities) {
+                failures.push({
+                    kind: 'has-client-capabilities-mismatch',
+                    hint: `expected hasClientCapabilities=${expected.hasClientCapabilities}, got ${hasClientCapabilities}${hasClientCapabilities ? ` (exports: ${gadgetUses.map((u) => u.name).join(', ')})` : ''}`,
+                });
+            }
+        }
+        if (expected.hasAgentTools !== undefined) {
+            if (hasAgentTools !== expected.hasAgentTools) {
+                failures.push({
+                    kind: 'has-agent-tools-mismatch',
+                    hint: `expected hasAgentTools=${expected.hasAgentTools}, got ${hasAgentTools}${hasAgentTools ? ` (tools: ${Object.keys(agentToolMap).join(', ')})` : ''}`,
+                });
+            }
+        }
+    }
+    if (expected.actionNames !== undefined &&
+        expected.actionNames.length > 0 &&
+        hasActionSpec) {
+        const allowed = expected.actionNames.map(toFold);
+        const got = Object.keys(contract.actionSpec ?? {});
+        const intersects = got.some((name) => allowed.some((a) => containsEither(toFold(name), a)));
+        if (!intersects) {
+            failures.push({
+                kind: 'action-names-disjoint',
+                hint: `synthesized actions [${got.join(', ')}] disjoint from allowed [${expected.actionNames.join(', ')}]`,
+            });
+        }
+    }
+    if (expected.contextSlots !== undefined &&
+        expected.contextSlots.length > 0 &&
+        hasContextSpec) {
+        const allowed = expected.contextSlots.map(toFold);
+        const got = Object.keys(contract.contextSpec ?? {});
+        const intersects = got.some((slot) => allowed.some((a) => containsEither(toFold(slot), a)));
+        if (!intersects) {
+            failures.push({
+                kind: 'context-slots-disjoint',
+                hint: `synthesized slots [${got.join(', ')}] disjoint from allowed [${expected.contextSlots.join(', ')}]`,
+            });
+        }
+    }
+    if (expected.capabilityHooks !== undefined &&
+        expected.capabilityHooks.length > 0 &&
+        hasClientCapabilities) {
+        const allowed = new Set(expected.capabilityHooks.map(toFold));
+        const got = gadgetUses.map((u) => u.name);
+        const intersects = got.some((h) => allowed.has(toFold(h)));
+        if (!intersects) {
+            failures.push({
+                kind: 'capability-hooks-disjoint',
+                hint: `synthesized hooks [${got.join(', ')}] disjoint from allowed [${expected.capabilityHooks.join(', ')}]`,
+            });
+        }
+    }
+    // Forbidden hooks check. Fires even when
+    // `hasClientCapabilities=false` is the headline expectation, because
+    // the LLM might violate by attaching the wrapper anyway. The check
+    // is silent when the contract has no clientCapabilities (nothing to
+    // forbid).
+    if (expected.forbiddenCapabilityHooks !== undefined &&
+        expected.forbiddenCapabilityHooks.length > 0) {
+        const forbidden = new Set(expected.forbiddenCapabilityHooks.map(toFold));
+        const got = gadgetUses.map((u) => u.name);
+        const violating = got.filter((h) => forbidden.has(toFold(h)));
+        if (violating.length > 0) {
+            failures.push({
+                kind: 'forbidden-capability-hooks-present',
+                hint: `synthesized hooks [${violating.join(', ')}] present in forbidden set [${expected.forbiddenCapabilityHooks.join(', ')}] — registered wrapper attached without intent justification`,
+            });
+        }
+    }
+    if (expected.agentToolNames !== undefined &&
+        expected.agentToolNames.length > 0 &&
+        hasAgentTools) {
+        const allowed = expected.agentToolNames.map(toFold);
+        const got = Object.keys(agentToolMap);
+        const intersects = got.some((name) => allowed.some((a) => containsEither(toFold(name), a)));
+        if (!intersects) {
+            failures.push({
+                kind: 'agent-tool-names-disjoint',
+                hint: `synthesized agentTools [${got.join(', ')}] disjoint from allowed [${expected.agentToolNames.join(', ')}]`,
+            });
+        }
+    }
+    // Pass when there are no NON-advisory failures. Advisory findings
+    // (free-vocabulary name mismatches) ride along on `failures` for
+    // report visibility but never gate the verdict.
+    const pass = failures.every((f) => ADVISORY_FAILURE_KINDS.has(f.kind));
+    return {
+        pass,
+        tolerated: tolerate && pass,
+        failures,
+    };
+}
+function toFold(s) {
+    return s.toLowerCase();
+}
+function containsEither(a, b) {
+    return a.includes(b) || b.includes(a);
+}
+export async function evaluateAgainstCorpus(deps, options = {}, corpus = BENCH_CORPUS) {
+    let subset = corpus;
+    if (options.shapeFilter !== undefined) {
+        const shapeFilter = options.shapeFilter;
+        subset = subset.filter((e) => contractShape(e.expected) === shapeFilter);
+    }
+    if (options.limit !== undefined) {
+        subset = subset.slice(0, options.limit);
+    }
+    const outcomes = [];
+    for (let i = 0; i < subset.length; i++) {
+        const entry = subset[i];
+        // Thread the entry's per-app registered catalog through to
+        // `synthesizeContract`. When absent, synth falls through to the
+        // static stdlib hint baked into the system prompt.
+        const synth = await synthesizeContract({ llm: deps.llm }, entry.intent, entry.appGadgets !== undefined
+            ? { appGadgets: entry.appGadgets }
+            : undefined);
+        let outcome;
+        if (synth.contract === null) {
+            outcome = {
+                entry,
+                contract: null,
+                score: {
+                    pass: false,
+                    tolerated: false,
+                    failures: [
+                        {
+                            kind: 'synth-declined',
+                            hint: `synth declined: ${synth.reason}`,
+                        },
+                    ],
+                },
+                findings: [],
+                latencyMs: synth.latencyMs,
+                attempts: synth.attempts,
+                synthReason: synth.reason,
+            };
+        }
+        else {
+            const findings = validateContractStructure(synth.contract).findings;
+            const score = scoreSynthesizedContract(synth.contract, entry.expected);
+            outcome = {
+                entry,
+                contract: synth.contract,
+                score,
+                findings,
+                latencyMs: synth.latencyMs,
+                attempts: synth.attempts,
+                synthReason: synth.reason,
+            };
+        }
+        outcomes.push(outcome);
+        options.onProgress?.(outcome, i, subset.length);
+    }
+    return summarize(outcomes);
+}
+export function summarize(outcomes) {
+    const all = outcomes.length;
+    const pass = outcomes.filter((o) => o.score.pass).length;
+    const synthDeclined = outcomes.filter((o) => o.contract === null).length;
+    const fail = all - pass;
+    const precision = all === 0 ? 0 : pass / all;
+    // Roll precision up by contract SHAPE (the retired archetype
+    // categories are gone). Canonical order keeps the report stable;
+    // any shape the corpus doesn't exercise simply drops out.
+    const shapeOrder = [
+        'props-only',
+        'context-only',
+        'context+action',
+        'stream',
+        'with-gadgets',
+        'empty',
+    ];
+    const presentShapes = new Set(outcomes.map((o) => contractShape(o.entry.expected)));
+    const byShape = Object.fromEntries(shapeOrder
+        .filter((shape) => presentShapes.has(shape))
+        .map((shape) => {
+        const subset = outcomes.filter((o) => contractShape(o.entry.expected) === shape);
+        const subsetAll = subset.length;
+        const subsetPass = subset.filter((o) => o.score.pass).length;
+        return [
+            shape,
+            {
+                all: subsetAll,
+                pass: subsetPass,
+                precision: subsetAll === 0 ? 0 : subsetPass / subsetAll,
+            },
+        ];
+    }));
+    const redundantActionFindings = outcomes
+        .flatMap((o) => o.findings)
+        .filter((f) => f.kind === 'redundant-action').length;
+    const latencies = outcomes.map((o) => o.latencyMs).sort((a, b) => a - b);
+    const p50 = percentile(latencies, 0.5);
+    const p95 = percentile(latencies, 0.95);
+    return {
+        outcomes,
+        totals: { all, pass, fail, synthDeclined, precision },
+        byShape,
+        redundantActionFindings,
+        latency: { p50Ms: p50, p95Ms: p95 },
+    };
+}
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
+    return sorted[idx] ?? 0;
+}
+export function formatBenchReport(report) {
+    const lines = [];
+    lines.push('=== synth bench report ===');
+    lines.push('');
+    lines.push('Precision by contract shape:');
+    for (const [shape, stats] of Object.entries(report.byShape)) {
+        if (stats.all === 0)
+            continue;
+        const pct = (stats.precision * 100).toFixed(1);
+        lines.push(`  ${shape.padEnd(16)} ${stats.pass}/${stats.all} (${pct}%)`);
+    }
+    lines.push('');
+    lines.push(`Overall:                  ${report.totals.pass}/${report.totals.all} (${(report.totals.precision * 100).toFixed(1)}%)`);
+    lines.push(`Synth declined:           ${report.totals.synthDeclined}`);
+    lines.push(`Redundant-action firings: ${report.redundantActionFindings}`);
+    // Free-vocabulary name mismatches — surfaced, non-gating.
+    const advisoryCount = report.outcomes.reduce((n, o) => n +
+        o.score.failures.filter((f) => ADVISORY_FAILURE_KINDS.has(f.kind))
+            .length, 0);
+    lines.push(`Name advisories (non-gating): ${advisoryCount}`);
+    lines.push(`Latency:                  p50=${report.latency.p50Ms}ms p95=${report.latency.p95Ms}ms`);
+    // Repair-loop turn distribution — `1t×N` means N entries synthesized
+    // on the first attempt (no repair retry). Early-skip outcomes
+    // (attempts=0) are excluded.
+    const turnDist = new Map();
+    for (const o of report.outcomes) {
+        if (o.attempts > 0) {
+            turnDist.set(o.attempts, (turnDist.get(o.attempts) ?? 0) + 1);
+        }
+    }
+    const turnStr = [...turnDist.keys()]
+        .sort((a, b) => a - b)
+        .map((t) => `${t}t×${turnDist.get(t)}`)
+        .join(' ');
+    lines.push(`Attempts:                 ${turnStr || '(none)'}`);
+    const failed = report.outcomes.filter((o) => !o.score.pass);
+    if (failed.length > 0) {
+        lines.push('');
+        lines.push('Failures:');
+        for (const o of failed) {
+            lines.push(`  [${contractShape(o.entry.expected)}] ${o.entry.id}: ${o.entry.intent.slice(0, 60)}`);
+            for (const f of o.score.failures) {
+                lines.push(`    ${f.kind}: ${f.hint}`);
+            }
+            if (o.findings.length > 0) {
+                for (const f of o.findings) {
+                    lines.push(`    [${f.severity}:${f.kind}] ${f.hint.slice(0, 120)}`);
+                }
+            }
+        }
+    }
+    return lines.join('\n');
+}
+export function runSynthBench(deps, options = {}) {
+    return evaluateAgainstCorpus(deps, options);
+}