@ggui-ai/negotiator 0.28.0 → 0.29.0

package/dist/synth-bench/run-repair-bench-cli.js

@@ -1,86 +0,0 @@
-#!/usr/bin/env node
-/**
- * Repair bench CLI — live round-trip-quality probe.
- *
- * Runs the production forgiving-handshake create-path
- * (`ensureConformingContract`) over REPAIR_CORPUS and reports the
- * round-trip-usable rate — the contract-quality number the shape bench
- * (bench-synth) cannot see. Reads ~/.ggui/credentials.json (or
- * ANTHROPIC_API_KEY) for the key.
- *
- * Costs ~$0.001 per repaired entry on Haiku 4.5 (clean drafts hit the
- * lint-clean fast path and make NO LLM call). NOT run in CI; opt-in.
- *
- * Usage:
- *   pnpm -F @ggui-ai/negotiator bench-repair
- *   pnpm -F @ggui-ai/negotiator bench-repair -- --limit 1
- *   pnpm -F @ggui-ai/negotiator bench-repair -- --json > repair-report.json
- *   ANTHROPIC_API_KEY=sk-... pnpm -F @ggui-ai/negotiator bench-repair
- *
- * Bench-only — not exported from the package index.
- */
-import { evaluateRepairCorpus, formatRepairBenchReport, } from './run-repair-bench.js';
-import { DEFAULT_MODEL, HAIKU_4_5_PRICE_INPUT_PER_TOKEN, HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN, buildAnthropicLlmCaller, getTokenUsage, resolveAnthropicKey, } from './cli-llm.js';
-function parseArgs(argv) {
-    let limit;
-    let model = DEFAULT_MODEL;
-    let json = false;
-    for (let i = 0; i < argv.length; i++) {
-        const a = argv[i];
-        if (a === '--limit' && argv[i + 1]) {
-            limit = Number(argv[++i]);
-        }
-        else if (a === '--model' && argv[i + 1]) {
-            model = argv[++i];
-        }
-        else if (a === '--json') {
-            json = true;
-        }
-    }
-    const result = { model, json };
-    if (limit !== undefined)
-        result.limit = limit;
-    return result;
-}
-async function main() {
-    const args = parseArgs(process.argv.slice(2));
-    const apiKey = resolveAnthropicKey('bench-repair');
-    const llm = buildAnthropicLlmCaller(apiKey, args.model);
-    if (!args.json) {
-        process.stdout.write(`bench-repair: model=${args.model}\n\n`);
-    }
-    const report = await evaluateRepairCorpus({ llm }, {
-        ...(args.limit !== undefined ? { limit: args.limit } : {}),
-        onProgress: args.json
-            ? undefined
-            : (outcome, idx, total) => {
-                const pass = outcome.roundTrip !== null
-                    ? outcome.roundTrip.pass
-                    : outcome.shape.pass;
-                const status = pass ? 'OK ' : 'NO ';
-                const id = outcome.entry.id.padEnd(22);
-                const origin = `origin:${outcome.origin}`.padEnd(13);
-                process.stdout.write(`[${status}] ${(idx + 1).toString().padStart(2)}/${total}  ${origin}  ${id}  ${outcome.latencyMs}ms\n`);
-            },
-    });
-    if (args.json) {
-        process.stdout.write(JSON.stringify(report, null, 2));
-        process.stdout.write('\n');
-        return;
-    }
-    process.stdout.write('\n');
-    process.stdout.write(formatRepairBenchReport(report));
-    process.stdout.write('\n\n');
-    const usage = getTokenUsage();
-    const totalCost = usage.input * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
-        usage.output * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
-    // Clean drafts (origin agent) make no LLM call — cost is per repaired entry.
-    const callsMade = report.totals.originSynth;
-    const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
-    process.stdout.write(`Tokens: input=${usage.input} output=${usage.output}\n`);
-    process.stdout.write(`Cost:   total=$${totalCost.toFixed(4)} per-repair=$${costPerCall.toFixed(4)}\n`);
-}
-main().catch((err) => {
-    process.stderr.write(`bench-repair failed: ${err instanceof Error ? err.message : String(err)}\n`);
-    process.exit(1);
-});

package/src/rerank-eval/run-probe-cli.ts

@@ -1,103 +0,0 @@
-#!/usr/bin/env node
-/**
- * Rerank quality probe CLI.
- *
- * Reads `~/.ggui/credentials.json` for the Anthropic API key, runs
- * the probe against `claude-haiku-5-5` (ggui#1804), prints the report. Its readings are not comparable to runs made before the switch.
- *
- * Usage:
- *   pnpm -F @ggui-ai/negotiator probe-rerank
- *   ANTHROPIC_API_KEY=sk-... pnpm -F @ggui-ai/negotiator probe-rerank
- *   pnpm -F @ggui-ai/negotiator probe-rerank -- --limit 5
- *
- * Cost: ~$0.025 for the full 25-pair run with Haiku 4.5.
- *
- * Eval-only — not exported from the package index.
- */
-import { runProbe, formatReport } from './run-probe.js';
-import {
-  buildAnthropicLlmCaller,
-  getTokenUsage,
-  resolveAnthropicKey,
-} from '../synth-bench/cli-llm.js';
-
-const DEFAULT_MODEL = 'claude-haiku-5-5';
-
-function parseArgs(argv: readonly string[]): { limit?: number; threshold?: number; model: string } {
-  let limit: number | undefined;
-  let threshold: number | undefined;
-  let model = DEFAULT_MODEL;
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i];
-    if (a === '--limit' && argv[i + 1]) {
-      limit = Number(argv[++i]);
-    } else if (a === '--threshold' && argv[i + 1]) {
-      threshold = Number(argv[++i]);
-    } else if (a === '--model' && argv[i + 1]) {
-      model = argv[++i]!;
-    }
-  }
-  const result: { limit?: number; threshold?: number; model: string } = { model };
-  if (limit !== undefined) result.limit = limit;
-  if (threshold !== undefined) result.threshold = threshold;
-  return result;
-}
-
-// Approximate Haiku 4.5 pricing as of 2026-05.
-// Input: $1.00 / Mtok, output: $5.00 / Mtok.
-const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
-const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
-
-async function main(): Promise<void> {
-  const args = parseArgs(process.argv.slice(2));
-  const apiKey = resolveAnthropicKey('probe-rerank');
-  const llm = buildAnthropicLlmCaller(apiKey, args.model);
-
-  process.stdout.write(`probe: model=${args.model}\n\n`);
-  const report = await runProbe(
-    { llm },
-    {
-      ...(args.limit !== undefined ? { limit: args.limit } : {}),
-      ...(args.threshold !== undefined ? { threshold: args.threshold } : {}),
-      onProgress: (outcome, idx, total) => {
-        const status = outcome.correct ? 'OK ' : 'NO ';
-        const conf = outcome.decision.confidence.toFixed(2);
-        const id = outcome.pair.id.padEnd(24);
-        process.stdout.write(
-          `[${status}] ${(idx + 1).toString().padStart(2)}/${total}  ${id}  conf=${conf}  pred=${outcome.predictedMatchId ?? 'null'}  gold=${outcome.pair.goldMatchId ?? 'null'}\n`,
-        );
-      },
-    },
-  );
-
-  process.stdout.write('\n');
-  process.stdout.write(formatReport(report));
-  process.stdout.write('\n\n');
-
-  const usage = getTokenUsage();
-  const totalCost =
-    usage.input * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
-    usage.output * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
-  const callsMade = report.outcomes.filter(
-    // ggui#1235 — `reason` is optional on the seam; a judge with no prose
-    // still made a call.
-    (o) => !/short-circuited/.test(o.decision.reason ?? ''),
-  ).length;
-  const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
-  process.stdout.write(
-    `Tokens:         input=${usage.input} · output=${usage.output}\n`,
-  );
-  process.stdout.write(
-    `Cost:           total=$${totalCost.toFixed(4)} · per-call=$${costPerCall.toFixed(4)}\n`,
-  );
-  process.stdout.write(
-    `  G4 cost ≤ $0.002/call →  ${costPerCall <= 0.002 ? 'PASS' : 'FAIL'} ($${costPerCall.toFixed(4)})\n`,
-  );
-}
-
-main().catch((err) => {
-  process.stderr.write(
-    `probe-rerank failed: ${err instanceof Error ? err.message : String(err)}\n`,
-  );
-  process.exit(1);
-});

package/src/synth-bench/cache-key-probe.ts

@@ -1,213 +0,0 @@
-#!/usr/bin/env node
-/**
- * Blueprint cache-key stability probe.
- *
- * The handshake blueprint cache hits ONLY on exact canonical-contract
- * equality: `blueprintKey(contract)` = sha256 of `canonicalizeContracts`
- * (RFC 8785), which preserves spec placement, action/slot/tool NAMES and
- * schema shapes (it only normalizes key order / whitespace and strips
- * `description`/`usage`). The semantic/fuzzy match is disabled whenever a
- * contract is present. So two requests for the SAME intent reuse a cached
- * blueprint only if the LLM authored a byte-identical canonical contract
- * both times.
- *
- * This probe synthesizes one intent N times and reports:
- *   1. how many DISTINCT blueprintKeys result (distinct > 1 ⇒ the
- *      exact-key cache structurally cannot hit on re-runs);
- *   2. the KEY-AFFECTING diff — every leaf path where the runs disagree
- *      (descriptions/usage stripped, since the key ignores them) — so we
- *      can see WHETHER the variance is trivial (normalizable: required
- *      order, additionalProperties, optional fields) or semantic
- *      (property names / structure → only a semantic match can bridge it).
- *
- * Usage:
- *   pnpm -F @ggui-ai/negotiator cache-probe -- --intent "my todo items" --n 5
- *   pnpm -F @ggui-ai/negotiator cache-probe -- --n 5 --dump /tmp/cache-probe.json
- */
-import { writeFileSync } from 'node:fs';
-import { blueprintKey } from '@ggui-ai/protocol/blueprint-key';
-import type { DataContract } from '@ggui-ai/protocol';
-import { synthesizeContract } from '../synthesize-contract.js';
-import {
-  DEFAULT_MODEL,
-  buildAnthropicLlmCaller,
-  resolveAnthropicKey,
-} from './cli-llm.js';
-
-interface Args {
-  intent: string;
-  n: number;
-  model: string;
-  dump?: string;
-}
-
-function parseArgs(argv: readonly string[]): Args {
-  let intent = 'my todo items';
-  let n = 3;
-  let model = DEFAULT_MODEL;
-  let dump: string | undefined;
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i];
-    if (a === '--intent' && argv[i + 1]) intent = argv[++i]!;
-    else if (a === '--n' && argv[i + 1]) n = Number(argv[++i]);
-    else if (a === '--model' && argv[i + 1]) model = argv[++i]!;
-    else if (a === '--dump' && argv[i + 1]) dump = argv[++i]!;
-  }
-  return { intent, n, model, ...(dump !== undefined ? { dump } : {}) };
-}
-
-/** One-line canonical-surface summary — the bits blueprintKey is sensitive to. */
-function shapeSummary(c: DataContract): string {
-  const parts: string[] = [];
-  const props = Object.keys(c.propsSpec?.properties ?? {});
-  const ctx = Object.keys(c.contextSpec ?? {});
-  const act = Object.keys(c.actionSpec ?? {});
-  const str = Object.keys(c.streamSpec ?? {});
-  const tools = Object.keys(c.agentCapabilities?.tools ?? {});
-  if (props.length) parts.push(`props{${props.join(',')}}`);
-  if (ctx.length) parts.push(`context{${ctx.join(',')}}`);
-  if (act.length) parts.push(`action{${act.join(',')}}`);
-  if (str.length) parts.push(`stream{${str.join(',')}}`);
-  if (tools.length) parts.push(`tools{${tools.join(',')}}`);
-  return parts.join(' ') || '(empty)';
-}
-
-/**
- * Flatten a contract to `path → json-encoded leaf`, applying the same
- * domain rule the cache key does: drop string-valued `description` /
- * `usage` (informational — `canonicalizeContracts` strips them). Objects
- * recurse; arrays are encoded whole (order is key-affecting under JCS,
- * which sorts object keys but NOT array elements).
- */
-function flattenKeyAffecting(
-  value: unknown,
-  prefix: string,
-  out: Map<string, string>,
-): void {
-  if (Array.isArray(value)) {
-    out.set(prefix, JSON.stringify(value));
-    return;
-  }
-  if (value !== null && typeof value === 'object') {
-    for (const [k, v] of Object.entries(value)) {
-      if ((k === 'description' || k === 'usage') && typeof v === 'string') {
-        continue; // stripped by the cache key
-      }
-      flattenKeyAffecting(v, prefix ? `${prefix}.${k}` : k, out);
-    }
-    return;
-  }
-  out.set(prefix, JSON.stringify(value));
-}
-
-/** Every leaf path where the N contracts do not all agree, with the
- *  distinct per-run values (or `∅` when a run omits the path). */
-function crossRunDiff(
-  contracts: readonly DataContract[],
-): Array<{ path: string; values: string[] }> {
-  const flat = contracts.map((c) => {
-    const m = new Map<string, string>();
-    flattenKeyAffecting(c, '', m);
-    return m;
-  });
-  const allPaths = new Set<string>();
-  for (const m of flat) for (const p of m.keys()) allPaths.add(p);
-
-  const divergent: Array<{ path: string; values: string[] }> = [];
-  for (const path of [...allPaths].sort()) {
-    const values = flat.map((m) => m.get(path) ?? '∅');
-    if (new Set(values).size > 1) divergent.push({ path, values });
-  }
-  return divergent;
-}
-
-async function main(): Promise<void> {
-  const args = parseArgs(process.argv.slice(2));
-  const llm = buildAnthropicLlmCaller(resolveAnthropicKey('cache-probe'), args.model);
-  process.stdout.write(
-    `cache-probe: intent=${JSON.stringify(args.intent)} n=${args.n} model=${args.model}\n\n`,
-  );
-
-  const contracts: DataContract[] = [];
-  const keys: string[] = [];
-  for (let i = 0; i < args.n; i++) {
-    const synth = await synthesizeContract({ llm }, args.intent);
-    if (synth.contract === null) {
-      process.stdout.write(`  run ${i + 1}: synth declined — ${synth.reason}\n`);
-      continue;
-    }
-    const key = blueprintKey(synth.contract);
-    contracts.push(synth.contract);
-    keys.push(key);
-    process.stdout.write(
-      `  run ${i + 1}: ${key}  ${shapeSummary(synth.contract)}\n`,
-    );
-  }
-
-  const distinct = new Set(keys);
-  process.stdout.write(
-    `\nDistinct blueprintKeys: ${distinct.size}/${keys.length}\n`,
-  );
-  if (distinct.size <= 1 && keys.length > 1) {
-    process.stdout.write(
-      'STABLE → the exact-key cache WOULD hit on a re-run of this intent.\n',
-    );
-  } else if (keys.length > 1) {
-    process.stdout.write(
-      'UNSTABLE → same intent yields different canonical contracts → the exact-key\n' +
-        'cache cannot hit; each request cold-gens a new blueprint.\n',
-    );
-  }
-
-  // Key-affecting cross-run diff — the "what varies" the user asked for.
-  if (contracts.length > 1) {
-    const diff = crossRunDiff(contracts);
-    process.stdout.write(
-      `\n=== KEY-AFFECTING DIFF (${diff.length} divergent paths; descriptions stripped) ===\n`,
-    );
-    for (const { path, values } of diff) {
-      process.stdout.write(`  ${path}\n`);
-      values.forEach((v, i) => {
-        const capped = v.length > 100 ? `${v.slice(0, 100)}…` : v;
-        process.stdout.write(`    run${i + 1}: ${capped}\n`);
-      });
-    }
-    // Heuristic taxonomy hint.
-    const isArrayOrderOnly = diff.every(({ values }) => {
-      const parsed = values
-        .filter((v) => v !== '∅')
-        .map((v) => {
-          try {
-            return JSON.stringify([...(JSON.parse(v) as unknown[])].sort());
-          } catch {
-            return v;
-          }
-        });
-      return new Set(parsed).size === 1;
-    });
-    process.stdout.write(
-      `\nHint: ${
-        diff.length === 0
-          ? 'no key-affecting differences — variance must be elsewhere'
-          : isArrayOrderOnly
-            ? 'ALL divergent paths are arrays differing only in ORDER → a canonical sort in canonicalizeContracts would collapse these keys (cheap fix, no semantic match needed).'
-            : 'divergent paths include non-array-order differences (names / types / structure / presence) → a semantic intent-match is needed to bridge them (canonicalization alone cannot).'
-      }\n`,
-    );
-  }
-
-  if (args.dump !== undefined) {
-    writeFileSync(
-      args.dump,
-      JSON.stringify({ intent: args.intent, model: args.model, keys, contracts }, null, 2),
-    );
-    process.stdout.write(`\nDumped ${contracts.length} contracts → ${args.dump}\n`);
-  }
-}
-
-main().catch((err) => {
-  process.stderr.write(
-    `cache-probe failed: ${err instanceof Error ? err.message : String(err)}\n`,
-  );
-  process.exit(1);
-});

package/src/synth-bench/cli-llm.ts

@@ -1,168 +0,0 @@
-/**
- * Shared CLI LLM plumbing for the synth-bench family of live probes
- * (bench-synth, bench-repair). Builds an Anthropic-backed
- * {@link LLMCaller} with forced tool-use, resolves the API key from env
- * or `~/.ggui/credentials.json`, and accumulates token usage for the
- * per-run cost line. Bench-only — not exported from the package index.
- */
-import { readFileSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { resolve as pathResolve } from 'node:path';
-import type { LLMCaller, ToolSchema } from '../llm-caller.js';
-import { anthropicRejectsForcedToolChoice } from '@ggui-ai/protocol';
-
-const ANTHROPIC_API = 'https://api.anthropic.com/v1/messages';
-
-/** Default bench model — the model the `ui-gen-default` seed generator declares. */
-export const DEFAULT_MODEL = 'claude-haiku-5-5';
-
-/** Haiku 4.5 token pricing (USD per token) for the cost report. */
-export const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
-export const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
-
-interface CredsFile {
-  apps?: { global?: { anthropic?: string } };
-}
-
-/**
- * Resolve the Anthropic API key: `ANTHROPIC_API_KEY` env var first, then
- * `~/.ggui/credentials.json` at `apps.global.anthropic`. `scriptName`
- * prefixes the error hints so a failure names the bench that needs it.
- */
-export function resolveAnthropicKey(scriptName: string): string {
-  const envKey = process.env['ANTHROPIC_API_KEY'];
-  if (envKey && envKey.length > 0) return envKey;
-  const credsPath = pathResolve(homedir(), '.ggui', 'credentials.json');
-  let parsed: CredsFile;
-  try {
-    parsed = JSON.parse(readFileSync(credsPath, 'utf8')) as CredsFile;
-  } catch (err) {
-    throw new Error(
-      `${scriptName}: could not read ${credsPath} (${err instanceof Error ? err.message : String(err)}). Set ANTHROPIC_API_KEY env var or run \`ggui auth set anthropic\`.`,
-    );
-  }
-  const key = parsed.apps?.global?.anthropic;
-  if (typeof key !== 'string' || key.length === 0) {
-    throw new Error(
-      `${scriptName}: no anthropic key found at apps.global.anthropic in ${credsPath}.`,
-    );
-  }
-  return key;
-}
-
-interface AnthropicContentBlock {
-  type: string;
-  name?: string;
-  input?: unknown;
-  text?: string;
-}
-
-interface AnthropicResponse {
-  content?: AnthropicContentBlock[];
-  usage?: { input_tokens?: number; output_tokens?: number };
-  stop_reason?: string;
-  error?: { type?: string; message?: string };
-}
-
-let totalInputTokens = 0;
-let totalOutputTokens = 0;
-
-/** Running token usage accumulated across `callStructured` calls in this
- *  process — read after a run for the cost line. */
-export function getTokenUsage(): {
-  readonly input: number;
-  readonly output: number;
-} {
-  return { input: totalInputTokens, output: totalOutputTokens };
-}
-
-/**
- * Thinking budget added to the caller's answer budget for a model that
- * refuses a forced tool (ggui#1264). Same value and reason as the
- * negotiator's production caller in `@ggui-ai/mcp-server`; the three
- * copies are consolidated under ggui#1271.
- */
-const ALWAYS_THINKING_HEADROOM_TOKENS = 16_000;
-
-/**
- * The dev caller the synth benches and the rerank probe CLI share
- * (Haiku-pinned by default; every CLI takes `--model`). Text mode is not
- * exercised — every consumer uses `callStructured`.
- */
-export function buildAnthropicLlmCaller(
-  apiKey: string,
-  model: string,
-): LLMCaller {
-  return {
-    async call(): Promise<string> {
-      throw new Error(
-        'negotiator dev caller: text-mode not exercised — use callStructured',
-      );
-    },
-    async callStructured(
-      systemPrompt: string,
-      userMessage: string,
-      tool: ToolSchema,
-      maxTokens?: number,
-    ): Promise<unknown> {
-      // `temperature` deprecated on Haiku 4.5+ — Anthropic rejects with
-      // HTTP 400. Residual stochasticity stays bounded via canonical-key
-      // normalization downstream.
-      //
-      // ggui#1264 — a forced `tool_choice` only where the model accepts
-      // one. The always-thinking models (`@ggui-ai/protocol`
-      // `anthropicRejectsForcedToolChoice`) 400 on it, so they get
-      // `auto` + at most one call + the tool named in the system prompt,
-      // and thinking headroom on top of the answer budget (their thinking
-      // counts against `max_tokens` and comes first).
-      const refusesForcedTool = anthropicRejectsForcedToolChoice(model);
-      const answerBudget = maxTokens ?? 1024;
-      const body = {
-        model,
-        max_tokens: refusesForcedTool
-          ? answerBudget + ALWAYS_THINKING_HEADROOM_TOKENS
-          : answerBudget,
-        system: refusesForcedTool
-          ? `${systemPrompt}\n\nAnswer by calling the \`${tool.name}\` tool exactly once. Do not answer in text.`
-          : systemPrompt,
-        messages: [{ role: 'user', content: userMessage }],
-        tools: [
-          {
-            name: tool.name,
-            description: tool.description,
-            input_schema: tool.input_schema,
-          },
-        ],
-        tool_choice: refusesForcedTool
-          ? { type: 'auto', disable_parallel_tool_use: true }
-          : { type: 'tool', name: tool.name },
-      };
-      const res = await fetch(ANTHROPIC_API, {
-        method: 'POST',
-        headers: {
-          'content-type': 'application/json',
-          'x-api-key': apiKey,
-          'anthropic-version': '2023-06-01',
-        },
-        body: JSON.stringify(body),
-      });
-      const json = (await res.json()) as AnthropicResponse;
-      if (!res.ok) {
-        const errType = json.error?.type ?? 'unknown';
-        const errMsg = json.error?.message ?? `HTTP ${res.status}`;
-        throw new Error(`anthropic ${errType}: ${errMsg}`);
-      }
-      if (json.usage) {
-        totalInputTokens += json.usage.input_tokens ?? 0;
-        totalOutputTokens += json.usage.output_tokens ?? 0;
-      }
-      const toolBlock = json.content?.find((b) => b.type === 'tool_use');
-      if (!toolBlock || toolBlock.input === undefined) {
-        throw new Error(
-          `anthropic: no tool_use block in response (stop_reason=${json.stop_reason ?? 'unknown'})`,
-        );
-      }
-      return toolBlock.input;
-    },
-  };
-}