@ggui-ai/negotiator 0.1.0-rc.1

package/src/rerank-eval/run-probe-cli.ts

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+/**
+ * Rerank quality probe CLI.
+ *
+ * Reads `~/.ggui/credentials.json` for the Anthropic API key, runs
+ * the probe against `claude-haiku-4-5`, prints the report.
+ *
+ * Usage:
+ *   pnpm -F @ggui-ai/negotiator probe-rerank
+ *   ANTHROPIC_API_KEY=sk-... pnpm -F @ggui-ai/negotiator probe-rerank
+ *   pnpm -F @ggui-ai/negotiator probe-rerank -- --limit 5
+ *
+ * Cost: ~$0.025 for the full 25-pair run with Haiku 4.5.
+ *
+ * Eval-only — not exported from the package index.
+ */
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { resolve as pathResolve } from 'node:path';
+import { runProbe, formatReport } from './run-probe.js';
+import type { LLMCaller, ToolSchema } from '../llm-caller.js';
+
+const ANTHROPIC_API = 'https://api.anthropic.com/v1/messages';
+const DEFAULT_MODEL = 'claude-haiku-4-5';
+
+interface CredsFile {
+  apps?: { global?: { anthropic?: string } };
+}
+
+function resolveAnthropicKey(): string {
+  const envKey = process.env['ANTHROPIC_API_KEY'];
+  if (envKey && envKey.length > 0) return envKey;
+  const credsPath = pathResolve(homedir(), '.ggui', 'credentials.json');
+  let parsed: CredsFile;
+  try {
+    parsed = JSON.parse(readFileSync(credsPath, 'utf8')) as CredsFile;
+  } catch (err) {
+    throw new Error(
+      `probe-rerank: could not read ${credsPath} (${err instanceof Error ? err.message : String(err)}). Set ANTHROPIC_API_KEY env var or run \`ggui auth set anthropic\`.`,
+    );
+  }
+  const key = parsed.apps?.global?.anthropic;
+  if (typeof key !== 'string' || key.length === 0) {
+    throw new Error(
+      `probe-rerank: no anthropic key found at apps.global.anthropic in ${credsPath}.`,
+    );
+  }
+  return key;
+}
+
+interface AnthropicContentBlock {
+  type: string;
+  name?: string;
+  input?: unknown;
+  text?: string;
+}
+
+interface AnthropicResponse {
+  content?: AnthropicContentBlock[];
+  usage?: { input_tokens?: number; output_tokens?: number };
+  stop_reason?: string;
+  error?: { type?: string; message?: string };
+}
+
+let totalInputTokens = 0;
+let totalOutputTokens = 0;
+
+function buildAnthropicLlmCaller(apiKey: string, model: string): LLMCaller {
+  return {
+    async call(): Promise<string> {
+      throw new Error('probe-cli: text-mode not exercised — use callStructured');
+    },
+    async callStructured<T>(
+      systemPrompt: string,
+      userMessage: string,
+      tool: ToolSchema,
+      maxTokens?: number,
+    ): Promise<T> {
+      const body = {
+        model,
+        max_tokens: maxTokens ?? 1024,
+        system: systemPrompt,
+        messages: [{ role: 'user', content: userMessage }],
+        tools: [
+          {
+            name: tool.name,
+            description: tool.description,
+            input_schema: tool.input_schema,
+          },
+        ],
+        tool_choice: { type: 'tool', name: tool.name },
+      };
+      const res = await fetch(ANTHROPIC_API, {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-api-key': apiKey,
+          'anthropic-version': '2023-06-01',
+        },
+        body: JSON.stringify(body),
+      });
+      const json = (await res.json()) as AnthropicResponse;
+      if (!res.ok) {
+        const errType = json.error?.type ?? 'unknown';
+        const errMsg = json.error?.message ?? `HTTP ${res.status}`;
+        throw new Error(`anthropic ${errType}: ${errMsg}`);
+      }
+      if (json.usage) {
+        totalInputTokens += json.usage.input_tokens ?? 0;
+        totalOutputTokens += json.usage.output_tokens ?? 0;
+      }
+      const toolBlock = json.content?.find((b) => b.type === 'tool_use');
+      if (!toolBlock || toolBlock.input === undefined) {
+        throw new Error(
+          `anthropic: no tool_use block in response (stop_reason=${json.stop_reason ?? 'unknown'})`,
+        );
+      }
+      return toolBlock.input as T;
+    },
+  };
+}
+
+function parseArgs(argv: readonly string[]): { limit?: number; threshold?: number; model: string } {
+  let limit: number | undefined;
+  let threshold: number | undefined;
+  let model = DEFAULT_MODEL;
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--limit' && argv[i + 1]) {
+      limit = Number(argv[++i]);
+    } else if (a === '--threshold' && argv[i + 1]) {
+      threshold = Number(argv[++i]);
+    } else if (a === '--model' && argv[i + 1]) {
+      model = argv[++i]!;
+    }
+  }
+  const result: { limit?: number; threshold?: number; model: string } = { model };
+  if (limit !== undefined) result.limit = limit;
+  if (threshold !== undefined) result.threshold = threshold;
+  return result;
+}
+
+// Approximate Haiku 4.5 pricing as of 2026-05.
+// Input: $1.00 / Mtok, output: $5.00 / Mtok.
+const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
+const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+  const apiKey = resolveAnthropicKey();
+  const llm = buildAnthropicLlmCaller(apiKey, args.model);
+
+  process.stdout.write(`probe: model=${args.model}\n\n`);
+  const report = await runProbe(
+    { llm },
+    {
+      ...(args.limit !== undefined ? { limit: args.limit } : {}),
+      ...(args.threshold !== undefined ? { threshold: args.threshold } : {}),
+      onProgress: (outcome, idx, total) => {
+        const status = outcome.correct ? 'OK ' : 'NO ';
+        const conf = outcome.decision.confidence.toFixed(2);
+        const id = outcome.pair.id.padEnd(24);
+        process.stdout.write(
+          `[${status}] ${(idx + 1).toString().padStart(2)}/${total}  ${id}  conf=${conf}  pred=${outcome.predictedMatchId ?? 'null'}  gold=${outcome.pair.goldMatchId ?? 'null'}\n`,
+        );
+      },
+    },
+  );
+
+  process.stdout.write('\n');
+  process.stdout.write(formatReport(report));
+  process.stdout.write('\n\n');
+
+  const totalCost =
+    totalInputTokens * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
+    totalOutputTokens * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
+  const callsMade = report.outcomes.filter(
+    (o) => !/short-circuited/.test(o.decision.reason),
+  ).length;
+  const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
+  process.stdout.write(
+    `Tokens:         input=${totalInputTokens} · output=${totalOutputTokens}\n`,
+  );
+  process.stdout.write(
+    `Cost:           total=$${totalCost.toFixed(4)} · per-call=$${costPerCall.toFixed(4)}\n`,
+  );
+  process.stdout.write(
+    `  G4 cost ≤ $0.002/call →  ${costPerCall <= 0.002 ? 'PASS' : 'FAIL'} ($${costPerCall.toFixed(4)})\n`,
+  );
+}
+
+main().catch((err) => {
+  process.stderr.write(
+    `probe-rerank failed: ${err instanceof Error ? err.message : String(err)}\n`,
+  );
+  process.exit(1);
+});

package/src/rerank-eval/run-probe.ts

@@ -0,0 +1,198 @@
+/**
+ * Rerank quality probe.
+ *
+ * Runs the configured `LLMCaller` over the {@link EVAL_PAIRS} eval
+ * set and returns precision@1 + adversarial false-positive +
+ * latency p95. The caller compares these against the quality gates
+ * (precision, adversarial false-positive rate, latency, cost)
+ * reported by the probe CLI.
+ *
+ * Eval-only — not exported from the package index.
+ */
+import { rerankCandidates, type RerankDecision } from '../llm-rerank.js';
+import type { LLMCaller } from '../llm-caller.js';
+import { EVAL_PAIRS, type EvalPair } from './pairs.js';
+
+export interface ProbeOutcome {
+  readonly pair: EvalPair;
+  readonly decision: RerankDecision;
+  /** Confidence threshold applied to compute correctness. */
+  readonly threshold: number;
+  /** Predicted matchId AFTER threshold gate (null if confidence below). */
+  readonly predictedMatchId: string | null;
+  /** Was the prediction correct vs gold? */
+  readonly correct: boolean;
+}
+
+export interface ProbeReport {
+  readonly outcomes: readonly ProbeOutcome[];
+  readonly totals: {
+    readonly all: number;
+    readonly correct: number;
+    readonly precision: number;
+  };
+  readonly byKind: Readonly<{
+    [K in EvalPair['kind']]: {
+      readonly all: number;
+      readonly correct: number;
+      readonly precision: number;
+    };
+  }>;
+  readonly adversarialFalsePositiveRate: number;
+  readonly latency: { readonly p50Ms: number; readonly p95Ms: number };
+  readonly threshold: number;
+}
+
+export interface RunProbeOptions {
+  /** Use a subset for smoke; default = all 25 pairs. */
+  readonly limit?: number;
+  /** Confidence threshold for considering a non-null match a hit. Default 0.6. */
+  readonly threshold?: number;
+  /**
+   * Optional progress callback. Fired AFTER each pair completes — the
+   * CLI uses this to print a per-pair line so the operator sees live
+   * progress instead of waiting for the full sweep to finish.
+   */
+  readonly onProgress?: (outcome: ProbeOutcome, index: number, total: number) => void;
+}
+
+/**
+ * Run the probe against `pairs` (defaults to {@link EVAL_PAIRS}).
+ * Sequential by design — small N (~25), and concurrent calls would
+ * complicate latency measurement without changing the gate decision.
+ */
+export async function runProbe(
+  deps: { readonly llm: LLMCaller },
+  options: RunProbeOptions = {},
+  pairs: readonly EvalPair[] = EVAL_PAIRS,
+): Promise<ProbeReport> {
+  const threshold = options.threshold ?? 0.6;
+  const subset = options.limit ? pairs.slice(0, options.limit) : pairs;
+  const outcomes: ProbeOutcome[] = [];
+
+  for (let i = 0; i < subset.length; i++) {
+    const pair = subset[i]!;
+    const decision = await rerankCandidates(deps, pair.query, pair.candidates);
+    const predictedMatchId =
+      decision.confidence >= threshold ? decision.matchId : null;
+    const correct = predictedMatchId === pair.goldMatchId;
+    const outcome: ProbeOutcome = {
+      pair,
+      decision,
+      threshold,
+      predictedMatchId,
+      correct,
+    };
+    outcomes.push(outcome);
+    options.onProgress?.(outcome, i, subset.length);
+  }
+
+  const totals = aggregate(outcomes);
+  const byKind = {
+    'should-match': aggregate(outcomes.filter((o) => o.pair.kind === 'should-match')),
+    'no-match': aggregate(outcomes.filter((o) => o.pair.kind === 'no-match')),
+    adversarial: aggregate(outcomes.filter((o) => o.pair.kind === 'adversarial')),
+  };
+  // Adversarial false-positive: judge accepted a candidate when gold
+  // says null. Compute as `(adversarial-pairs-with-prediction-when-
+  // gold-was-null) / (adversarial-pairs-with-gold-null)`.
+  const adversarial = outcomes.filter((o) => o.pair.kind === 'adversarial');
+  const adversarialNullGold = adversarial.filter((o) => o.pair.goldMatchId === null);
+  const adversarialFalsePositives = adversarialNullGold.filter(
+    (o) => o.predictedMatchId !== null,
+  );
+  const adversarialFalsePositiveRate =
+    adversarialNullGold.length === 0
+      ? 0
+      : adversarialFalsePositives.length / adversarialNullGold.length;
+
+  const latencies = outcomes
+    .map((o) => o.decision.latencyMs)
+    .sort((a, b) => a - b);
+  const latency = {
+    p50Ms: percentile(latencies, 0.5),
+    p95Ms: percentile(latencies, 0.95),
+  };
+
+  return {
+    outcomes,
+    totals,
+    byKind,
+    adversarialFalsePositiveRate,
+    latency,
+    threshold,
+  };
+}
+
+function aggregate(outcomes: readonly ProbeOutcome[]): {
+  readonly all: number;
+  readonly correct: number;
+  readonly precision: number;
+} {
+  const all = outcomes.length;
+  const correct = outcomes.filter((o) => o.correct).length;
+  const precision = all === 0 ? 0 : correct / all;
+  return { all, correct, precision };
+}
+
+function percentile(sorted: readonly number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  const idx = Math.min(
+    sorted.length - 1,
+    Math.floor(p * sorted.length),
+  );
+  return sorted[idx] ?? 0;
+}
+
+/** Pretty-print a {@link ProbeReport} for terminal output. */
+export function formatReport(report: ProbeReport): string {
+  const lines: string[] = [];
+  lines.push('━━━ rerank quality probe ━━━');
+  lines.push('');
+  lines.push(`Threshold: confidence ≥ ${report.threshold}`);
+  lines.push('');
+  lines.push('Precision by kind:');
+  for (const [kind, stats] of Object.entries(report.byKind) as Array<[
+    EvalPair['kind'],
+    { all: number; correct: number; precision: number },
+  ]>) {
+    const pct = (stats.precision * 100).toFixed(1);
+    lines.push(`  ${kind.padEnd(14)} ${stats.correct}/${stats.all} (${pct}%)`);
+  }
+  lines.push('');
+  lines.push(
+    `Overall:        ${report.totals.correct}/${report.totals.all} (${(report.totals.precision * 100).toFixed(1)}%)`,
+  );
+  lines.push(
+    `Adversarial FP: ${(report.adversarialFalsePositiveRate * 100).toFixed(1)}%`,
+  );
+  lines.push(
+    `Latency:        p50=${report.latency.p50Ms}ms · p95=${report.latency.p95Ms}ms`,
+  );
+  lines.push('');
+  lines.push('Gates:');
+  lines.push(
+    `  G1 precision@1 ≥ 85%    →  ${report.totals.precision >= 0.85 ? 'PASS' : 'FAIL'} (${(report.totals.precision * 100).toFixed(1)}%)`,
+  );
+  lines.push(
+    `  G2 adversarial FP ≤ 5%  →  ${report.adversarialFalsePositiveRate <= 0.05 ? 'PASS' : 'FAIL'} (${(report.adversarialFalsePositiveRate * 100).toFixed(1)}%)`,
+  );
+  lines.push(
+    `  G3 latency p95 ≤ 600ms  →  ${report.latency.p95Ms <= 600 ? 'PASS' : 'FAIL'} (p95=${report.latency.p95Ms}ms)`,
+  );
+
+  // Per-pair detail for failed predictions
+  const failed = report.outcomes.filter((o) => !o.correct);
+  if (failed.length > 0) {
+    lines.push('');
+    lines.push('Failed predictions:');
+    for (const o of failed) {
+      lines.push(
+        `  [${o.pair.kind}] ${o.pair.id}: predicted=${o.predictedMatchId ?? 'null'}, gold=${o.pair.goldMatchId ?? 'null'}, conf=${o.decision.confidence.toFixed(2)}`,
+      );
+      lines.push(`    reason: ${o.decision.reason}`);
+    }
+  }
+
+  return lines.join('\n');
+}

package/src/session.ts

@@ -0,0 +1,41 @@
+/**
+ * `SessionState` — what the decision engine sees about the live
+ * session at the moment of a negotiation call.
+ *
+ * Captures the UI stack (pages previously pushed into this
+ * session), recent conversation, and optional interface context
+ * (viewport, device class). Consumed by `NegotiatorDecisionInput`
+ * to drive `create / update / compose / replace` decisions — the
+ * stack tells the engine whether something already on screen can
+ * absorb this push as an `update`, and the conversation history
+ * feeds the decision LLM.
+ *
+ * Kept in `@ggui-ai/negotiator` (not `mcp-server-core`): this is
+ * the decision engine's input shape, not a storage seam. An MCP
+ * server implementer binding against the public `Negotiator`
+ * interface sees `NegotiatorInput` / `NegotiatorResult` — never
+ * this internal input shape. Community adapters that want to
+ * call `makeDecision` directly (bypassing the `Negotiator` wrapper)
+ * import this type.
+ */
+
+import type { DataContract, InterfaceContext } from '@ggui-ai/protocol';
+
+/**
+ * One entry on the session's UI stack — a page the agent previously
+ * pushed. Minimal shape: every field the decision engine actually
+ * reads to decide reuse vs. create.
+ */
+export interface SessionStackEntry {
+  id: string;
+  prompt?: string;
+  contract?: DataContract;
+  componentCode: string;
+}
+
+/** Current state of a session — stack + conversation history. */
+export interface SessionState {
+  stack: SessionStackEntry[];
+  conversationHistory: Array<{ role: 'user' | 'agent'; content: string }>;
+  interfaceContext?: InterfaceContext;
+}

package/src/suggestion.ts

@@ -0,0 +1,73 @@
+/**
+ * Suggestion engine — detects structured data patterns in free-form
+ * agent text and proposes a ggui_push call that would show them
+ * interactively.
+ *
+ * Pure regex heuristics — zero I/O, zero LLM. The agent can call this
+ * opportunistically on streamed output to decide whether to surface a
+ * UI suggestion. Callers must pass a deduplication set to avoid
+ * re-suggesting the same UI repeatedly in a live session.
+ */
+
+import { computeIntentId } from './intent.js';
+
+/** Suggestion event sent to the agent. */
+export interface NegotiatorSuggestion {
+  type: 'negotiator:suggest';
+  trigger: 'stream-data-detected' | 'session-context' | 'user-pattern';
+  message: string;
+  suggestedAction: {
+    tool: 'ggui_push';
+    input: { data?: Record<string, unknown>; prompt?: string };
+  };
+  confidence: number;
+  intentId: string;
+}
+
+interface DetectionPattern {
+  name: string;
+  regex: RegExp;
+  uiType: string;
+  confidence: number;
+}
+
+const PATTERNS: DetectionPattern[] = [
+  { name: 'temperature', regex: /\b\d+\s*°[CF]\b/i, uiType: 'weather display', confidence: 0.8 },
+  { name: 'percentage', regex: /\b\d+(\.\d+)?\s*%/, uiType: 'metrics display', confidence: 0.6 },
+  { name: 'currency', regex: /\$\s*[\d,]+(\.\d{2})?|\b\d+(\.\d{2})?\s*(USD|EUR|GBP|JPY|KRW)\b/i, uiType: 'financial display', confidence: 0.7 },
+  { name: 'speed-or-distance', regex: /\b\d+(\.\d+)?\s*(km\/h|mph|km|mi|m\/s)\b/i, uiType: 'data display', confidence: 0.6 },
+  { name: 'numbered-list', regex: /(?:^|\n)\s*[1-9]\.\s+\S/m, uiType: 'step flow or list', confidence: 0.5 },
+  { name: 'comparison', regex: /\b(vs\.?|versus|compared to|on the other hand|alternatively)\b/i, uiType: 'comparison view', confidence: 0.6 },
+  { name: 'table-like', regex: /\|.*\|.*\|/, uiType: 'table or grid', confidence: 0.8 },
+];
+
+/** Detect structured data patterns in streamed text. */
+export function detectDataPatterns(text: string): Array<{ pattern: string; uiType: string; confidence: number }> {
+  const matches: Array<{ pattern: string; uiType: string; confidence: number }> = [];
+  for (const p of PATTERNS) {
+    if (p.regex.test(text)) {
+      matches.push({ pattern: p.name, uiType: p.uiType, confidence: p.confidence });
+    }
+  }
+  return matches;
+}
+
+/** Build a suggestion event from detected patterns. */
+export function buildSuggestion(
+  sessionId: string,
+  detections: Array<{ pattern: string; uiType: string; confidence: number }>,
+  activeIntentIds: Set<string>,
+): NegotiatorSuggestion | null {
+  if (detections.length === 0) return null;
+  const best = detections.reduce((a, b) => (a.confidence > b.confidence ? a : b));
+  const intentId = computeIntentId(sessionId, { detectedPattern: best.pattern }, 'create');
+  if (activeIntentIds.has(intentId)) return null;
+  return {
+    type: 'negotiator:suggest',
+    trigger: 'stream-data-detected',
+    message: `I noticed structured data in your response (${best.pattern}). Consider calling ggui_push for an interactive ${best.uiType} instead.`,
+    suggestedAction: { tool: 'ggui_push', input: { prompt: best.uiType } },
+    confidence: best.confidence,
+    intentId,
+  };
+}