@tangle-network/agent-bench 0.1.0

package/src/search-bench/tasks.ts

@@ -0,0 +1,129 @@
+/**
+ * Fresh-docs coding tasks with DETERMINISTIC oracles — the substrate for
+ * comparing web-search backends inside coding harnesses.
+ *
+ * Each task asks for code that hinges on a specific current API detail. The
+ * oracle is a deterministic check on the agent's final answer (contains / omits
+ * / regex) — no LLM judge, so a you.com-vs-native delta is defensible. A task
+ * earns its place only if the detail is precise enough that a model guessing
+ * from memory plausibly gets it wrong but a correct doc lookup gets it right.
+ */
+
+export interface TaskOracle {
+  /** All of these (case-insensitive) must appear in the answer to pass. */
+  containsAll?: string[]
+  /** Passing answers must contain at least one of these (case-insensitive). */
+  containsAny?: string[]
+  /** None of these (e.g. hallucinated/deprecated APIs) may appear. */
+  notContains?: string[]
+  /** Every regex must match the answer. */
+  regex?: string[]
+}
+
+export interface SearchTask {
+  id: string
+  domain: string
+  /** The coding prompt sent to the agent. */
+  prompt: string
+  /** Why a correct answer depends on current documentation (provenance note). */
+  needsFreshDocs: string
+  oracle: TaskOracle
+  /** Canonical query + expected authority for an optional retrieval-quality check. */
+  searchQuery?: string
+  expectedUrlIncludes?: string[]
+  /** Primary-doc URL the correct answer was verified against (provenance). */
+  sourceUrl?: string
+  /** The verified correct answer (provenance — not shown to the agent). */
+  correctAnswer?: string
+}
+
+const ANSWER_CONTRACT =
+  'Answer with the exact code/identifiers requested. Cite the source URL(s) you used. Be precise — exact method, option, and import names matter.'
+
+export function taskToPrompt(task: SearchTask): string {
+  return `${task.prompt}\n\n${ANSWER_CONTRACT}`
+}
+
+/** Evaluate the deterministic oracle. Returns 1 (pass) or 0 (fail) + reasons. */
+export function scoreTask(task: SearchTask, answer: string): { score: 0 | 1; reasons: string[] } {
+  const hay = answer.toLowerCase()
+  const reasons: string[] = []
+  const o = task.oracle
+  for (const s of o.containsAll ?? []) {
+    if (!hay.includes(s.toLowerCase())) reasons.push(`missing required "${s}"`)
+  }
+  if (o.containsAny && !o.containsAny.some((s) => hay.includes(s.toLowerCase()))) {
+    reasons.push(`missing any of [${o.containsAny.join(', ')}]`)
+  }
+  for (const s of o.notContains ?? []) {
+    if (hay.includes(s.toLowerCase())) reasons.push(`contains forbidden "${s}"`)
+  }
+  for (const r of o.regex ?? []) {
+    if (!new RegExp(r, 'i').test(answer)) reasons.push(`regex /${r}/ did not match`)
+  }
+  return { score: reasons.length === 0 ? 1 : 0, reasons }
+}
+
+/**
+ * Seed tasks — stable, precisely-checkable current APIs. These prove the
+ * pipeline; the discriminating fresh-docs set is generated + verified
+ * separately (search-bench/generate). Answers verified against primary docs.
+ */
+export const seedTasks: SearchTask[] = [
+  {
+    id: 'cf-do-alarms',
+    domain: 'cloudflare-workers',
+    prompt:
+      'In a Cloudflare Workers Durable Object, write the handler that runs when an alarm fires and the single line that schedules an alarm 60 seconds from now. Give the exact method names.',
+    needsFreshDocs:
+      'The DO alarm API (storage.setAlarm + the alarm() handler) is easy to misname (e.g. scheduleAlarm/onAlarm) without the current docs.',
+    oracle: {
+      containsAll: ['setAlarm', 'alarm('],
+      notContains: ['scheduleAlarm', 'onAlarm'],
+    },
+    searchQuery: 'Cloudflare Workers Durable Objects alarms docs',
+    expectedUrlIncludes: ['/durable-objects'],
+  },
+  {
+    id: 'prisma-createmany-skipdup',
+    domain: 'prisma',
+    prompt:
+      'Write a Prisma Client `createMany` call that inserts an array of users and silently skips rows that would violate a unique constraint. Give the exact option name.',
+    needsFreshDocs:
+      'The exact option is `skipDuplicates: true` — models often invent `ignoreDuplicates`/`onConflict`.',
+    oracle: {
+      containsAll: ['createmany', 'skipduplicates'],
+      notContains: ['ignoreduplicates', 'onconflict'],
+    },
+    searchQuery: 'Prisma createMany skipDuplicates docs',
+    expectedUrlIncludes: ['createmany', 'skipduplicates'],
+  },
+  {
+    id: 'hono-middleware-header',
+    domain: 'hono',
+    prompt:
+      'Using the Hono web framework (TypeScript), write a middleware that sets a custom response header and then continues to the next handler. Show the exact import and the call used to continue the chain.',
+    needsFreshDocs:
+      'Hono middleware uses `c.header(...)` + `await next()`; models confuse it with Express `res.set`/`next()`.',
+    oracle: {
+      containsAll: ['hono', 'c.header', 'next()'],
+      notContains: ['res.set', 'res.setheader'],
+    },
+    searchQuery: 'Hono middleware TypeScript docs',
+    expectedUrlIncludes: ['middleware'],
+  },
+  {
+    id: 'vitest-vi-hoisted',
+    domain: 'vitest',
+    prompt:
+      'In Vitest, you need a variable referenced inside a `vi.mock` factory to be initialized before the mock is hoisted. Show the exact API that makes a value run before hoisting, with a minimal example.',
+    needsFreshDocs:
+      'The exact API is `vi.hoisted(() => …)`; without docs models suggest top-level consts (which fail hoisting) or jest patterns.',
+    oracle: {
+      containsAll: ['vi.hoisted'],
+      notContains: ['jest.mock'],
+    },
+    searchQuery: 'Vitest vi.hoisted docs',
+    expectedUrlIncludes: ['vi.hoisted'],
+  },
+]

package/src/search-tool.ts

@@ -0,0 +1,95 @@
+/**
+ * A host-side web-search tool for the `router-tools` worker backend. This is the
+ * capability axis the research benches were missing: finsearch's prompt demands
+ * "live web/market sources", but the plain `router` chat backend has no tools, so
+ * the worker can only recall/hallucinate. Wiring this tool into a `router-tools`
+ * loop gives the off-box worker a real `web_search(query)` it can call, backed by
+ * the Tangle router's search providers (you.com / exa). Off-box = unaffected by a
+ * sandbox's egress allowlist.
+ *
+ * The tool result the model reads is a compact text block of the top hits
+ * (title · date · snippet · url). Transport/HTTP failures fail loud (throw) so a
+ * broken search backend infra-excludes the cell instead of silently degrading to
+ * a recall-only answer; a valid-but-empty result returns an explicit "no results"
+ * string the model can reason about.
+ */
+import type { ToolSpec } from '@tangle-network/agent-runtime/loops'
+
+export const webSearchTool: ToolSpec = {
+  type: 'function',
+  function: {
+    name: 'web_search',
+    description:
+      'Search the live web for current facts, market data, filings, and news. Returns the top results as title, date, snippet, and URL. Call it as many times as needed, refining the query, before you answer.',
+    parameters: {
+      type: 'object',
+      properties: {
+        query: {
+          type: 'string',
+          description: 'The search query. Be specific — include entity, metric, period, and units.',
+        },
+      },
+      required: ['query'],
+    },
+  },
+}
+
+interface SearchHit {
+  title?: string
+  url?: string
+  snippet?: string
+  publishedAt?: string
+}
+
+interface SearchResponse {
+  data?: SearchHit[]
+}
+
+const formatHits = (hits: SearchHit[], maxHits: number): string => {
+  if (hits.length === 0) return 'No results found for that query. Try a different query.'
+  return hits
+    .slice(0, maxHits)
+    .map((h, i) => {
+      const date = h.publishedAt ? ` (${h.publishedAt.slice(0, 10)})` : ''
+      return `[${i + 1}] ${h.title ?? '(untitled)'}${date}\n${h.snippet ?? ''}\n${h.url ?? ''}`
+    })
+    .join('\n\n')
+}
+
+/**
+ * Build the `executeToolCall` handler for a `router-tools` worker. `provider` is a
+ * router search provider (`you` | `exa`). `maxHits` bounds the text folded back
+ * into the model's context per call.
+ */
+export const makeSearchExecutor = (cfg: {
+  routerBaseUrl: string
+  routerKey: string
+  provider: string
+  maxHits?: number
+}): ((name: string, args: Record<string, unknown>, task: unknown) => Promise<string>) => {
+  const maxHits = cfg.maxHits ?? 6
+  return async (name, args) => {
+    if (name !== webSearchTool.function.name) {
+      throw new Error(`search executor received unknown tool '${name}' (only '${webSearchTool.function.name}')`)
+    }
+    const query = typeof args.query === 'string' ? args.query.trim() : ''
+    if (!query) return 'web_search requires a non-empty "query" string.'
+
+    const res = await fetch(`${cfg.routerBaseUrl}/search`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` },
+      body: JSON.stringify({ provider: cfg.provider, query }),
+    })
+    if (!res.ok) {
+      // Fail loud: a search-backend fault must infra-exclude the cell, not silently
+      // become a recall-only (no-tool) answer that pollutes the capability test.
+      throw new Error(`web_search ${cfg.provider} HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+    }
+    const body = (await res.json()) as SearchResponse
+    const hits = body.data ?? []
+    // Tool use is a behavior worth seeing in the bench log — a research worker that
+    // never searches is the failure mode this tool exists to fix.
+    console.error(`  [web_search:${cfg.provider}] "${query.slice(0, 80)}" -> ${hits.length} hits`)
+    return formatHits(hits, maxHits)
+  }
+}

package/src/selector.test.mts

@@ -0,0 +1,189 @@
+import assert from 'node:assert/strict'
+import type { AttemptRecord, RunRecord } from './corpus'
+import {
+  bootstrapDeltaCI,
+  flipRate,
+  normalizeAnswer,
+  scoreSelectorOnRun,
+  scoreVerifierSelectorOnRun,
+  selfConsistencySelect,
+  summarizeSelector,
+  summarizeVerifierSelector,
+  verifierGroundedSelect,
+} from './selector'
+
+// --- verifierGroundedSelect: highest pass-count, ties → earliest, validation ---
+assert.equal(verifierGroundedSelect([0, 3, 1]), 1, 'highest pass-count wins')
+assert.equal(verifierGroundedSelect([2, 2, 1]), 0, 'tie → earliest')
+assert.equal(verifierGroundedSelect([0, 0, 0]), 0, 'all-zero → earliest')
+assert.equal(verifierGroundedSelect([1]), 0, 'single candidate')
+assert.equal(verifierGroundedSelect([false ? 1 : 0, 1]), 1, 'boolean pass/fail as {0,1}')
+assert.throws(() => verifierGroundedSelect([]), /no candidate pass-counts/)
+assert.throws(() => verifierGroundedSelect([0, -1]), /invalid pass-count/)
+assert.throws(() => verifierGroundedSelect([0, Number.NaN]), /invalid pass-count/)
+
+// --- normalizeAnswer ---
+assert.equal(normalizeAnswer('  The Paris.  '), 'paris')
+assert.equal(normalizeAnswer('"London"'), 'london')
+assert.equal(normalizeAnswer('a Cat'), 'cat')
+assert.equal(normalizeAnswer('New   York'), 'new york')
+
+// --- selfConsistencySelect: majority, ties, single ---
+assert.equal(selfConsistencySelect(['Paris', 'Paris', 'London']), 0, 'largest cluster')
+assert.equal(selfConsistencySelect(['London', 'Paris', 'Paris']), 1, 'first member of the largest cluster')
+assert.equal(selfConsistencySelect(['a', 'b']), 0, 'tie → earliest cluster')
+assert.equal(selfConsistencySelect(['only']), 0)
+assert.throws(() => selfConsistencySelect([]), /no candidate outputs/)
+
+// helper: build a RunRecord whose attempts carry (output, valid)
+const att = (output: string, valid: boolean): AttemptRecord => ({
+  round: 0,
+  prompt: '',
+  output,
+  valid,
+  score: valid ? 1 : 0,
+  costUsd: 0,
+  tokensIn: 0,
+  tokensOut: 0,
+  eventCount: 0,
+  eventTypes: {},
+})
+const rec = (atts: AttemptRecord[], blind = atts[0]?.valid === true): RunRecord => ({
+  ts: '',
+  benchmark: 'test',
+  instanceId: 'i',
+  condition: 'random@3',
+  model: 'm',
+  blindResolved: blind,
+  resolved: atts.some((a) => a.valid === true),
+  attempts: atts,
+  infraError: false,
+})
+
+// --- THE firewall regression ---
+// Majority answer is WRONG (valid=false), a minority answer is RIGHT (valid=true).
+// A non-oracle self-consistency selector must pick the wrong majority — proving it
+// ranked on OUTPUT TEXT only and never read `valid`. An oracle would pick the right
+// one; that this picks wrong is the honest, deployable behavior.
+{
+  const r = rec([att('Paris', false), att('Paris', false), att('London', true)])
+  const o = scoreSelectorOnRun(r, selfConsistencySelect)
+  assert.ok(o)
+  assert.equal(o.selectorResolved, false, 'selector picked the wrong majority — it cannot see valid')
+  assert.equal(o.oracle, true, 'an oracle COULD have recovered it (London was right)')
+  assert.ok(Math.abs(o.randomExpected - 1 / 3) < 1e-9, 'random@3 = 1/3 here')
+}
+
+// --- selector beats random when the majority correlates with correctness ---
+// Across instances where the right answer is usually the majority, selector@k > random@k.
+{
+  const records: RunRecord[] = [
+    rec([att('A', true), att('A', true), att('B', false)]), // majority right
+    rec([att('X', true), att('Y', false), att('X', true)]), // majority right
+    rec([att('Q', false), att('Q', false), att('R', true)]), // majority wrong (selector loses this one)
+  ]
+  const s = summarizeSelector(records, selfConsistencySelect)
+  assert.equal(s.n, 3)
+  // selector resolves 2/3 (first two), random expected = (2/3 + 2/3 + 1/3)/3 = 5/9
+  assert.ok(Math.abs(s.selectorRate - 2 / 3) < 1e-9)
+  assert.ok(Math.abs(s.randomRate - 5 / 9) < 1e-9)
+  assert.ok(s.dVsRandom > 0, 'self-consistency beats random when majority correlates with correctness')
+  assert.ok(s.selectorRate <= s.oracleRate + 1e-9, 'selector can never exceed the oracle ceiling')
+  assert.ok(Math.abs(s.gapToOracle - (s.oracleRate - s.selectorRate)) < 1e-9)
+}
+
+// --- unscoreable record (no outputs) is skipped, not counted as a pass ---
+{
+  const blank: RunRecord = {
+    ts: '', benchmark: 'test', instanceId: 'i', condition: 'random@3', model: 'm',
+    blindResolved: false, resolved: false, infraError: true,
+    attempts: [{ round: 0, prompt: '', costUsd: 0, tokensIn: 0, tokensOut: 0, eventCount: 0, eventTypes: {} }],
+  }
+  assert.equal(scoreSelectorOnRun(blank, selfConsistencySelect), null)
+  const s = summarizeSelector([blank], selfConsistencySelect)
+  assert.equal(s.n, 0)
+  assert.equal(s.skipped, 1)
+}
+
+// --- test-retest: a deterministic selector against itself never flips (the floor) ---
+{
+  const records = [rec([att('A', true), att('A', true), att('B', false)]), rec([att('X', true), att('Y', false)])]
+  assert.equal(flipRate(records, selfConsistencySelect, selfConsistencySelect), 0)
+}
+
+// --- verifier-grounded selection over a CONTINUOUS graded reward ---
+// An attempt carrying an arbitrary deployable-checker score (commit0 pass-rate /
+// aec verify.py partial credit). resolve = score >= 1 (full credit).
+const attS = (score: number, output = `out-${score}`): AttemptRecord => ({
+  round: 0,
+  prompt: '',
+  output,
+  valid: score >= 1,
+  score,
+  costUsd: 0,
+  tokensIn: 0,
+  tokensOut: 0,
+  eventCount: 0,
+  eventTypes: {},
+})
+const recS = (scores: number[]): RunRecord => ({
+  ts: '', benchmark: 'aec', instanceId: 'i', condition: 'random@4', model: 'm',
+  blindResolved: (scores[0] ?? 0) >= 1,
+  resolved: scores.some((s) => s >= 1),
+  attempts: scores.map((s, i) => ({ ...attS(s), round: i })),
+  infraError: false,
+})
+
+// the verifier picks argmax(score); the picked reward == the max (best-of-k)
+{
+  const o = scoreVerifierSelectorOnRun(recS([0.2, 0.9, 0.5, 0.1]))
+  assert.ok(o)
+  assert.ok(Math.abs(o.selectorReward - 0.9) < 1e-9, 'verifier picks the highest-scoring attempt')
+  assert.ok(Math.abs(o.oracleReward - 0.9) < 1e-9, 'oracle = max = selector on the continuous metric')
+  assert.ok(Math.abs(o.randomReward - 0.425) < 1e-9, 'random = mean of the four scores')
+  assert.ok(Math.abs(o.blindReward - 0.2) < 1e-9, 'blind = round-1 score')
+  assert.equal(o.selectorResolved, false, 'no attempt hit full credit here')
+}
+
+// the GATE on a band that lives entirely below full-credit (the aec failure mode):
+// binary resolve = 0 everywhere, yet verifier-select beats a blind draw on reward.
+{
+  const records = [recS([0.2, 0.8, 0.5, 0.3]), recS([0.1, 0.6, 0.9, 0.4]), recS([0.5, 0.5, 0.5, 0.5])]
+  const r = summarizeVerifierSelector(records)
+  assert.equal(r.n, 3)
+  assert.equal(r.skipped, 0)
+  assert.equal(r.selectorResolveRate, 0, 'nothing fully resolves — binary metric is dead here')
+  assert.equal(r.oracleResolveRate, 0)
+  // selector reward = mean(0.8,0.9,0.5)=0.7333; random = mean(0.45,0.5,0.5)=0.4833
+  assert.ok(Math.abs(r.selectorReward - 0.7333333) < 1e-5)
+  assert.ok(Math.abs(r.randomReward - 0.4833333) < 1e-5)
+  assert.ok(r.rewardVsRandom > 0, 'verifier-select beats a blind draw on the continuous reward')
+  assert.ok(Math.abs(r.selectorReward - r.oracleReward) < 1e-9, 'selector == oracle on continuous reward by construction')
+  // the third instance is concordant-zero (all scores equal → delta 0) → 2/3 discordant
+  assert.equal(r.ci.discordant, 2, 'flat-reward instances contribute a zero delta')
+  assert.ok(r.ci.lo <= r.rewardVsRandom && r.rewardVsRandom <= r.ci.hi, 'point estimate inside its CI')
+}
+
+// bootstrap is deterministic (same input → same CI) and signs correctly
+{
+  const a = bootstrapDeltaCI([0.1, 0.2, 0.3, 0.25, 0.15])
+  const b = bootstrapDeltaCI([0.1, 0.2, 0.3, 0.25, 0.15])
+  assert.deepEqual(a, b, 'deterministic seed → identical CI')
+  assert.ok(a.lo > 0, 'an all-positive delta vector is significantly > 0')
+  assert.equal(bootstrapDeltaCI([]).p, 1, 'empty → p=1')
+}
+
+// unscoreable (no checker score on any attempt) is skipped, not counted
+{
+  const noScore: RunRecord = {
+    ts: '', benchmark: 'aec', instanceId: 'i', condition: 'random@4', model: 'm',
+    blindResolved: false, resolved: false, infraError: false,
+    attempts: [{ round: 0, prompt: '', output: 'x', costUsd: 0, tokensIn: 0, tokensOut: 0, eventCount: 0, eventTypes: {} }],
+  }
+  assert.equal(scoreVerifierSelectorOnRun(noScore), null)
+  const r = summarizeVerifierSelector([noScore])
+  assert.equal(r.n, 0)
+  assert.equal(r.skipped, 1)
+}
+
+console.log('selector.test: all assertions passed')