@qvac/harness 1.0.0-rc.1 → 1.0.0-rc.3

package/bench/README.md

@@ -0,0 +1,94 @@
+# bench
+
+Runs the harness on published benchmarks with local models. `harness.ts` builds the tool list the
+sidecar builds, so a score is the product's. Results render on one page with `report.ts`.
+
+The other files here (`cpu.ts`, `graph-metrics.ts`, `issue129-matrix.mjs` and friends) are the
+completion engine's own performance runners and are unrelated. [`skill-eval/`](#skill-eval)
+measures a bundled skill's wording on the real model.
+
+## BFCL v4
+
+The eleven single-turn categories of the Berkeley Function Calling Leaderboard, downloaded from a
+pinned gorilla commit and judged by a port of its AST checker. Each case's functions are the run's
+only tools. `strict` scores the first round as BFCL does; `across rounds` also credits parallel
+calls made one round at a time.
+
+```
+bare bench/bfcl/run.ts --models Qwen3.5-2B-Q4_K_M,Qwen3.5-9B-Q4_K_M --limit 50 --out output/bfcl/run
+```
+
+Set `QVAC_BENCH_CODE_MODE=1` to give the model `run_code` beside the case's functions; a script's
+inner calls are the ones scored.
+
+## BFCL v4 multi-turn
+
+The four multi-turn categories, run on gorilla's own backends and scored by BFCL's checker; the report says how. Needs a Python 3.12 venv with `bfcl-eval` at the gorilla commit the single-turn adapter pins, without its deps.
+
+```
+python3.12 -m venv output/venvs/bfcl && output/venvs/bfcl/bin/pip install --no-deps \
+  "bfcl-eval @ git+https://github.com/ShishirPatil/gorilla@6ea57973c7a6097fd7c5915698c54c17c5b1b6c8#subdirectory=berkeley-function-call-leaderboard"
+output/venvs/bfcl/bin/pip install mpmath==1.3.0 filelock==3.20.0 overrides tenacity \
+  tree_sitter==0.21.3 tree-sitter-java==0.21.0 tree-sitter-javascript==0.21.4
+output/venvs/bfcl/bin/python bench/bridge/python/run_bfcl_multi_turn.py \
+  --models Qwen3.5-2B-Q4_K_M,Qwen3.5-9B-Q4_K_M --limit 10 --out output/bfcl-multi-turn/run
+```
+
+`--limit` is entries per category, spread evenly over its 200; `--out` must be a new dir. Tests,
+with no model: `cd bench/bridge/python && ../../../output/venvs/bfcl/bin/python -m unittest test_bfcl_multi_turn`.
+
+## tau2-bench
+
+The harness is the agent over the stdio bridge; tau2's orchestrator runs the tools and scores the
+task. The simulated user is a raw local model behind `bridge/openai.ts`, so scores are tau2's own
+but not comparable to its GPT-4.1-user leaderboard. Needs a Python 3.12 venv with tau2 installed.
+
+```
+python3.12 -m venv output/venvs/tau2 && output/venvs/tau2/bin/pip install -e <tau2-bench checkout> websockets
+output/venvs/tau2/bin/python bench/bridge/python/run_tau2.py --domain airline --tasks 20 \
+  --model Qwen3.5-4B-Q4_K_M --user-model Qwen3.5-9B-Q4_K_M --out output/tau2/run
+```
+
+Set `QVAC_BRIDGE_PLAN=1` to give the agent the `update_plan` tool, and compare against a run without it.
+
+## Terminal-Bench 2.0
+
+A Harbor agent runs the harness with one bash tool in the task container. Needs a container
+runtime: with OrbStack, give it 4 or more CPUs and 8 GB or more of memory before a full run. Run
+one task at a time (`-n 1`): every trial loads the model in its own bridge process.
+
+```
+QVAC_HARNESS_REPO=$PWD PYTHONPATH=bench/bridge/python uv tool run harbor run \
+  --dataset terminal-bench@2.0 -a qvac_bench.harbor_agent:QvacHarnessAgent \
+  -m Qwen3.5-9B-Q4_K_M -n 1 --ak max_tool_rounds=10
+```
+
+`max_tool_rounds` is the run's round limit; leave it out for the harness default. Each trial's
+`agent/qvac.jsonl` records the bash calls, the done frame and the load average at the start and end.
+
+## Skill eval
+
+Runs a `SKILL.md` variant against the current one on the real local model, with the skill's tools
+answered from recorded fixtures and every reply judged by fixed rules, never by a second model. A
+pack in `skill-eval/packs/` is one skill's prompts, fixtures and scoring; `weather` and `diagrams`
+ship. See [the decision](../docs/decisions/2026-09-10-skill-wording-is-measured-offline.md).
+
+```
+bun run skill-eval --pack weather --repeats 3
+bun run skill-eval --help
+```
+
+## Report
+
+```
+bare bench/report.ts \
+  --bfcl main=output/bfcl/run \
+  --bfcl-multi-turn main=output/bfcl-multi-turn/run \
+  --tau2 "before=output/tau2/old" --tau2 "after=output/tau2/run" \
+  --terminal "10 rounds=jobs/<job>" \
+  --out docs/bench/benchmarks.html
+```
+
+Each `--bfcl`, `--bfcl-multi-turn`, `--tau2` and `--terminal` run is `label=dir` (for `--terminal`, Harbor's job dir); the first BFCL run and the last τ² run are the ones the headline cards report, and τ² runs are listed in the order given.
+
+Outside a checkout with vendored wheels, set `QVAC_PYTHON_WHEELS_DIR` to a checkout's `wheels/`.

package/bench/bfcl/checker.ts

@@ -0,0 +1,313 @@
+import { isRecord } from '../../lib/guards.ts'
+
+// a port of BFCL's single-turn AST checker (bfcl_eval/eval_checker/ast_eval/ast_checker.py)
+export interface ParamDoc {
+  readonly type: string
+  readonly description?: string
+  readonly items?: ParamDoc
+  readonly properties?: Readonly<Record<string, ParamDoc>>
+  readonly enum?: readonly unknown[]
+  readonly required?: readonly string[]
+  readonly default?: unknown
+}
+
+export interface FunctionDoc {
+  readonly name: string
+  readonly description: string
+  readonly parameters: {
+    readonly type: string
+    readonly properties: Readonly<Record<string, ParamDoc>>
+    readonly required?: readonly string[]
+  }
+}
+
+// one call as BFCL writes it: { name: { param: value } }
+export type Call = Readonly<Record<string, Readonly<Record<string, unknown>>>>
+export type Answer = Readonly<Record<string, Readonly<Record<string, readonly unknown[]>>>>
+
+export interface Verdict {
+  readonly valid: boolean
+  readonly error: string | null
+}
+
+type Kind = 'string' | 'integer' | 'float' | 'boolean' | 'array' | 'dict' | 'null'
+
+const PYTHON_KIND: Record<string, Kind> = {
+  string: 'string',
+  integer: 'integer',
+  float: 'float',
+  boolean: 'boolean',
+  array: 'array',
+  tuple: 'array',
+  dict: 'dict',
+  any: 'string'
+}
+
+const NESTED = ['array', 'tuple']
+const VALID: Verdict = { valid: true, error: null }
+
+export interface Subject {
+  readonly category: string
+  readonly docs: readonly FunctionDoc[]
+  readonly answers: readonly Answer[] | null
+}
+
+interface Expectation {
+  readonly possible: readonly unknown[]
+  readonly expected: Kind
+  readonly nested: Kind | null
+}
+
+export function check(subject: Subject, calls: readonly Call[]): Verdict {
+  const { category, docs } = subject
+  if (category.includes('irrelevance')) {
+    return calls.length ? invalid('called a function when none applied') : VALID
+  }
+
+  if (category.includes('relevance')) {
+    return calls.length ? VALID : invalid('made no call')
+  }
+
+  const expected = subject.answers ?? []
+  if (calls.length !== expected.length) {
+    return invalid(`wrong number of functions: ${calls.length} for ${expected.length}`)
+  }
+
+  if (!category.includes('parallel')) {
+    return simple(docFor(docs, nameOf(expected[0])), calls[0], expected[0])
+  }
+
+  // unordered: each expected call claims the first unclaimed call that satisfies it
+  const matched = new Set<number>()
+  for (const answer of expected) {
+    const doc = docFor(docs, nameOf(answer))
+    const index = calls.findIndex((call, at) => !matched.has(at) && simple(doc, call, answer).valid)
+    if (index < 0) {
+      return invalid(`no call matches ${nameOf(answer)}`)
+    }
+
+    matched.add(index)
+  }
+
+  return VALID
+}
+
+function simple(doc: FunctionDoc | undefined, call: Call, answer: Answer): Verdict {
+  const name = nameOf(answer)
+  const possible = answer[name]
+  if (!doc || !(name in call)) {
+    return invalid(`function ${name} not called`)
+  }
+
+  const params = call[name]
+  const missing = (doc.parameters.required ?? []).find((param) => !(param in params))
+  if (missing) {
+    return invalid(`missing required parameter ${missing}`)
+  }
+
+  for (const [param, value] of Object.entries(params)) {
+    const detail = doc.parameters.properties[param]
+    if (!detail || !(param in possible)) {
+      return invalid(`unexpected parameter ${param}`)
+    }
+
+    const verdict = checkParam(param, value, expectationOf(detail, possible[param]))
+    if (!verdict.valid) {
+      return verdict
+    }
+  }
+
+  const unset = Object.keys(possible).find(
+    (param) => !(param in params) && !possible[param].includes('')
+  )
+
+  return unset ? invalid(`optional parameter ${unset} not provided`) : VALID
+}
+
+function expectationOf(detail: ParamDoc, possible: readonly unknown[]): Expectation {
+  const nested =
+    NESTED.includes(detail.type) && detail.items ? PYTHON_KIND[detail.items.type] : null
+  return { possible, expected: PYTHON_KIND[detail.type] ?? 'string', nested }
+}
+
+function checkParam(param: string, value: unknown, expectation: Expectation): Verdict {
+  const { possible, expected, nested } = expectation
+  const typed = typeCheck(value, expectation)
+  if (typed === null) {
+    return invalid(`wrong type for ${param}: ${JSON.stringify(value)} is not ${expected}`)
+  }
+
+  // a value of the answer's type where the schema wanted another is a variable: compare it plainly
+  const special = typed.variable ? null : specialCheck(value, expected, nested)
+  const matches = special ? special(possible) : possible.some((option) => equal(value, option))
+
+  return matches ? VALID : invalid(`invalid value for ${param}: ${JSON.stringify(value)}`)
+}
+
+function specialCheck(
+  value: unknown,
+  expected: Kind,
+  nested: Kind | null
+): ((possible: readonly unknown[]) => boolean) | null {
+  if (expected === 'dict') {
+    return (possible) => dictFits(value, possible)
+  }
+
+  if (expected === 'array' && nested === 'dict') {
+    return (possible) => listDictFits(value, possible)
+  }
+
+  if (expected === 'string') {
+    return (possible) => stringFits(value, possible)
+  }
+
+  if (expected === 'array') {
+    return (possible) => listFits(value, possible)
+  }
+
+  return null
+}
+
+interface Typed {
+  readonly variable: boolean
+}
+
+function typeCheck(value: unknown, expectation: Expectation): Typed | null {
+  const { possible, expected, nested } = expectation
+  const answerKind = possibleKind(possible)
+  const variable = answerKind !== null && !sameKind(answerKind, expected)
+  if (sameKind(kindOf(value), expected)) {
+    if (nested === null || !Array.isArray(value)) {
+      return { variable }
+    }
+
+    const itemsFit = possible.some(
+      (option) => !Array.isArray(option) || value.every((item) => sameKind(kindOf(item), nested))
+    )
+
+    return itemsFit ? { variable } : null
+  }
+
+  return answerKind !== null && sameKind(kindOf(value), answerKind) ? { variable: true } : null
+}
+
+function stringFits(value: unknown, possible: readonly unknown[]): boolean {
+  const wanted = possible.filter((option) => typeof option === 'string').map(standardize)
+  return wanted.includes(standardize(String(value)))
+}
+
+function listFits(value: unknown, possible: readonly unknown[]): boolean {
+  if (!Array.isArray(value)) {
+    return false
+  }
+
+  const output = value.map(standardizeItem)
+
+  return possible.filter(Array.isArray).some((option) => equal(output, option.map(standardizeItem)))
+}
+
+function dictFits(value: unknown, possible: readonly unknown[]): boolean {
+  return isDict(value) && possible.some((option) => isDict(option) && dictMatches(value, option))
+}
+
+function dictMatches(output: Record<string, unknown>, option: Record<string, unknown>): boolean {
+  const valuesFit = Object.entries(output).every(([key, item]) => {
+    const candidates = option[key]
+    return (
+      Array.isArray(candidates) &&
+      candidates.map(standardizeItem).some((candidate) => equal(standardizeItem(item), candidate))
+    )
+  })
+  const keysFit = Object.entries(option).every(
+    ([key, candidates]) => key in output || (Array.isArray(candidates) && candidates.includes(''))
+  )
+
+  return valuesFit && keysFit
+}
+
+function listDictFits(value: unknown, possible: readonly unknown[]): boolean {
+  if (!Array.isArray(value)) {
+    return false
+  }
+
+  return possible.some(
+    (option) =>
+      Array.isArray(option) &&
+      option.length === value.length &&
+      value.every((item, index) => dictFits(item, [option[index]]))
+  )
+}
+
+// BFCL forgives spacing and ",./-_*^" punctuation and case: "April 1, 2024" is "April 1 2024"
+export function standardize(text: string): string {
+  return text
+    .replace(/[ ,./\-_*^]/g, '')
+    .toLowerCase()
+    .replaceAll("'", '"')
+}
+
+function standardizeItem(item: unknown): unknown {
+  return typeof item === 'string' ? standardize(item) : item
+}
+
+// JSON keeps no int/float distinction in JavaScript, so 5 and 5.0 are one number here
+function kindOf(value: unknown): Kind {
+  if (value === null) {
+    return 'null'
+  }
+
+  if (Array.isArray(value)) {
+    return 'array'
+  }
+
+  if (typeof value === 'number') {
+    return Number.isInteger(value) ? 'integer' : 'float'
+  }
+
+  if (typeof value === 'boolean') {
+    return 'boolean'
+  }
+
+  return isRecord(value) ? 'dict' : 'string'
+}
+
+function sameKind(left: Kind, right: Kind): boolean {
+  const numeric = (kind: Kind) => kind === 'integer' || kind === 'float'
+  return left === right || (numeric(left) && numeric(right))
+}
+
+function possibleKind(possible: readonly unknown[]): Kind | null {
+  const first = possible.find((option) => option !== '')
+  return first === undefined ? null : kindOf(first)
+}
+
+function equal(left: unknown, right: unknown): boolean {
+  if (Array.isArray(left) && Array.isArray(right)) {
+    return left.length === right.length && left.every((item, index) => equal(item, right[index]))
+  }
+
+  if (isDict(left) && isDict(right)) {
+    const keys = Object.keys(left)
+    return (
+      keys.length === Object.keys(right).length && keys.every((key) => equal(left[key], right[key]))
+    )
+  }
+
+  return left === right
+}
+
+function isDict(value: unknown): value is Record<string, unknown> {
+  return isRecord(value) && !Array.isArray(value)
+}
+
+function nameOf(answer: Answer): string {
+  return Object.keys(answer)[0]
+}
+
+function docFor(docs: readonly FunctionDoc[], name: string): FunctionDoc | undefined {
+  return docs.find((doc) => doc.name === name)
+}
+
+function invalid(error: string): Verdict {
+  return { valid: false, error }
+}

package/bench/bfcl/data.ts

@@ -0,0 +1,93 @@
+import fs from 'fs'
+import path from 'path'
+import fetch from '#fetch'
+import type { Message } from '../../lib/contract/index.ts'
+import type { Answer, FunctionDoc } from './checker.ts'
+
+// the gorilla commit the data and the checker port were read from
+export const BFCL_COMMIT = '6ea57973c7a6097fd7c5915698c54c17c5b1b6c8'
+const BFCL_DATA = `https://raw.githubusercontent.com/ShishirPatil/gorilla/${BFCL_COMMIT}/berkeley-function-call-leaderboard/bfcl_eval/data`
+
+// the single-turn categories BFCL scores by AST; multi-turn, memory and web search need executors
+export const CATEGORIES = [
+  'simple_python',
+  'multiple',
+  'parallel',
+  'parallel_multiple',
+  'irrelevance',
+  'live_simple',
+  'live_multiple',
+  'live_parallel',
+  'live_parallel_multiple',
+  'live_irrelevance',
+  'live_relevance'
+]
+
+// the params the #157 stopgap rewrites, keyed as sdkProperty keys them
+export const SHAPES = ['nested', 'default-only', 'untouched'] as const
+export type Shape = (typeof SHAPES)[number]
+
+export function shapeOf(functions: readonly FunctionDoc[]): Shape {
+  const params = functions.flatMap((doc) => Object.values(doc.parameters.properties))
+  if (params.some((param) => param.properties || param.items)) {
+    return 'nested'
+  }
+
+  return params.some((param) => param.default !== undefined) ? 'default-only' : 'untouched'
+}
+
+export interface BfclCase {
+  readonly id: string
+  readonly category: string
+  readonly turns: Message[]
+  readonly functions: FunctionDoc[]
+  readonly answers: Answer[] | null
+}
+
+export interface LoadOptions {
+  readonly dir: string
+  // cases per category, spread evenly across it
+  readonly limit: number
+}
+
+export async function loadCategory(category: string, options: LoadOptions): Promise<BfclCase[]> {
+  const entries = await lines(`BFCL_v4_${category}.json`, options.dir)
+  // relevance and irrelevance are judged by whether a call was made, so they ship no answers
+  const answers = category.includes('relevance')
+    ? new Map()
+    : new Map(
+        (await lines(`possible_answer/BFCL_v4_${category}.json`, options.dir)).map((entry) => [
+          entry.id,
+          entry.ground_truth
+        ])
+      )
+  const all = entries.map((entry): BfclCase => ({
+    id: entry.id,
+    category,
+    turns: entry.question[0],
+    functions: entry.function,
+    answers: answers.get(entry.id) ?? null
+  }))
+  const step = Math.max(1, Math.floor(all.length / Math.min(options.limit, all.length)))
+
+  return all.filter((_, index) => index % step === 0).slice(0, options.limit)
+}
+
+async function lines(file: string, dir: string) {
+  const local = path.join(dir, file)
+  if (!fs.existsSync(local)) {
+    const response = await fetch(`${BFCL_DATA}/${file}`)
+    if (!response.ok) {
+      throw new Error(`BFCL ${file}: HTTP ${response.status}`)
+    }
+
+    fs.mkdirSync(path.dirname(local), { recursive: true })
+    fs.writeFileSync(local, await response.text())
+  }
+
+  return fs
+    .readFileSync(local, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line))
+}

package/bench/bfcl/observe.ts

@@ -0,0 +1,93 @@
+import type {
+  CompletionEvent,
+  LlmGenerationParams,
+  MetricsEvent
+} from '../../lib/contract/index.ts'
+import { errorMessage } from '../../lib/guards.ts'
+import type { Call } from './checker.ts'
+
+export interface Made {
+  readonly first: Call[]
+  readonly all: Call[]
+  readonly reply: string
+  readonly error: string | null
+  readonly metrics: MetricsEvent | null
+  readonly firstCallRound: number | null
+  // what the second round was sent, where a re-ask is forced
+  readonly reaskParams: LlmGenerationParams | null
+}
+
+export interface Observing {
+  readonly original: (name: string) => string
+  readonly stop: () => void
+  // distinct calls that end the run; null ends it with the first round instead
+  readonly enough: number | null
+}
+
+export async function observe(
+  events: AsyncIterable<CompletionEvent>,
+  observing: Observing
+): Promise<Made> {
+  const { original, stop, enough } = observing
+  const first: Call[] = []
+  const all: Call[] = []
+  let firstRound: number | undefined
+  let doneRound: number | undefined
+  let stopped = false
+  let reply = ''
+  let error: string | null = null
+  let metrics: MetricsEvent | null = null
+  let reaskParams: LlmGenerationParams | null = null
+  try {
+    for await (const event of events) {
+      if (event.type === 'metrics') {
+        metrics = event
+      }
+
+      // a stopped run is drained, not left, so its speed still arrives; the rest is not scored
+      if (stopped) {
+        continue
+      }
+
+      // stop only at a round boundary, so every call of the round that finished the work is seen
+      if (doneRound !== undefined && (event.round ?? doneRound) > doneRound) {
+        stopped = true
+        stop()
+        continue
+      }
+
+      // a script's own calls arrive as tool calls of the same round, so they are the ones scored
+      if (event.type === 'tool-call' && event.name !== 'run_code') {
+        firstRound ??= event.round
+        record(
+          { [original(event.name)]: event.args },
+          event.round === firstRound ? first : null,
+          all
+        )
+        doneRound ??= enough === null || all.length >= enough ? event.round : undefined
+      } else if (event.type === 'content' && firstRound === undefined) {
+        reply += event.text
+      } else if (event.type === 'error') {
+        error = event.message
+      } else if (event.type === 'aborted') {
+        error = 'aborted'
+      } else if (event.type === 'inference-call' && event.call.round === 1) {
+        reaskParams ??= event.call.generationParams ?? null
+      }
+    }
+  } catch (err) {
+    if (!stopped) {
+      error = errorMessage(err)
+    }
+  }
+
+  return { first, all, reply, error, metrics, firstCallRound: firstRound ?? null, reaskParams }
+}
+
+// a repeat of the same call is one call to BFCL, not a second
+function record(call: Call, first: Call[] | null, all: Call[]): void {
+  first?.push(call)
+  if (!all.some((seen) => JSON.stringify(seen) === JSON.stringify(call))) {
+    all.push(call)
+  }
+}