@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/bfcl/run.ts

@@ -0,0 +1,213 @@
+// BFCL v4 single-turn categories through Harness.completion, scored by the ported AST checker
+import fs from 'fs'
+import path from 'path'
+import process from 'process'
+import { fileURLToPath } from 'url'
+import { AbortController } from 'bare-abort-controller'
+import type { Harness } from '../../index.ts'
+import type { AgentConfig, LlmGenerationParams } from '../../lib/contract/index.ts'
+import { QWEN_35_4B } from '../../lib/models/model-catalog-constants.ts'
+import { ensureBundledSkillsDir } from '../../lib/tools/skills/bundled-dir.ts'
+import { openBenchHarness } from '../harness.ts'
+import { speedOf, type Speed } from '../speed.ts'
+import { check, type Call, type Verdict } from './checker.ts'
+import {
+  BFCL_COMMIT,
+  CATEGORIES,
+  loadCategory,
+  shapeOf,
+  type BfclCase,
+  type Shape
+} from './data.ts'
+import { observe } from './observe.ts'
+import { caseTools } from './tools.ts'
+
+const REPO_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const REASONING_MODES = ['disabled', 'enabled', 'inherit'] as const
+
+const USAGE = `usage: bare bench/bfcl/run.ts [options]
+  --categories <a,b>   default ${CATEGORIES.join(',')}
+  --models <a,b>       default ${QWEN_35_4B}
+  --limit <n>          cases per category, spread evenly across it; default all
+  --timeout <ms>       per case, default 120000
+  --reasoning <mode>   disabled|enabled|inherit, default inherit
+  --out <dir>          default output/bfcl/<stamp>
+`
+
+interface CaseRow {
+  readonly model: string
+  readonly category: string
+  readonly id: string
+  readonly shape: Shape
+  // BFCL's rule: every call in the model's first response
+  readonly valid: boolean
+  readonly error: string | null
+  // the harness asks for one action at a time, so parallel calls are credited across the run's rounds
+  readonly wholeValid: boolean
+  readonly wholeError: string | null
+  readonly calls: Call[]
+  readonly firstCallRound: number | null
+  readonly reaskParams: LlmGenerationParams | null
+  readonly reply: string
+  readonly wallMs: number
+  readonly speed: Speed | null
+}
+
+const argv = process.argv.slice(2)
+if (argv.includes('--help')) {
+  console.log(USAGE)
+  process.exit(0)
+}
+
+const flag = (name: string): string | undefined => {
+  const index = argv.indexOf(`--${name}`)
+  return index >= 0 ? argv[index + 1] : undefined
+}
+
+const categories = flag('categories')?.split(',') ?? CATEGORIES
+const models = flag('models')?.split(',') ?? [QWEN_35_4B]
+const limit = flag('limit') === undefined ? Infinity : Number(flag('limit'))
+const timeoutMs = Number(flag('timeout') ?? 120_000)
+const reasoning = REASONING_MODES.find((mode) => mode === flag('reasoning')) ?? 'inherit'
+const stamp = new Date().toISOString().replace(/[:.]/g, '-')
+const outDir = flag('out') ?? path.join(REPO_ROOT, 'output', 'bfcl', stamp)
+const dataDir = path.join(REPO_ROOT, 'output', 'bfcl-data', BFCL_COMMIT)
+
+fs.mkdirSync(outDir, { recursive: true })
+const say = (line: string) => process.stdout.write(`${line}\n`)
+// the SDK and addons print whole prompts through console; keep them out of the progress lines
+const log = fs.openSync(path.join(outDir, 'bench.log'), 'a')
+console.log =
+  console.info =
+  console.debug =
+  console.warn =
+  console.error =
+    (...args: unknown[]) => fs.writeSync(log, `${args.map(String).join(' ')}\n`)
+
+process.env.QVAC_NO_SKILL_BIN_PROBE = '1'
+const loaded = await Promise.all(
+  categories.map((category) => loadCategory(category, { dir: dataDir, limit }))
+)
+const cases = loaded.flat()
+const harness = await openBenchHarness({
+  root: path.join(REPO_ROOT, 'output', 'bench', 'sdk-root'),
+  skillsDir: await ensureBundledSkillsDir()
+})
+say(`${cases.length} BFCL cases × ${models.length} models → ${outDir}`)
+
+const rows: CaseRow[] = []
+for (const model of models) {
+  for (const bfclCase of cases) {
+    const row = await runCase(harness, bfclCase, model)
+    rows.push(row)
+    fs.appendFileSync(path.join(outDir, 'runs.jsonl'), `${JSON.stringify(row)}\n`)
+  }
+
+  const own = rows.filter((row) => row.model === model)
+  say(`${model}: strict ${rate(own, strictly)} · across rounds ${rate(own, wholly)}`)
+}
+
+fs.writeFileSync(path.join(outDir, 'summary.md'), summary(rows))
+say(`\n${summary(rows)}\nwritten to ${outDir}`)
+await harness.close()
+
+async function runCase(host: Harness, bfclCase: BfclCase, model: string): Promise<CaseRow> {
+  const { tools, original } = caseTools(bfclCase.functions)
+  const system = bfclCase.turns.filter((turn) => turn.role === 'system')
+  // only the case's functions: no product tool, no skill
+  const agent: AgentConfig = {
+    id: 'bfcl',
+    toolsScope: 'selected',
+    enabledTools: [
+      ...tools.map((tool) => tool.name),
+      ...(process.env.QVAC_BENCH_CODE_MODE === '1' ? ['run_code'] : [])
+    ],
+    skillsScope: 'selected',
+    enabledSkills: [],
+    bio: system.length ? system.map((turn) => turn.content).join('\n\n') : undefined,
+    reasoning
+  }
+  const controller = new AbortController()
+  const deadline = setTimeout(() => controller.abort(new Error('deadline')), timeoutMs)
+  const started = Date.now()
+  // only parallel work is credited past the first round, so other runs stop when the second begins
+  const across = bfclCase.category.includes('parallel')
+  const made = await observe(
+    host.completion({
+      model,
+      agent,
+      messages: bfclCase.turns.filter((turn) => turn.role !== 'system'),
+      tools,
+      cacheKey: `bfcl-${stamp}-${model}-${bfclCase.id}`,
+      captureInferenceCalls: true,
+      signal: controller.signal
+    }),
+    {
+      original,
+      stop: () => controller.abort(new Error('scored')),
+      // a parallel run is done once it has made every call BFCL expects; the rest after one round
+      enough: across ? (bfclCase.answers?.length ?? 0) : null
+    }
+  )
+  clearTimeout(deadline)
+
+  const subject = {
+    category: bfclCase.category,
+    docs: bfclCase.functions,
+    answers: bfclCase.answers
+  }
+  const score = (calls: Call[]): Verdict =>
+    made.error && !calls.length ? { valid: false, error: made.error } : check(subject, calls)
+  const strict = score(made.first)
+  // only parallel work can span rounds; a later call elsewhere is the model reacting to a stub result
+  const whole = across ? score(made.all) : strict
+
+  return {
+    model,
+    category: bfclCase.category,
+    id: bfclCase.id,
+    shape: shapeOf(bfclCase.functions),
+    valid: strict.valid,
+    error: strict.error,
+    wholeValid: whole.valid,
+    wholeError: whole.error,
+    calls: made.all,
+    firstCallRound: made.firstCallRound,
+    reaskParams: made.reaskParams,
+    reply: made.reply.slice(0, 500),
+    wallMs: Date.now() - started,
+    speed: speedOf(made.metrics)
+  }
+}
+
+function strictly(row: CaseRow): boolean {
+  return row.valid
+}
+
+function wholly(row: CaseRow): boolean {
+  return row.wholeValid
+}
+
+function rate(rows: readonly CaseRow[], passed: (row: CaseRow) => boolean): string {
+  const count = rows.filter(passed).length
+
+  return `${count}/${rows.length} (${Math.round((count / Math.max(1, rows.length)) * 100)}%)`
+}
+
+function summary(all: readonly CaseRow[]): string {
+  const names = [...new Set(all.map((row) => row.model))]
+  const header = names.map((model) => `${model} strict | ${model} across rounds`).join(' | ')
+  const lines = [
+    `| category | ${header} |`,
+    `| --- | ${names.map(() => '--- | ---').join(' | ')} |`
+  ]
+  for (const category of categories) {
+    const cells = names.map((model) => {
+      const own = all.filter((row) => row.model === model && row.category === category)
+      return `${rate(own, strictly)} | ${rate(own, wholly)}`
+    })
+    lines.push(`| ${category} | ${cells.join(' | ')} |`)
+  }
+
+  return lines.join('\n') + '\n'
+}

package/bench/bfcl/tools.ts

@@ -0,0 +1,72 @@
+import type {
+  ToolDefinition,
+  ToolParameterSchema,
+  ToolParameterType,
+  ToolParameters
+} from '../../lib/contract/index.ts'
+import type { FunctionDoc, ParamDoc } from './checker.ts'
+
+const TYPES: Record<string, ToolParameterType> = {
+  dict: 'object',
+  float: 'number',
+  tuple: 'array',
+  any: 'string',
+  integer: 'integer',
+  string: 'string',
+  boolean: 'boolean',
+  array: 'array'
+}
+
+export interface CaseTools {
+  readonly tools: ToolDefinition[]
+  // the name a model saw back to BFCL's, which may carry dots
+  original(name: string): string
+}
+
+// dots become underscores as BFCL's own function-calling handlers rename them for providers
+export function caseTools(docs: readonly FunctionDoc[]): CaseTools {
+  const names = new Map(docs.map((doc) => [toolName(doc.name), doc.name]))
+  return {
+    tools: docs.map((doc): ToolDefinition => ({
+      name: toolName(doc.name),
+      toolset: null,
+      description: doc.description,
+      parameters: parameters(doc.parameters),
+      // the answer is the call itself; an intent field would be a parameter BFCL never asked for
+      autoApprove: true,
+      effect: () => 'read',
+      execute: () => Promise.resolve({ ok: true })
+    })),
+    original: (name) => names.get(name) ?? name
+  }
+}
+
+function toolName(name: string): string {
+  return name.replace(/[^\w-]/g, '_')
+}
+
+function parameters(doc: FunctionDoc['parameters']): ToolParameters {
+  return {
+    type: 'object',
+    properties: Object.fromEntries(
+      Object.entries(doc.properties).map(([key, value]) => [key, schema(value)])
+    ),
+    required: [...(doc.required ?? [])]
+  }
+}
+
+function schema(doc: ParamDoc): ToolParameterSchema {
+  return {
+    type: TYPES[doc.type] ?? 'string',
+    description: doc.description,
+    enum: doc.enum?.map(String),
+    items: doc.items ? schema(doc.items) : undefined,
+    properties: doc.properties
+      ? Object.fromEntries(
+          Object.entries(doc.properties).map(([key, value]) => [key, schema(value)])
+        )
+      : undefined,
+    required: doc.required ? [...doc.required] : undefined,
+    default: doc.default
+  }
+}

package/bench/bridge/bridge.ts

@@ -0,0 +1,392 @@
+import { AbortController } from 'bare-abort-controller'
+import type { CompletionContext } from '../../index.ts'
+import type {
+  AgentConfig,
+  CompletionEvent,
+  Message,
+  MetricsEvent,
+  ToolDefinition,
+  ToolParameterSchema,
+  ToolParameters
+} from '../../lib/contract/index.ts'
+import { errorMessage } from '../../lib/guards.ts'
+import { Transcript } from './transcript.ts'
+
+// the native addon prints prompts to the same stdout, so a frame is a line that starts with this
+export const FRAME = '\u001eqvac '
+// a round that outlives this is a runaway generation; the benchmark sees it fail, as a user's cancel
+export const ROUND_DEADLINE_MS = 180_000
+
+// a tool the benchmark owns: the harness decides to call it, the benchmark's environment runs it
+export interface ToolSpec {
+  readonly name: string
+  readonly description: string
+  readonly parameters: JsonParameters
+  // the benchmark's own environment changes on this call, so the run asks before it goes out
+  readonly write?: boolean | undefined
+}
+
+interface JsonSchema extends Omit<ToolParameterSchema, 'type' | 'items' | 'properties'> {
+  readonly type?: ToolParameterSchema['type']
+  readonly $ref?: string
+  readonly anyOf?: readonly JsonSchema[]
+  readonly items?: JsonSchema
+  readonly properties?: Readonly<Record<string, JsonSchema>>
+}
+
+export interface JsonParameters extends Omit<ToolParameters, 'properties'> {
+  readonly $defs?: Readonly<Record<string, JsonSchema>>
+  readonly properties: Readonly<Record<string, JsonSchema>>
+}
+
+export interface RunFrame {
+  readonly type: 'run'
+  readonly id: string
+  readonly model: string
+  readonly system?: string | undefined
+  readonly messages: Message[]
+  readonly tools: ToolSpec[]
+  // epoch ms the run should take as now, for a benchmark whose world is not today
+  readonly now?: number | undefined
+  // the run's round limit; absent, the harness default
+  readonly maxToolRounds?: number | undefined
+  // one key per conversation, as a host sends it, so a later turn reuses the earlier prefill
+  readonly cacheKey?: string | undefined
+}
+
+export interface ResultFrame {
+  readonly type: 'result'
+  readonly callId: string
+  readonly result: unknown
+}
+
+// a benchmark that ends a task mid-turn abandons the run; this ends it and its waiting calls
+export interface CancelFrame {
+  readonly type: 'cancel'
+  readonly id: string
+}
+
+// the user's answer to an approve frame; anything but approved runs nothing
+export interface DecisionFrame {
+  readonly type: 'decision'
+  readonly callId: string
+  readonly approved: boolean
+}
+
+// a conversation is over, so its cached prefill can go
+export interface ForgetFrame {
+  readonly type: 'forget'
+  readonly cacheKey: string
+}
+
+export type Inbound = RunFrame | ResultFrame | CancelFrame | DecisionFrame | ForgetFrame
+
+interface ActiveRun {
+  readonly controller: AbortController
+  readonly calls: Set<string>
+  readonly approvals: Set<string>
+  readonly deadline: Restartable
+}
+
+export interface CallFrame {
+  readonly type: 'call'
+  readonly id: string
+  readonly callId: string
+  readonly name: string
+  readonly args: Record<string, unknown>
+}
+
+// a write waits on the benchmark's user, who sees the call and answers with a decision frame
+export interface ApproveFrame {
+  readonly type: 'approve'
+  readonly id: string
+  readonly callId: string
+  readonly name: string
+  readonly args: Record<string, unknown>
+}
+
+export interface DoneFrame {
+  readonly type: 'done'
+  readonly id: string
+  readonly reply: string
+  readonly calls: number
+  readonly error: string | null
+  // the run's speed, also when it was cancelled; null when no round finished
+  readonly metrics: MetricsEvent | null
+  // the run's turns as a host stores them, to carry the conversation into the next run
+  readonly history: Message[]
+}
+
+// sent once the model runtime is open, so a client never writes into a process still booting
+export interface ReadyFrame {
+  readonly type: 'ready'
+}
+
+export type Outbound = ReadyFrame | CallFrame | ApproveFrame | DoneFrame
+
+export type Complete = (context: CompletionContext) => AsyncIterable<CompletionEvent>
+
+export interface BridgeOptions {
+  readonly roundDeadlineMs?: number
+  // offer the agent update_plan, to score a run with the plan against one without
+  readonly plan?: boolean
+  // drops a finished conversation's cached prefill, on a forget frame
+  readonly forget?: (cacheKey: string) => Promise<unknown>
+}
+
+export class Bridge {
+  private readonly _complete: Complete
+  private readonly _send: (frame: Outbound) => void
+  private readonly _pending = new Map<string, (result: unknown) => void>()
+  private readonly _approvals = new Map<string, (approved: boolean) => void>()
+  private readonly _runs = new Map<string, ActiveRun>()
+  private readonly _roundDeadlineMs: number
+  private readonly _plan: boolean
+  private readonly _forget: (cacheKey: string) => Promise<unknown>
+  private _next = 0
+
+  constructor(complete: Complete, send: (frame: Outbound) => void, opts: BridgeOptions = {}) {
+    this._complete = complete
+    this._send = send
+    this._roundDeadlineMs = opts.roundDeadlineMs ?? ROUND_DEADLINE_MS
+    this._plan = opts.plan === true
+    this._forget = opts.forget ?? (() => Promise.resolve())
+  }
+
+  receive(frame: Inbound): Promise<void> {
+    if (frame.type === 'result') {
+      this._settle(frame.callId, frame.result)
+      return Promise.resolve()
+    }
+
+    if (frame.type === 'decision') {
+      this._decide(frame)
+      return Promise.resolve()
+    }
+
+    if (frame.type === 'cancel') {
+      this._cancel(frame.id)
+      return Promise.resolve()
+    }
+
+    if (frame.type === 'forget') {
+      return this._forget(frame.cacheKey).then(() => {})
+    }
+
+    return this._run(frame)
+  }
+
+  private async _run(frame: RunFrame): Promise<void> {
+    const run: ActiveRun = {
+      controller: new AbortController(),
+      calls: new Set<string>(),
+      approvals: new Set<string>(),
+      // a long turn of many short rounds is legitimate, so the clock restarts at each round
+      deadline: restartable(() => this._cancel(frame.id), this._roundDeadlineMs)
+    }
+    this._runs.set(frame.id, run)
+    const tools = frame.tools.map((spec) => this._remote(run, frame.id, spec))
+    // only the benchmark's tools: no product tool, no skill
+    const agent: AgentConfig = {
+      id: 'bridge',
+      toolsScope: 'selected',
+      enabledTools: tools.map((tool) => tool.name),
+      skillsScope: 'selected',
+      enabledSkills: [],
+      bio: frame.system,
+      plan: this._plan
+    }
+    let reply = ''
+    let calls = 0
+    let error: string | null = null
+    let metrics: MetricsEvent | null = null
+    const history = new Transcript()
+    const { deadline } = run
+    const now = frame.now
+    try {
+      const events = this._complete({
+        model: frame.model,
+        agent,
+        messages: frame.messages,
+        tools,
+        now: now === undefined ? undefined : () => now,
+        maxToolRounds: frame.maxToolRounds,
+        cacheKey: frame.cacheKey,
+        signal: run.controller.signal,
+        // the wait for the user is not the model's time, so the round clock stops until it answers
+        approve: (call) =>
+          new Promise<boolean>((resolve) => {
+            const callId = `${frame.id}:${this._next++}`
+            deadline.clear()
+            run.approvals.add(callId)
+            this._approvals.set(callId, (approved) => {
+              run.approvals.delete(callId)
+              deadline.restart()
+              resolve(approved)
+            })
+            this._send({
+              type: 'approve',
+              id: frame.id,
+              callId,
+              name: call.name,
+              args: call.arguments
+            })
+          })
+      })
+      for await (const event of events) {
+        history.add(event)
+        // the answer is what follows the last tool result; content before it is narration
+        if (event.type === 'tool-result' || event.type === 'round-reset') {
+          reply = ''
+          deadline.restart()
+        } else if (event.type === 'content') {
+          reply += event.text
+        } else if (event.type === 'tool-call') {
+          calls++
+        } else if (event.type === 'error') {
+          error = event.message
+        } else if (event.type === 'metrics') {
+          metrics = event
+        }
+      }
+    } catch (err) {
+      error = errorMessage(err)
+    } finally {
+      deadline.clear()
+      this._runs.delete(frame.id)
+    }
+
+    this._send({
+      type: 'done',
+      id: frame.id,
+      reply,
+      calls,
+      error,
+      metrics,
+      history: history.turns()
+    })
+  }
+
+  private _settle(callId: string, result: unknown): void {
+    this._pending.get(callId)?.(result)
+    this._pending.delete(callId)
+  }
+
+  private _decide({ callId, approved }: Omit<DecisionFrame, 'type'>): void {
+    this._approvals.get(callId)?.(approved)
+    this._approvals.delete(callId)
+  }
+
+  private _cancel(id: string): void {
+    const run = this._runs.get(id)
+    if (!run) {
+      return
+    }
+
+    run.controller.abort(new Error('cancelled'))
+    for (const callId of run.calls) {
+      this._settle(callId, { error: 'cancelled' })
+    }
+
+    for (const callId of run.approvals) {
+      this._decide({ callId, approved: false })
+    }
+  }
+
+  private _remote(run: ActiveRun, id: string, spec: ToolSpec): ToolDefinition {
+    return {
+      name: spec.name,
+      toolset: null,
+      description: spec.description,
+      parameters: typedParameters(spec.parameters),
+      // the benchmark owns the sandbox; only a write it names waits on its user
+      autoApprove: spec.write !== true,
+      effect: () => (spec.write === true ? 'write' : 'read'),
+      // the benchmark's environment runs the call, not the model, so the round clock waits for it
+      execute: (args) =>
+        new Promise((resolve) => {
+          const callId = `${id}:${this._next++}`
+          run.deadline.clear()
+          run.calls.add(callId)
+          this._pending.set(callId, (result) => {
+            run.calls.delete(callId)
+            // a result that comes back after its run ended, or was cancelled, starts no clock
+            if (
+              run.calls.size === 0 &&
+              this._runs.get(id) === run &&
+              !run.controller.signal.aborted
+            ) {
+              run.deadline.restart()
+            }
+
+            resolve(result)
+          })
+          this._send({ type: 'call', id, callId, name: spec.name, args })
+        })
+    }
+  }
+}
+
+// τ² sends pydantic schemas; inline $ref and anyOf so the harness can describe nested fields
+export function typedParameters({
+  $defs: defs = {},
+  properties,
+  ...rest
+}: JsonParameters): ToolParameters {
+  return { ...rest, properties: typedProperties(properties, defs) }
+}
+
+// A node with no type (pydantic's Any, or an anyOf of only null) is dropped: the harness needs one.
+function typed(
+  schema: JsonSchema,
+  defs: Readonly<Record<string, JsonSchema>>
+): ToolParameterSchema | undefined {
+  const { $ref: ref, anyOf, items, properties, ...rest } = schema
+  const base = ref === undefined ? undefined : defs[ref.slice(ref.lastIndexOf('/') + 1)]
+  const branch = anyOf?.find((option) => option.type !== 'null')
+  if (base ?? branch) {
+    return typed({ ...(base ?? branch), ...rest }, defs)
+  }
+
+  if (rest.type === undefined) {
+    return undefined
+  }
+
+  return {
+    ...rest,
+    type: rest.type,
+    items: items && typed(items, defs),
+    properties: properties && typedProperties(properties, defs)
+  }
+}
+
+function typedProperties(
+  properties: Readonly<Record<string, JsonSchema>>,
+  defs: Readonly<Record<string, JsonSchema>>
+): Record<string, ToolParameterSchema> {
+  return Object.fromEntries(
+    Object.entries(properties).flatMap(([key, value]) => {
+      const schema = typed(value, defs)
+      return schema ? [[key, schema]] : []
+    })
+  )
+}
+
+interface Restartable {
+  restart(): void
+  clear(): void
+}
+
+function restartable(fire: () => void, ms: number): Restartable {
+  let timer = setTimeout(fire, ms)
+
+  return {
+    restart() {
+      clearTimeout(timer)
+      timer = setTimeout(fire, ms)
+    },
+    clear() {
+      clearTimeout(timer)
+    }
+  }
+}