@qvac/harness 1.0.0-rc.1 → 1.0.0-rc.3

package/bench/bridge/python/test_bfcl_multi_turn.py

@@ -0,0 +1,185 @@
+"""output/venvs/bfcl/bin/python -m unittest test_bfcl_multi_turn, from this folder: no model is loaded.
+
+A scripted model plays BFCL's own ground truth through the adapter, so BFCL's checker must pass it;
+the same script with a turn dropped or an argument changed must fail as BFCL fails it.
+"""
+
+import ast
+import asyncio
+import itertools
+import unittest
+
+import json
+
+from bfcl_eval.constants.default_prompts import DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC, MAXIMUM_STEP_LIMIT
+from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
+
+from qvac_bench.bfcl_multi_turn import CATEGORIES, Episode, executed, load
+
+names = (f"test_{index}" for index in itertools.count())
+
+
+def parse(source: str, tools: list[dict]) -> tuple[str, dict]:
+    call = ast.parse(source, mode="eval").body
+    name = call.func.id
+    params = next(tool for tool in tools if tool["name"] == name)["parameters"]["properties"]
+    args = dict(zip(params, (ast.literal_eval(arg) for arg in call.args)))
+    args.update({keyword.arg: ast.literal_eval(keyword.value) for keyword in call.keywords})
+    return name, args
+
+
+def scripted(answer: list[list[str]], change=lambda turn, name, args: args, seen=None, refused=None):
+    """A run that makes one turn's ground-truth calls, one round each, and returns the bridge's done frame."""
+    turns = iter(answer)
+
+    async def run(messages, tools, execute):
+        if seen is not None:
+            seen.append((list(messages), [tool["name"] for tool in tools]))
+        turn = next(turns)
+        history = []
+        if refused:
+            history.append({"role": "assistant", "content": "", "toolCalls": [refused]})
+        for index, source in enumerate(turn):
+            name, args = parse(source, tools)
+            args = change(turn, name, args)
+            call_id = f"c{index}"
+            result = await execute(name, args)
+            history.append({"role": "assistant", "content": "", "toolCalls": [{"id": call_id, "name": name, "arguments": args}]})
+            history.append({"role": "tool", "toolCallId": call_id, "name": name, "content": result})
+        history.append({"role": "assistant", "content": "Done."})
+        return {"type": "done", "reply": "Done.", "calls": len(turn), "error": None, "metrics": None, "history": history}
+
+    return run
+
+
+def play(entry, answer, run):
+    return asyncio.run(Episode(entry, answer, next(names)).play(run))
+
+
+def stepping(steps: int):
+    """A run that makes the same call for `steps` rounds every turn."""
+
+    async def run(messages, tools, execute):
+        history = []
+        for _ in range(steps):
+            await execute("pwd", {})
+            history.append({"role": "assistant", "content": "", "toolCalls": [{"id": "c", "name": "pwd", "arguments": {}}]})
+        return {"reply": "", "error": None, "metrics": None, "history": history}
+
+    return run
+
+
+class GroundTruth(unittest.TestCase):
+    def test_bfcl_passes_its_own_answers_in_every_category(self):
+        for category in CATEGORIES:
+            for entry, answer in load(category, 3):
+                with self.subTest(entry=entry["id"]):
+                    self.assertEqual(play(entry, answer, scripted(answer))["valid"], True)
+
+    def test_a_missed_function_is_offered_only_from_its_turn(self):
+        entry, answer = load("multi_turn_miss_func", 1)[0]
+        [(turn, [missed])] = entry["missed_function"].items()
+        seen = []
+        play(entry, answer, scripted(answer, seen=seen))
+
+        self.assertNotIn(missed["name"], seen[int(turn) - 1][1])
+        self.assertIn(missed["name"], seen[int(turn)][1])
+        self.assertEqual(seen[int(turn)][0][-1], {"role": "user", "content": DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC})
+
+    def test_each_turn_sees_the_conversation_so_far(self):
+        entry, answer = load("multi_turn_base", 1)[0]
+        seen = []
+        play(entry, answer, scripted(answer, seen=seen))
+
+        second = seen[1][0]
+        self.assertEqual(second[0], entry["question"][0][0])
+        self.assertEqual(second[-1], entry["question"][1][0])
+        self.assertTrue(any(message.get("toolCalls") for message in second))
+
+
+class Failures(unittest.TestCase):
+    def setUp(self):
+        self.entry, self.answer = load("multi_turn_base", 1)[0]
+
+    def test_a_turn_without_its_calls_fails_as_an_empty_turn(self):
+        dropped = [self.answer[0], []] + self.answer[2:]
+        verdict = play(self.entry, self.answer, scripted(dropped))
+
+        self.assertEqual(verdict["error_type"], "multi_turn:empty_turn_model_response")
+
+    def test_a_wrong_argument_fails_on_the_backend_state(self):
+        def rename(turn, name, args):
+            return {**args, "dir_name": "elsewhere"} if name == "mkdir" else args
+
+        verdict = play(self.entry, self.answer, scripted(self.answer, rename))
+
+        self.assertEqual(verdict["error_type"], "multi_turn:instance_state_mismatch")
+
+    def test_a_harness_error_fails_the_entry_as_bfcl_fails_a_raised_inference(self):
+        async def broken(messages, tools, execute):
+            return {"reply": "", "error": "context overflow", "metrics": None, "history": []}
+
+        verdict = play(self.entry, self.answer, broken)
+
+        self.assertEqual((verdict["error_type"], verdict["error"]), ("multi_turn:inference_error", "context overflow"))
+
+    def test_a_turn_past_the_step_limit_is_force_terminated(self):
+        verdict = play(self.entry, self.answer, stepping(MAXIMUM_STEP_LIMIT + 1))
+
+        self.assertEqual(verdict["error_type"], "multi_turn:force_terminated")
+
+    def test_a_turn_at_the_step_limit_is_scored(self):
+        verdict = play(self.entry, self.answer, stepping(MAXIMUM_STEP_LIMIT))
+
+        self.assertNotEqual(verdict["error_type"], "multi_turn:force_terminated")
+
+    def test_a_call_the_harness_refused_is_not_scored(self):
+        cut_off = {"id": "x", "name": "mkdir", "arguments": {"dir_name": "te"}}
+
+        self.assertTrue(play(self.entry, self.answer, scripted(self.answer, refused=cut_off))["valid"])
+
+
+    def test_refused_calls_leave_the_rounds_that_ran_as_steps(self):
+        def turn(*calls):
+            return {"role": "assistant", "content": "", "toolCalls": [{"id": "x", "name": name, "arguments": args} for name, args in calls]}
+
+        history = [
+            turn(("cd", {"folder": "do"}), ("ls", {"a": "tr"})),
+            turn(("cd", {"folder": "document"})),
+            turn(("ls", {"a": True})),
+        ]
+        ran = [{"cd": {"folder": "document"}}, {"ls": {"a": True}}]
+
+        self.assertEqual(executed(history, ran), [[ran[0]], [ran[1]]])
+
+
+class Execution(unittest.TestCase):
+    def test_a_call_returns_what_bfcl_sends_the_model(self):
+        entry, answer = load("multi_turn_base", 1)[0]
+        own, _ = execute_multi_turn_func_call(
+            ["cd(folder='document')"], entry["initial_config"], entry["involved_classes"], next(names), entry["id"]
+        )
+        result = asyncio.run(Episode(entry, answer, next(names)).execute("cd", {"folder": "document"}))
+
+        self.assertEqual(result, json.loads(own[0]), "a JSON result goes out as the object, so it is encoded once")
+
+    def test_a_text_result_stays_text(self):
+        entry, answer = load("multi_turn_base", 1)[0]
+        result = asyncio.run(Episode(entry, answer, next(names)).execute("cd", {"path": "document"}))
+
+        self.assertTrue(result.startswith("Error during execution: "), result)
+
+    def test_a_long_context_entry_runs_the_long_backend(self):
+        def listing(category):
+            entry, answer = load(category, 1)[0]
+            episode = Episode(entry, answer, next(names))
+            asyncio.run(episode.execute("cd", {"folder": "document"}))
+            return asyncio.run(episode.execute("ls", {}))
+
+        base, long = listing("multi_turn_base"), listing("multi_turn_long_context")
+
+        self.assertGreater(len(long["current_directory_content"]), len(base["current_directory_content"]))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/bench/bridge/python/test_harbor_agent.py

@@ -0,0 +1,107 @@
+"""python3 -m unittest test_harbor_agent, from this folder: harbor is not needed."""
+
+import asyncio
+import json
+import os
+import sys
+import tempfile
+import types
+import unittest
+from unittest import mock
+
+
+class BaseAgent:
+    def __init__(self, logs_dir, model_name=None, **kwargs):
+        self.logs_dir = logs_dir
+        self.model_name = model_name
+
+
+for name, attrs in {
+    "harbor": {},
+    "harbor.agents": {},
+    "harbor.agents.base": {"BaseAgent": BaseAgent},
+    "harbor.environments": {},
+    "harbor.environments.base": {"BaseEnvironment": object},
+    "harbor.models": {},
+    "harbor.models.agent": {},
+    "harbor.models.agent.context": {"AgentContext": object},
+}.items():
+    sys.modules.setdefault(name, types.ModuleType(name)).__dict__.update(attrs)
+
+from qvac_bench import harbor_agent  # noqa: E402
+
+DONE = {
+    "type": "done",
+    "id": "run-0",
+    "reply": "Done.",
+    "calls": 1,
+    "error": None,
+    "metrics": {"rounds": [{"promptTokens": 300, "generatedTokens": 20}, {"promptTokens": 50, "generatedTokens": 5}]},
+}
+
+
+class FakeBridge:
+    runs = []
+    results = []
+
+    def __init__(self, repo, model):
+        pass
+
+    async def start(self):
+        pass
+
+    async def run(self, messages, tools, execute, system=None, max_tool_rounds=None):
+        FakeBridge.runs.append(max_tool_rounds)
+        FakeBridge.results.append(await execute("bash", {"command": "ls"}))
+        return DONE
+
+    async def close(self):
+        pass
+
+
+class Environment:
+    async def exec(self, command, timeout_sec=None):
+        return types.SimpleNamespace(stdout="a b", stderr="", return_code=0)
+
+
+class TimeoutEnvironment:
+    async def exec(self, command, timeout_sec=None):
+        raise RuntimeError(f"Command timed out after {timeout_sec} seconds")
+
+
+def run_agent(environment, max_tool_rounds="40"):
+    with tempfile.TemporaryDirectory() as logs, mock.patch.object(harbor_agent, "QvacBridge", FakeBridge), \
+            mock.patch.dict(os.environ, {"QVAC_HARNESS_REPO": "/repo"}):
+        agent = harbor_agent.QvacHarnessAgent(logs_dir=logs, model_name="m", max_tool_rounds=max_tool_rounds)
+        context = types.SimpleNamespace()
+        asyncio.run(agent.run("list the files", environment, context))
+        with open(os.path.join(logs, "qvac.jsonl")) as log:
+            return [json.loads(line) for line in log], context
+
+
+class Agent(unittest.TestCase):
+    def setUp(self):
+        FakeBridge.runs = []
+        FakeBridge.results = []
+
+    def test_the_round_limit_reaches_the_run_and_the_log_records_it(self):
+        rows, context = run_agent(Environment())
+
+        self.assertEqual(FakeBridge.runs, [40])
+        self.assertEqual([row["type"] for row in rows], ["start", "call", "done", "end"])
+        self.assertEqual(rows[0]["max_tool_rounds"], 40)
+        self.assertEqual(len(rows[0]["load"]), 3)
+        self.assertEqual((rows[1]["command"], rows[1]["return_code"]), ("ls", 0))
+        self.assertEqual(rows[2], DONE)
+        self.assertEqual((context.n_input_tokens, context.n_output_tokens), (350, 25))
+
+    def test_a_command_timeout_returns_a_failed_result_and_the_trial_goes_on(self):
+        rows, _ = run_agent(TimeoutEnvironment())
+
+        self.assertEqual(FakeBridge.results, [{"stdout": "", "stderr": "Command timed out after 180 seconds", "return_code": 124}])
+        self.assertEqual([row["type"] for row in rows], ["start", "call", "done", "end"])
+        self.assertEqual((rows[1]["command"], rows[1]["return_code"]), ("ls", 124))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/bench/bridge/serve.ts

@@ -0,0 +1,52 @@
+// a benchmark in another process drives the harness over stdio: bare bench/bridge/serve.ts
+import fs from 'fs'
+import path from 'path'
+import process from 'process'
+import { fileURLToPath } from 'url'
+import { ensureBundledSkillsDir } from '../../lib/tools/skills/bundled-dir.ts'
+import { openBenchHarness } from '../harness.ts'
+import { Bridge, FRAME, type Inbound, type Outbound } from './bridge.ts'
+
+const REPO_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const logFile = process.env.QVAC_BRIDGE_LOG ?? path.join(REPO_ROOT, 'output', 'bridge.log')
+
+fs.mkdirSync(path.dirname(logFile), { recursive: true })
+const log = fs.openSync(logFile, 'a')
+// stdout carries the frames; everything the SDK says goes to the log
+console.log =
+  console.info =
+  console.debug =
+  console.warn =
+  console.error =
+    (...args: unknown[]) => fs.writeSync(log, `${args.map(String).join(' ')}\n`)
+
+process.env.QVAC_NO_SKILL_BIN_PROBE = '1'
+const harness = await openBenchHarness({
+  root: path.join(REPO_ROOT, 'output', 'bench', 'sdk-root'),
+  skillsDir: await ensureBundledSkillsDir()
+})
+const send = (frame: Outbound) => process.stdout.write(`${FRAME}${JSON.stringify(frame)}\n`)
+const bridge = new Bridge((context) => harness.completion(context), send, {
+  plan: process.env.QVAC_BRIDGE_PLAN === '1',
+  forget: (cacheKey) => harness.deleteKvCache(cacheKey)
+})
+
+let buffered = ''
+process.stdin.on('data', (chunk: Uint8Array) => {
+  buffered += chunk.toString()
+  for (let newline = buffered.indexOf('\n'); newline >= 0; newline = buffered.indexOf('\n')) {
+    const line = buffered.slice(0, newline).trim()
+    buffered = buffered.slice(newline + 1)
+    if (line) {
+      receive(JSON.parse(line))
+    }
+  }
+})
+process.stdin.on('end', () => {
+  harness.close().then(() => process.exit(0))
+})
+send({ type: 'ready' })
+
+function receive(frame: Inbound): void {
+  bridge.receive(frame).catch((err: unknown) => console.error('[bridge]', err))
+}

package/bench/bridge/transcript.ts

@@ -0,0 +1,52 @@
+import type { CompletionEvent, Message } from '../../lib/contract/index.ts'
+
+// A run's turns as a host replays them: one assistant turn per round, then a tool turn per call.
+export class Transcript {
+  private readonly _turns: Message[] = []
+  private readonly _rounds = new Map<number, Message>()
+
+  add(event: CompletionEvent): void {
+    const round = event.round ?? 0
+    if (event.type === 'content') {
+      this._answer(round).content += event.text
+    } else if (event.type === 'tool-call') {
+      const call = { id: event.callId ?? '', name: event.name, arguments: event.args }
+      const turn = this._round(round)
+      turn.toolCalls = [...(turn.toolCalls ?? []), call]
+    } else if (event.type === 'tool-result') {
+      const content = JSON.stringify(event.result) ?? ''
+      this._turns.push({ role: 'tool', toolCallId: event.callId ?? '', name: event.name, content })
+    } else if (event.type === 'round-reset') {
+      this._round(round).content = ''
+    }
+  }
+
+  turns(): Message[] {
+    return this._turns.filter(
+      (turn) => turn.role !== 'assistant' || turn.content || turn.toolCalls?.length
+    )
+  }
+
+  // a wrap-up streams under its last tool round, but it answers that round's results
+  private _answer(round: number): Message {
+    const turn = this._round(round)
+    if (turn === this._turns.at(-1)) {
+      return turn
+    }
+
+    this._rounds.delete(round)
+
+    return this._round(round)
+  }
+
+  private _round(round: number): Message {
+    let turn = this._rounds.get(round)
+    if (!turn) {
+      turn = { role: 'assistant', content: '' }
+      this._rounds.set(round, turn)
+      this._turns.push(turn)
+    }
+
+    return turn
+  }
+}

package/bench/bun-tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "erasableSyntaxOnly": true,
+    "verbatimModuleSyntax": true,
+    "allowImportingTsExtensions": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "allowJs": true,
+    "checkJs": false,
+    "noEmit": true
+  }
+}

package/bench/cpu-main.mjs

@@ -0,0 +1,216 @@
+// Pre-state-machine CPU baseline runner.
+import process from 'process'
+import { CompletionEngine } from '../lib/capabilities/completion-engine/index.ts'
+
+const TOKENS = 2000
+const FIRST_TOKEN_MS = 110
+
+function options() {
+  const args = process.argv.slice(2)
+  let runs = 3
+  let spacing = 'all'
+  let workload = 'all'
+  let json = false
+  let positionalRuns = false
+  for (const arg of args) {
+    if (arg === '--json') {
+      json = true
+    } else if (arg.startsWith('--spacing=')) {
+      spacing = arg.slice('--spacing='.length)
+    } else if (arg.startsWith('--workload=')) {
+      workload = arg.slice('--workload='.length)
+    } else if (!arg.startsWith('-') && !positionalRuns) {
+      runs = Number(arg)
+      positionalRuns = true
+    } else {
+      throw new Error(`Unknown option: ${arg}`)
+    }
+  }
+
+  if (!Number.isInteger(runs) || runs < 1) {
+    throw new Error('runs must be a positive integer')
+  }
+
+  const spacings = spacing === 'all' ? ['immediate', 'timeout'] : [spacing]
+  const workloads = workload === 'all' ? ['tools', 'stream'] : [workload]
+  if (spacings.some((value) => !['immediate', 'timeout'].includes(value))) {
+    throw new Error(`Unknown spacing: ${spacing}`)
+  }
+
+  if (workloads.some((value) => !['tools', 'stream'].includes(value))) {
+    throw new Error(`Unknown workload: ${workload}`)
+  }
+
+  return { runs, spacings, workloads, json }
+}
+
+async function collect(run) {
+  const events = []
+  for await (const event of run) {
+    events.push(event)
+  }
+
+  return events
+}
+
+function cpuMs(start) {
+  const usage = process.cpuUsage(start)
+  return (usage.user + usage.system) / 1000
+}
+
+function median(values) {
+  const sorted = [...values].sort((a, b) => a - b)
+  return sorted[Math.floor(sorted.length / 2)]
+}
+
+function history() {
+  return Array.from({ length: 101 }, (_, i) => ({
+    role: i % 2 ? 'assistant' : 'user',
+    content: `${i} ${'x'.repeat(2000)}`
+  }))
+}
+
+function readTool(name) {
+  return {
+    name,
+    description: 'read something',
+    toolset: null,
+    parameters: { type: 'object', properties: { path: { type: 'string' } } },
+    effect: () => 'read',
+    autoApprove: true,
+    execute: () => Promise.resolve({ content: `${name} ${'r'.repeat(2048)}` })
+  }
+}
+
+function toolWorkloadSdk(tools) {
+  let round = 0
+  return {
+    completion: () => {
+      const at = round++
+      if (at < 3) {
+        return {
+          tokenStream: (async function* () {})(),
+          final: Promise.resolve({
+            toolCalls: tools.map((tool, i) => ({
+              id: `call-${at}-${i}`,
+              name: tool.name,
+              arguments: { path: `f${at}${i}` }
+            })),
+            raw: { fullText: '' }
+          })
+        }
+      }
+
+      return {
+        tokenStream: (async function* () {
+          yield 'done'
+        })(),
+        final: Promise.resolve({ toolCalls: [], raw: { fullText: 'done' } })
+      }
+    }
+  }
+}
+
+function streamingSdk(spacing) {
+  return {
+    completion: () => ({
+      tokenStream: (async function* () {
+        await new Promise((resolve) => setTimeout(resolve, FIRST_TOKEN_MS))
+        for (let i = 0; i < TOKENS; i++) {
+          await new Promise((resolve) =>
+            spacing === 'immediate' ? setImmediate(resolve) : setTimeout(resolve, 0)
+          )
+          yield 't'
+        }
+      })(),
+      final: Promise.resolve({ toolCalls: [], raw: { fullText: 't'.repeat(TOKENS) } })
+    })
+  }
+}
+
+async function measureTools(runs, tools) {
+  // Match the engine benchmark's untimed pre-sample pass for each selected cell.
+  await collect(
+    CompletionEngine.run(toolWorkloadSdk(tools), {
+      modelId: 'm',
+      messages: history(),
+      tools
+    })
+  )
+
+  const samples = []
+  for (let i = 0; i < runs; i++) {
+    const args = { modelId: 'm', messages: history(), tools }
+    const start = process.cpuUsage()
+    await collect(CompletionEngine.run(toolWorkloadSdk(tools), args))
+    samples.push(cpuMs(start))
+  }
+
+  return {
+    workload: 'tools',
+    mode: 'main',
+    runs,
+    cpuMs: median(samples),
+    cpuSamplesMs: samples
+  }
+}
+
+async function measureStream(runs, spacing) {
+  // Run one untimed stream before collecting the timed samples.
+  await collect(CompletionEngine.run(streamingSdk(spacing), { modelId: 'm', messages: history() }))
+
+  const samples = []
+  let contentEvents = 0
+  for (let i = 0; i < runs; i++) {
+    const args = { modelId: 'm', messages: history() }
+    const start = process.cpuUsage()
+    const events = await collect(CompletionEngine.run(streamingSdk(spacing), args))
+    samples.push(cpuMs(start))
+    contentEvents = events.filter((event) => event.type === 'content').length
+  }
+
+  const cpu = median(samples)
+
+  return {
+    workload: 'stream',
+    mode: 'main',
+    spacing,
+    runs,
+    cpuMs: cpu,
+    cpuSamplesMs: samples,
+    usPerToken: (cpu * 1000) / TOKENS,
+    contentEvents
+  }
+}
+
+async function main() {
+  const { runs, spacings, workloads, json } = options()
+  const results = []
+  const tools = [readTool('read_a'), readTool('read_b')]
+  if (workloads.includes('tools')) {
+    results.push(await measureTools(runs, tools))
+  }
+
+  if (workloads.includes('stream')) {
+    for (const spacing of spacings) {
+      results.push(await measureStream(runs, spacing))
+    }
+  }
+
+  const report = {
+    runtime: globalThis.Bun ? 'bun' : (process.release?.name ?? 'unknown'),
+    version: globalThis.Bun?.version ?? process.version,
+    runs,
+    results
+  }
+  if (json) {
+    console.log(JSON.stringify(report))
+  } else {
+    console.log(JSON.stringify(report, null, 2))
+  }
+}
+
+main().catch((error) => {
+  console.error(error)
+  process.exitCode = 1
+})