@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/bridge/openai.ts

@@ -0,0 +1,150 @@
+// a raw local model behind an OpenAI-style endpoint, for a benchmark's own calls such as its user
+import { createHash } from 'crypto'
+import http from 'bare-http1'
+import fs from 'fs'
+import path from 'path'
+import process from 'process'
+import { fileURLToPath } from 'url'
+import { errorMessage } from '../../lib/guards.ts'
+import { resolveLlmGenerationParams } from '../../lib/models/llm-presets.ts'
+import type { ModelLifecycleSdk } from '../../lib/models/model-pool.ts'
+import { resolveModelSrc } from '../../lib/models/model-source.ts'
+import { createSdkLoader } from '../../lib/sdk-loader.ts'
+import { configureSdkStorage } from '../../lib/sdk-storage.ts'
+import { MODEL_CACHE_PATH_ENV, resolveStorage } from '../../lib/storage.ts'
+
+const REPO_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const port = Number(process.env.QVAC_OPENAI_PORT ?? 8765)
+const CTX_SIZE = 16384
+const REASONING_OFF = false
+const STARTED = Date.now()
+// τ² ends a task only on its exact ###STOP### tokens; a small customer model drops the hashes
+const END_TOKENS = process.env.QVAC_OPENAI_END_TOKENS === '1'
+const END_WORDS = new Set(['STOP', 'TRANSFER', 'OUT-OF-SCOPE'])
+
+interface ChatMessage {
+  readonly role: string
+  readonly content?: unknown
+}
+
+interface ChatRequest {
+  readonly model: string
+  readonly messages: ReadonlyArray<ChatMessage>
+}
+
+const log = fs.openSync(path.join(REPO_ROOT, 'output', 'openai.log'), 'a')
+console.log =
+  console.info =
+  console.debug =
+  console.warn =
+  console.error =
+    (...args: unknown[]) => fs.writeSync(log, `${args.map(String).join(' ')}\n`)
+
+const storage = resolveStorage('dev', {
+  root: path.join(REPO_ROOT, 'output', 'bench', 'sdk-root'),
+  models: process.env[MODEL_CACHE_PATH_ENV]
+})
+configureSdkStorage(storage.sdk)
+const sdk = await createSdkLoader().load()
+// lunte-disable-next-line qvac/no-as-cast -- SDK loadModel overloads omit this slice, as in index.ts
+const lifecycle = sdk as unknown as ModelLifecycleSdk
+const loaded = new Map<string, Promise<string>>()
+// the model answers one request at a time; the rest wait their turn
+let queue: Promise<unknown> = Promise.resolve()
+
+http
+  .createServer((req, res) => {
+    let body = ''
+    req.on('data', (chunk: unknown) => {
+      body += String(chunk)
+    })
+    req.on('end', () => {
+      const answered = queue.then(() => chat(JSON.parse(body)))
+      queue = answered.catch(noop)
+      answered.then(
+        (reply) => respond(res, 200, reply),
+        (err: unknown) => respond(res, 500, { error: { message: errorMessage(err) } })
+      )
+    })
+  })
+  .listen(port, '127.0.0.1')
+process.stdout.write(`qvac openai endpoint on http://127.0.0.1:${port}/v1\n`)
+
+async function chat(request: ChatRequest) {
+  const model = request.model.replace(/^openai\//, '')
+  const result = sdk.completion({
+    modelId: await modelId(model),
+    history: request.messages.map((message) => ({
+      role: message.role,
+      content: String(message.content ?? '')
+    })),
+    stream: true,
+    kvCache: conversationKey(model, request.messages),
+    // the simulated user speaks without reasoning first, as the benchmark's own user model would
+    generationParams: resolveLlmGenerationParams(model, REASONING_OFF)
+  })
+  let content = ''
+  for await (const token of result.tokenStream) {
+    content += token
+  }
+
+  await result.final
+
+  return {
+    id: `qvac-${Date.now()}`,
+    object: 'chat.completion',
+    created: Math.floor(Date.now() / 1000),
+    model: request.model,
+    choices: [
+      {
+        index: 0,
+        message: {
+          role: 'assistant',
+          content: endToken(content.replace(/<think>[\s\S]*?<\/think>/g, '').trim())
+        },
+        finish_reason: 'stop'
+      }
+    ]
+  }
+}
+
+function endToken(reply: string): string {
+  const word = reply.replace(/^[#\s]+|[#.!\s]+$/g, '').toUpperCase()
+  if (!END_TOKENS || !END_WORDS.has(word) || reply === `###${word}###`) {
+    return reply
+  }
+
+  console.log(`end token normalised: ${JSON.stringify(reply)} -> ###${word}###`)
+
+  return `###${word}###`
+}
+
+// the SDK sends only what a key has not seen, so a key names one conversation of one endpoint run
+function conversationKey(model: string, messages: ReadonlyArray<ChatMessage>): string {
+  const opening = JSON.stringify([STARTED, model, ...messages.slice(0, 2)])
+  return `openai-${createHash('sha256').update(opening).digest('hex').slice(0, 24)}`
+}
+
+function modelId(model: string): Promise<string> {
+  let id = loaded.get(model)
+  if (!id) {
+    id = resolveModelSrc(sdk, model).then((modelSrc) =>
+      lifecycle.loadModel({
+        modelSrc,
+        modelType: 'llamacpp-completion',
+        modelConfig: { ctx_size: CTX_SIZE, system_prompt: '' }
+      })
+    )
+    loaded.set(model, id)
+  }
+
+  return id
+}
+
+function respond(res: http.ServerResponse, status: 200 | 500, payload: unknown): void {
+  res.statusCode = status
+  res.setHeader('content-type', 'application/json')
+  res.end(JSON.stringify(payload))
+}
+
+function noop(): void {}

package/bench/bridge/python/qvac_bench/__init__.py

@@ -0,0 +1,3 @@
+from .client import QvacBridge
+
+__all__ = ["QvacBridge"]

package/bench/bridge/python/qvac_bench/airline.py

@@ -0,0 +1,42 @@
+"""What a tau2 airline write will do and cost, quoted to the user before it runs."""
+
+import json
+from typing import Any
+
+# the airline policy charges each non-free bag at this price
+BAG_PRICE = 50
+
+
+def _book_reservation(args: dict[str, Any]) -> str:
+    passengers = len(args.get("passengers") or [])
+    flights = ", ".join(f"{f.get('flight_number')} on {f.get('date')}" for f in args.get("flights") or [])
+    charged = sum(float(p.get("amount") or 0) for p in args.get("payment_methods") or [])
+    return (
+        f"Book {passengers} passenger(s) in {args.get('cabin')} on {flights}, "
+        f"{args.get('total_baggages', 0)} bag(s) ({int(args.get('nonfree_baggages') or 0)} paid), "
+        f"insurance {args.get('insurance')}, for a total of ${charged:g}."
+    )
+
+
+def _update_reservation_baggages(args: dict[str, Any]) -> str:
+    nonfree = int(args.get("nonfree_baggages") or 0)
+    return (
+        f"Set reservation {args.get('reservation_id')} to {args.get('total_baggages')} bag(s), "
+        f"{nonfree} paid at ${BAG_PRICE} each (${nonfree * BAG_PRICE}), paid with {args.get('payment_id')}."
+    )
+
+
+BUILDERS = {
+    "book_reservation": _book_reservation,
+    "cancel_reservation": lambda args: f"Cancel reservation {args.get('reservation_id')}.",
+    "update_reservation_baggages": _update_reservation_baggages,
+    "send_certificate": lambda args: f"Send a ${args.get('amount')} certificate to {args.get('user_id')}.",
+}
+
+
+def quote(name: str, args: dict[str, Any]) -> str:
+    """A plain summary when the call has no builder or its arguments are malformed."""
+    try:
+        return BUILDERS[name](args)
+    except (KeyError, AttributeError, TypeError, ValueError):
+        return f"Call {name} with {json.dumps(args)}."

package/bench/bridge/python/qvac_bench/approval.py

@@ -0,0 +1,26 @@
+"""Whether a simulated user's reply to a quoted write approves it."""
+
+import re
+
+# a model user rarely answers a bare "yes": it restates the change it confirms, or says it already did
+AGREES = re.compile(
+    r"^(yes|yeah|yep|sure|ok|okay|correct)\b"
+    r"|\b(i|i've|i have|i've already|i already|i have already) (confirm|confirmed)\b"
+    r"|\balready gave my (verbal )?confirmation\b"
+    r"|^\w+ confirmed\b"
+    r"|^(proceed|go ahead)\b"
+    r"|\bplease (proceed|go ahead)\b"
+)
+# a reply that disputes the quote or changes it goes back to the model as words
+HOLDS = re.compile(r"\b(but|however|wait|before|clarify|discrepancy|instead|cannot|can't|unable|not)\b|doesn't match")
+SENTENCE_END = re.compile(r"[.!?](\s|$)")
+
+
+def confirms(reply: str | None) -> bool:
+    text = (reply or "").strip().lower().replace("’", "'")
+    agrees = AGREES.search(text)
+    if not agrees or HOLDS.search(text):
+        return False
+    # a question up to the confirming sentence doubts the quote; one after it is the user's next request
+    end = SENTENCE_END.search(text, agrees.end())
+    return "?" not in text[: end.end() if end else len(text)]

package/bench/bridge/python/qvac_bench/bfcl_multi_turn.py

@@ -0,0 +1,152 @@
+"""BFCL v4 multi-turn with the harness as the model: BFCL's own entries, backends and checker.
+
+Each user turn is one harness run over the conversation so far. A call the run makes executes
+on gorilla's Python backends as BFCL's inference executes it, and the entry is scored by BFCL's
+multi_turn_checker as its eval runner calls it. Needs bfcl-eval from the pinned gorilla commit.
+"""
+
+import copy
+import json
+from typing import Any, Awaitable, Callable
+
+from bfcl_eval.constants.default_prompts import (
+    DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+    MAXIMUM_STEP_LIMIT,
+)
+from bfcl_eval.constants.enums import ModelStyle
+from bfcl_eval.constants.type_mappings import GORILLA_TO_OPENAPI
+from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_checker import multi_turn_checker
+from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import (
+    execute_multi_turn_func_call,
+    is_empty_execute_response,
+)
+from bfcl_eval.model_handler.utils import convert_to_function_call, convert_to_tool
+from bfcl_eval.utils import load_dataset_entry, load_ground_truth_entry
+
+from .client import Execute
+from .speed import add_speed, no_speed
+
+CATEGORIES = [
+    "multi_turn_base",
+    "multi_turn_miss_func",
+    "multi_turn_miss_param",
+    "multi_turn_long_context",
+]
+# BFCL force-quits a turn on its step past the limit, so the run may take one round more
+MAX_TOOL_ROUNDS = MAXIMUM_STEP_LIMIT + 1
+
+Run = Callable[[list[dict[str, Any]], list[dict[str, Any]], Execute], Awaitable[dict[str, Any]]]
+
+
+def load(category: str, limit: int | None) -> list[tuple[dict, list]]:
+    """Entries with their ground truth, spread evenly across the category as data.ts spreads them."""
+    entries = load_dataset_entry(category)
+    answers = {row["id"]: row["ground_truth"] for row in load_ground_truth_entry(category)}
+    count = min(limit or len(entries), len(entries))
+    step = max(1, len(entries) // count)
+    picked = [entry for index, entry in enumerate(entries) if index % step == 0][:count]
+    return [(entry, answers[entry["id"]]) for entry in picked]
+
+
+def specs(functions: list[dict]) -> list[dict]:
+    """The docs as BFCL's function-calling handlers send them."""
+    tools = convert_to_tool(functions, GORILLA_TO_OPENAPI, ModelStyle.OPENAI_COMPLETIONS)
+    return [{key: tool["function"][key] for key in ("name", "description", "parameters")} for tool in tools]
+
+
+class Episode:
+    """One entry. Its backends live in BFCL's own instances, keyed by run_name, so a run never sees another's state."""
+
+    def __init__(self, entry: dict, answer: list, run_name: str):
+        self.entry = entry
+        self.answer = answer
+        self.run_name = run_name
+        self.category = entry["id"].rsplit("_", 1)[0]
+        self.turns: list[list[list[dict]]] = []
+        # the calls the backends ran, in order; a call the harness refused never reaches them
+        self.ran: list[dict] = []
+        self.replies: list[str] = []
+        self.speed = no_speed()
+
+    # async only because the bridge client awaits every tool
+    async def execute(self, name: str, args: dict[str, Any]) -> Any:
+        self.ran.append({name: args})
+        results, _ = execute_multi_turn_func_call(
+            convert_to_function_call([{name: args}]),
+            self.entry["initial_config"],
+            self.entry["involved_classes"],
+            self.run_name,
+            self.entry["id"],
+            long_context="long_context" in self.category or "composite" in self.category,
+            is_evaL_run=False,
+        )
+        return as_sent(results[0])
+
+    async def play(self, run: Run) -> dict[str, Any]:
+        functions = copy.deepcopy(self.entry["function"])
+        holdout = self.entry.get("missed_function", {})
+        history: list[dict[str, Any]] = []
+        for index, messages in enumerate(self.entry["question"]):
+            if str(index) in holdout:
+                functions.extend(holdout[str(index)])
+                messages = [{"role": "user", "content": DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC}]
+            history.extend(messages)
+            ran = len(self.ran)
+            done = await run(history, specs(functions), self.execute)
+            history.extend(done["history"])
+            steps = executed(done["history"], self.ran[ran:])
+            self.turns.append(steps)
+            self.replies.append(done["reply"])
+            add_speed(self.speed, done.get("metrics"))
+            # BFCL fails an entry whose inference raised, without scoring it
+            if done["error"]:
+                return failed("multi_turn:inference_error", done["error"])
+            if len(steps) > MAXIMUM_STEP_LIMIT:
+                break
+        return self.score()
+
+    def score(self) -> dict[str, Any]:
+        """BFCL's eval runner's own steps; it is not imported, as it imports every provider's SDK."""
+        if len(self.turns) != len(self.answer):
+            return failed(
+                "multi_turn:force_terminated",
+                f"Model was force-terminated during inference phase. The length of the model result turns ({len(self.turns)}) does not match the length of the ground truth turns ({len(self.answer)}).",
+            )
+        decoded = [
+            [calls for calls in map(convert_to_function_call, turn) if not is_empty_execute_response(calls)]
+            for turn in self.turns
+        ]
+        result = multi_turn_checker(decoded, self.answer, self.entry, self.category, self.run_name)
+        if result["valid"]:
+            return {"valid": True, "error_type": None, "error": None}
+        return failed(result["error_type"], result["error_message"])
+
+
+def as_sent(result: str) -> Any:
+    """BFCL sends a backend's JSON as it is; the harness encodes a tool's result, so it gets the object."""
+    try:
+        parsed = json.loads(result)
+    except ValueError:
+        return result
+    return parsed if isinstance(parsed, dict) else result
+
+
+def executed(history: list[dict[str, Any]], ran: list[dict]) -> list[list[dict]]:
+    """The calls the backends ran, grouped by the round that made them, as BFCL's steps.
+
+    A call the harness refused, such as one cut off at the output limit, never ran, so it is not scored.
+    """
+    pending = list(ran)
+    steps = []
+    for turn in history:
+        step = []
+        for call in turn.get("toolCalls") or []:
+            if pending and {call["name"]: call["arguments"]} == pending[0]:
+                step.append(pending.pop(0))
+        if step:
+            steps.append(step)
+    return steps + ([pending] if pending else [])
+
+
+def failed(error_type: str, message: str) -> dict[str, Any]:
+    return {"valid": False, "error_type": error_type, "error": message}

package/bench/bridge/python/qvac_bench/client.py

@@ -0,0 +1,88 @@
+"""Drive @qvac/harness from Python: the harness runs the loop, the caller's environment runs the tools."""
+
+import asyncio
+import json
+import os
+from typing import Any, Awaitable, Callable
+
+FRAME = "\x1eqvac "
+# the native addon prints whole prompts on the same stdout, one line each
+LINE_LIMIT = 1 << 26
+
+Execute = Callable[[str, dict[str, Any]], Awaitable[Any]]
+
+
+class QvacBridge:
+    def __init__(self, repo: str, model: str, bare: str | None = None):
+        self.repo = repo
+        self.model = model
+        self.bare = bare or os.path.join(repo, "node_modules", ".bin", "bare")
+        self._proc: asyncio.subprocess.Process | None = None
+        self._runs = 0
+
+    async def start(self) -> None:
+        self._proc = await asyncio.create_subprocess_exec(
+            self.bare,
+            "bench/bridge/serve.ts",
+            cwd=self.repo,
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.DEVNULL,
+            limit=LINE_LIMIT,
+        )
+        await self._expect("ready")
+
+    async def run(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        execute: Execute,
+        system: str | None = None,
+        max_tool_rounds: int | None = None,
+        cache_key: str | None = None,
+    ) -> dict[str, Any]:
+        run_id = f"run-{self._runs}"
+        self._runs += 1
+        run = {"type": "run", "id": run_id, "model": self.model, "system": system, "messages": messages, "tools": tools}
+        # the harness reads a null limit as invalid, so an unset one is left out
+        if max_tool_rounds is not None:
+            run["maxToolRounds"] = max_tool_rounds
+        if cache_key is not None:
+            run["cacheKey"] = cache_key
+        self._send(run)
+        while True:
+            frame = await self._frame()
+            if frame["type"] == "call":
+                result = await execute(frame["name"], frame["args"])
+                self._send({"type": "result", "callId": frame["callId"], "result": result})
+            elif frame["type"] == "done" and frame["id"] == run_id:
+                return frame
+
+    def forget(self, cache_key: str) -> None:
+        """Drops a finished conversation's cached prefill."""
+        self._send({"type": "forget", "cacheKey": cache_key})
+
+    async def close(self) -> None:
+        if self._proc is None:
+            return
+        self._proc.stdin.close()
+        await self._proc.wait()
+        self._proc = None
+
+    def _send(self, frame: dict[str, Any]) -> None:
+        self._proc.stdin.write((json.dumps(frame) + "\n").encode())
+
+    async def _expect(self, kind: str) -> dict[str, Any]:
+        frame = await self._frame()
+        if frame["type"] != kind:
+            raise RuntimeError(f"expected {kind}, got {frame}")
+        return frame
+
+    async def _frame(self) -> dict[str, Any]:
+        while True:
+            line = await self._proc.stdout.readline()
+            if not line:
+                raise RuntimeError("the harness bridge exited")
+            text = line.decode(errors="replace")
+            if text.startswith(FRAME):
+                return json.loads(text[len(FRAME):])

package/bench/bridge/python/qvac_bench/harbor_agent.py

@@ -0,0 +1,85 @@
+"""A Harbor agent: the harness's loop with one bash tool that runs in the task container.
+
+    QVAC_HARNESS_REPO=$PWD PYTHONPATH=bench/bridge/python uv tool run harbor run \\
+      --dataset terminal-bench@2.0 -a qvac_bench.harbor_agent:QvacHarnessAgent \\
+      -m Qwen3.5-9B-Q4_K_M -n 1 --ak max_tool_rounds=10
+"""
+
+import json
+import os
+import time
+
+from harbor.agents.base import BaseAgent
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+from .client import QvacBridge
+from .tools import BASH
+
+# keep a tool result inside a small model's context
+OUTPUT_TAIL = 8000
+COMMAND_TIMEOUT_SEC = 180
+# what timeout(1) exits with
+TIMEOUT_RETURN_CODE = 124
+
+SYSTEM = (
+    "You are working in a Linux terminal to complete a task. Use the bash tool to inspect "
+    "the environment, make changes and verify them. When the task is complete, say so."
+)
+
+
+class QvacHarnessAgent(BaseAgent):
+    def __init__(self, *args, max_tool_rounds: int | str | None = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.max_tool_rounds = None if max_tool_rounds in (None, "") else int(max_tool_rounds)
+
+    @staticmethod
+    def name() -> str:
+        return "qvac-harness"
+
+    def version(self) -> str | None:
+        return "bridge-1"
+
+    async def setup(self, environment: BaseEnvironment) -> None:
+        return None
+
+    async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None:
+        model = self.model_name or "Qwen3.5-9B-Q4_K_M"
+        bridge = QvacBridge(os.environ["QVAC_HARNESS_REPO"], model)
+        # the load average shows whether other work on the machine slowed the run
+        log = open(os.path.join(self.logs_dir, "qvac.jsonl"), "a")
+
+        def write(row: dict) -> None:
+            log.write(json.dumps(row) + "\n")
+            log.flush()
+
+        async def execute(name: str, args: dict) -> dict:
+            command = str(args.get("command", ""))
+            started = time.monotonic()
+            # DockerEnvironment.exec raises on timeout; the model gets it as a failed command
+            try:
+                result = await environment.exec(command, timeout_sec=COMMAND_TIMEOUT_SEC)
+                stdout, stderr, return_code = result.stdout or "", result.stderr or "", result.return_code
+            except RuntimeError as error:
+                stdout, stderr, return_code = "", str(error), TIMEOUT_RETURN_CODE
+            write({"type": "call", "command": command, "return_code": return_code, "seconds": time.monotonic() - started})
+            return {"stdout": stdout[-OUTPUT_TAIL:], "stderr": stderr[-OUTPUT_TAIL:], "return_code": return_code}
+
+        write({"type": "start", "model": model, "max_tool_rounds": self.max_tool_rounds, "load": os.getloadavg()})
+        try:
+            await bridge.start()
+            done = await bridge.run(
+                [{"role": "user", "content": instruction}],
+                [BASH],
+                execute,
+                system=SYSTEM,
+                max_tool_rounds=self.max_tool_rounds,
+            )
+            write(done)
+            rounds = (done.get("metrics") or {}).get("rounds") or []
+            context.n_input_tokens = sum(record.get("promptTokens") or 0 for record in rounds)
+            context.n_output_tokens = sum(record.get("generatedTokens") or 0 for record in rounds)
+        finally:
+            write({"type": "end", "load": os.getloadavg()})
+            log.close()
+            await bridge.close()

package/bench/bridge/python/qvac_bench/session.py

@@ -0,0 +1,107 @@
+"""A synchronous bridge session, for benchmarks whose agent API returns one message at a time."""
+
+import json
+import os
+import subprocess
+import time
+from typing import Any
+
+from .client import FRAME
+
+# the addon prints whole prompts on the same stdout; the harness's own log lines are short
+TRACE_LINE_LIMIT = 4000
+
+
+class QvacSession:
+    def __init__(self, repo: str, model: str, bare: str | None = None, trace: str | None = None):
+        self.model = model
+        self._trace = open(trace, "a") if trace else None
+        self._runs = 0
+        # the run whose frames this session answers; a frame of any other run is stale
+        self._active: str | None = None
+        self._proc = subprocess.Popen(
+            [bare or os.path.join(repo, "node_modules", ".bin", "bare"), "bench/bridge/serve.ts"],
+            cwd=repo,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            env={**os.environ, "QVAC_HARNESS_LOG": "1"} if trace else None,
+            text=True,
+            bufsize=1,
+        )
+        self._expect("ready")
+
+    def start(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        system: str | None,
+        now: int | None = None,
+    ) -> dict[str, Any]:
+        """Starts a run and returns its first call, approve or done frame."""
+        self.finish()
+        self._runs += 1
+        self._active = f"run-{self._runs}"
+        self._send({
+            "type": "run",
+            "id": self._active,
+            "model": self.model,
+            "system": system,
+            "messages": messages,
+            "tools": tools,
+            "now": now,
+        })
+        return self._next()
+
+    def answer(self, call_id: str, result: Any) -> dict[str, Any]:
+        """Returns a call's result and the active run's next call, approve or done frame."""
+        self._send({"type": "result", "callId": call_id, "result": result})
+        return self._next()
+
+    def decide(self, call_id: str, approved: bool) -> dict[str, Any]:
+        """Settles a write the run asked the user about and returns its next frame."""
+        self._send({"type": "decision", "callId": call_id, "approved": approved})
+        return self._next()
+
+    def finish(self) -> None:
+        """Ends a run the benchmark abandoned mid-turn, so its frames never reach the next task."""
+        if self._active is None:
+            return
+        self._send({"type": "cancel", "id": self._active})
+        while self._next()["type"] != "done":
+            pass
+
+    def close(self) -> None:
+        self._proc.stdin.close()
+        self._proc.wait()
+        if self._trace:
+            self._trace.close()
+
+    def _send(self, frame: dict[str, Any]) -> None:
+        self._proc.stdin.write(json.dumps(frame) + "\n")
+        self._proc.stdin.flush()
+
+    def _next(self) -> dict[str, Any]:
+        while True:
+            frame = self._frame()
+            if frame.get("id") != self._active or frame["type"] not in ("call", "approve", "done"):
+                continue
+            if frame["type"] == "done":
+                self._active = None
+            return frame
+
+    def _expect(self, kind: str) -> None:
+        frame = self._frame()
+        if frame["type"] != kind:
+            raise RuntimeError(f"expected {kind}, got {frame}")
+
+    def _frame(self) -> dict[str, Any]:
+        while True:
+            line = self._proc.stdout.readline()
+            if not line:
+                raise RuntimeError("the harness bridge exited")
+            if line.startswith(FRAME):
+                return json.loads(line[len(FRAME):])
+            if self._trace and len(line) < TRACE_LINE_LIMIT:
+                self._trace.write(f"{time.strftime('%H:%M:%S')} {line}")
+                self._trace.flush()