@qvac/harness 1.0.0-rc.1 → 1.0.0-rc.3

package/bench/bridge/python/qvac_bench/speed.py

@@ -0,0 +1,30 @@
+"""A run's speed as sums, in the shape of bench/speed.ts, so runs add up."""
+
+
+def add_speed(speed: dict, metrics: dict | None) -> None:
+    """Adds one run's metrics to a task's sums, in the shape of bench/speed.ts."""
+    if not metrics:
+        return
+    speed["runs"] += 1
+    for record in metrics.get("rounds", []):
+        speed["rounds"] += 1
+        prompt, prefill_rate = record.get("promptTokens"), record.get("promptTokensPerSecond")
+        if prompt and prefill_rate:
+            speed["promptTokens"] += prompt
+            speed["prefillMs"] += prompt / prefill_rate * 1000
+        generated, decode_rate = record.get("generatedTokens"), record.get("tokensPerSecond")
+        if generated and decode_rate:
+            speed["generatedTokens"] += generated
+            speed["decodeMs"] += generated / decode_rate * 1000
+    speed["modelMs"] += metrics.get("modelMs", 0)
+    speed["toolMs"] += metrics.get("toolMs", 0)
+    speed["wallMs"] += metrics["durationMs"]
+    if metrics.get("timeToFirstToken") is not None:
+        speed["ttftMs"].append(metrics["timeToFirstToken"])
+
+
+def no_speed() -> dict:
+    return {
+        "runs": 0, "rounds": 0, "promptTokens": 0, "prefillMs": 0, "generatedTokens": 0,
+        "decodeMs": 0, "modelMs": 0, "toolMs": 0, "wallMs": 0, "ttftMs": [],
+    }

package/bench/bridge/python/qvac_bench/tau2_agent.py

@@ -0,0 +1,106 @@
+"""A tau2-bench agent whose every turn is a harness run; tau2's orchestrator still runs the tools.
+
+A call frame becomes an assistant message carrying that one tool call, and the ToolMessage
+tau2 sends back answers it, so the same harness run continues until it replies to the user.
+A write first comes back as an approve frame: the agent quotes it to the user, and a reply that
+confirms it lets the same run make the call. Any other answer starts a new run, so the model
+reads the user's words as the conversation goes on.
+"""
+
+import json
+import sys
+from typing import Any, Optional
+
+from tau2.agent.base_agent import HalfDuplexAgent
+from tau2.agent.llm_agent import AGENT_INSTRUCTION, SYSTEM_PROMPT
+from tau2.data_model.message import AssistantMessage, MultiToolMessage, ToolCall, ToolMessage, UserMessage
+from tau2.environment.toolkit import Tool
+
+from .airline import quote
+from .approval import confirms
+from .session import QvacSession
+from .speed import add_speed, no_speed
+
+
+# the airline domain's writes; every other tool reads, so it runs without asking
+WRITE_TOOLS = {
+    "book_reservation",
+    "cancel_reservation",
+    "update_reservation_baggages",
+    "update_reservation_flights",
+    "update_reservation_passengers",
+    "send_certificate",
+}
+
+
+class QvacTau2Agent(HalfDuplexAgent[dict]):
+    def __init__(self, tools: list[Tool], domain_policy: str, session: QvacSession, now: int | None = None):
+        super().__init__(tools=tools, domain_policy=domain_policy)
+        self.session = session
+        # the policy's own date, in epoch ms: the harness otherwise stamps today on every turn
+        self.now = now
+        self.system = SYSTEM_PROMPT.format(domain_policy=domain_policy, agent_instruction=AGENT_INSTRUCTION)
+        self.specs = [
+            {**spec, "write": spec["name"] in WRITE_TOOLS}
+            for spec in (tool.openai_schema["function"] for tool in tools)
+        ]
+        # a turn the harness ended without a reply, and why: the SDK's errors count against the agent
+        self.replies = 0
+        self.failed_turns = 0
+        self.errors: dict[str, int] = {}
+        # τ² runs carry no cache key, so every round primes the cache cold
+        self.speed = no_speed()
+
+    def get_init_state(self, message_history: Optional[list] = None) -> dict:
+        return {"history": [], "calls": {}}
+
+    def generate_next_message(self, message: Any, state: dict) -> tuple[AssistantMessage, dict]:
+        if isinstance(message, UserMessage):
+            asked = state.pop("approval", None)
+            state["history"].append({"role": "user", "content": message.content or ""})
+            if asked and confirms(message.content):
+                frame = self.session.decide(asked, True)
+            else:
+                frame = self.session.start(state["history"], self.specs, self.system, self.now)
+        elif isinstance(message, ToolMessage):
+            frame = self.answer(message, state)
+        elif isinstance(message, MultiToolMessage):
+            frame = None
+            for tool_message in message.tool_messages:
+                frame = self.answer(tool_message, state)
+        else:
+            raise TypeError(f"unexpected message {type(message).__name__}")
+
+        if frame["type"] == "call":
+            state["calls"][frame["callId"]] = frame["name"]
+            replayed = {"id": frame["callId"], "name": frame["name"], "arguments": frame["args"]}
+            state["history"].append({"role": "assistant", "content": "", "toolCalls": [replayed]})
+            call = ToolCall(id=frame["callId"], name=frame["name"], arguments=frame["args"])
+            return AssistantMessage(role="assistant", content=None, tool_calls=[call]), state
+
+        if frame["type"] == "approve":
+            state["approval"] = frame["callId"]
+            ask = f"{quote(frame['name'], frame['args'])} Reply yes to confirm."
+            state["history"].append({"role": "assistant", "content": ask})
+            return AssistantMessage(role="assistant", content=ask), state
+
+        self.replies += 1
+        add_speed(self.speed, frame.get("metrics"))
+        if not frame["reply"].strip() or frame["error"]:
+            self.failed_turns += 1
+            error = frame["error"] or "empty reply"
+            self.errors[error] = self.errors.get(error, 0) + 1
+            print(f"[qvac] failed turn: calls={frame['calls']} error={error!r}", file=sys.stderr, flush=True)
+        reply = frame["reply"].strip() or "I'm sorry, could you say that again?"
+        state["history"].append({"role": "assistant", "content": reply})
+        return AssistantMessage(role="assistant", content=reply), state
+
+    def answer(self, message: ToolMessage, state: dict) -> dict[str, Any]:
+        # the run got the text as its result, so the replay carries it as a JSON string
+        state["history"].append({
+            "role": "tool",
+            "toolCallId": message.id,
+            "name": state["calls"][message.id],
+            "content": json.dumps(message.content),
+        })
+        return self.session.answer(message.id, message.content)

package/bench/bridge/python/qvac_bench/tools.py

@@ -0,0 +1,11 @@
+"""Tool specs a benchmark hands the bridge; the benchmark's environment runs them."""
+
+BASH = {
+    "name": "bash",
+    "description": "Run one shell command in the task's Linux container and get its stdout, stderr and exit code.",
+    "parameters": {
+        "type": "object",
+        "properties": {"command": {"type": "string", "description": "The command to run"}},
+        "required": ["command"],
+    },
+}

package/bench/bridge/python/run_bfcl_multi_turn.py

@@ -0,0 +1,95 @@
+"""BFCL v4 multi-turn with the harness as the model, scored by BFCL's own checker.
+
+    output/venvs/bfcl/bin/python bench/bridge/python/run_bfcl_multi_turn.py \\
+      --models Qwen3.5-2B-Q4_K_M,Qwen3.5-9B-Q4_K_M --limit 10 --out output/bfcl-multi-turn/run
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(__file__))
+from qvac_bench.bfcl_multi_turn import CATEGORIES, MAX_TOOL_ROUNDS, Episode, load  # noqa: E402
+from qvac_bench.client import QvacBridge  # noqa: E402
+
+REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+
+
+async def run_model(model: str, cases: list, args: argparse.Namespace, stamp: str, commit: str) -> None:
+    bridge = QvacBridge(REPO, model)
+    await bridge.start()
+    run_name = f"qvac_{model}_{stamp}"
+    try:
+        for entry, answer in cases:
+            episode = Episode(entry, answer, run_name)
+            key = f"bfcl-multi-turn-{stamp}-{model}-{entry['id']}"
+
+            def run(messages, tools, execute, key=key):
+                return bridge.run(messages, tools, execute, max_tool_rounds=args.max_tool_rounds, cache_key=key)
+
+            started = time.time()
+            verdict = await episode.play(run)
+            bridge.forget(key)
+            row = {
+                "model": model,
+                "category": episode.category,
+                "id": entry["id"],
+                "commit": commit,
+                **verdict,
+                "turns": len(answer),
+                "played": len(episode.turns),
+                "calls": [[call for step in turn for call in step] for turn in episode.turns],
+                "replies": [reply[:300] for reply in episode.replies],
+                "seconds": round(time.time() - started, 1),
+                "speed": episode.speed,
+            }
+            with open(os.path.join(args.out, "runs.jsonl"), "a") as f:
+                f.write(json.dumps(row) + "\n")
+            outcome = "pass" if verdict["valid"] else verdict["error_type"]
+            print(f"{model} {entry['id']}: {outcome} in {row['seconds']}s", flush=True)
+    finally:
+        await bridge.close()
+
+
+def summary(out: str) -> str:
+    with open(os.path.join(out, "runs.jsonl")) as f:
+        rows = [json.loads(line) for line in f if line.strip()]
+    lines = []
+    for model in dict.fromkeys(row["model"] for row in rows):
+        for category in CATEGORIES:
+            own = [row for row in rows if row["model"] == model and row["category"] == category]
+            if own:
+                passed = sum(row["valid"] for row in own)
+                lines.append(f"{model} {category}: {passed}/{len(own)} ({round(passed / len(own) * 100)}%)")
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--categories", default=",".join(CATEGORIES))
+    parser.add_argument("--models", default="Qwen3.5-9B-Q4_K_M")
+    parser.add_argument("--limit", type=int, default=None, help="entries per category, spread evenly across it")
+    parser.add_argument("--max-tool-rounds", type=int, default=MAX_TOOL_ROUNDS, help="per turn; default BFCL's step limit")
+    parser.add_argument("--out", default=None)
+    args = parser.parse_args()
+
+    stamp = time.strftime("%Y%m%d-%H%M%S")
+    args.out = args.out or os.path.join(REPO, "output", "bfcl-multi-turn", stamp)
+    os.makedirs(args.out, exist_ok=True)
+    # report.ts reads every row, so a second run into the same dir would count each entry twice
+    if os.path.exists(os.path.join(args.out, "runs.jsonl")):
+        sys.exit(f"{args.out} already holds a run; pick another --out")
+    commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=REPO, capture_output=True, text=True).stdout.strip()
+    cases = [case for category in args.categories.split(",") for case in load(category, args.limit)]
+    print(f"{len(cases)} BFCL multi-turn entries -> {args.out}", flush=True)
+    for model in args.models.split(","):
+        asyncio.run(run_model(model, cases, args, stamp, commit))
+    print(summary(args.out))
+
+
+if __name__ == "__main__":
+    main()

package/bench/bridge/python/run_tau2.py

@@ -0,0 +1,150 @@
+"""tau2-bench with the harness as the agent and a local model as the simulated user.
+
+    output/venvs/tau2/bin/python bench/bridge/python/run_tau2.py --domain airline --tasks 10
+
+Scores are tau2's own rewards, but not comparable to its leaderboard, whose user is GPT-4.1.
+"""
+
+import argparse
+import datetime
+import json
+import os
+import re
+import signal
+import subprocess
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(__file__))
+from qvac_bench.session import QvacSession  # noqa: E402
+from qvac_bench.tau2_agent import QvacTau2Agent  # noqa: E402
+from tau2.evaluator.evaluator import EvaluationType  # noqa: E402
+from tau2.orchestrator.orchestrator import Orchestrator  # noqa: E402
+from tau2.runner import build_environment, build_user, get_tasks, run_simulation  # noqa: E402
+
+REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+PORT = 8765
+
+
+def start_user_endpoint() -> subprocess.Popen:
+    endpoint = subprocess.Popen(
+        [os.path.join(REPO, "node_modules", ".bin", "bare"), "bench/bridge/openai.ts"],
+        cwd=REPO,
+        env={**os.environ, "QVAC_OPENAI_PORT": str(PORT), "QVAC_OPENAI_END_TOKENS": "1"},
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        text=True,
+        # the bin shim spawns the runtime, so only a process group takes both down
+        start_new_session=True,
+    )
+    for line in endpoint.stdout:
+        if "endpoint on" in line:
+            return endpoint
+    raise RuntimeError("the user endpoint exited before listening; is port %d taken?" % PORT)
+
+
+def stop(process: subprocess.Popen) -> None:
+    os.killpg(process.pid, signal.SIGTERM)
+    process.wait()
+
+
+# the task files run roughly from refusals to multi-step changes, so the first N are not a sample
+def sample(tasks: list, count: int | None) -> list:
+    if not count or count >= len(tasks):
+        return tasks
+    step = len(tasks) / count
+    return [tasks[int(index * step)] for index in range(count)]
+
+
+# the policy's own hand-off line; τ²'s customer should answer it with ###TRANSFER###, which ends the task
+TRANSFER_MESSAGE = "YOU ARE BEING TRANSFERRED TO A HUMAN AGENT"
+# τ²'s end tokens are ###STOP### and the like; a customer that drops the hashes never ends the task
+END_WORDS = {"STOP", "TRANSFER", "OUT-OF-SCOPE"}
+
+
+def customer_never_ended(result) -> bool:
+    if not str(result.termination_reason).endswith("MAX_STEPS"):
+        return False
+    handed_off = any(m.role == "assistant" and TRANSFER_MESSAGE in (m.content or "") for m in result.messages)
+    bare_end = any(m.role == "user" and (m.content or "").strip(" #.").upper() in END_WORDS for m in result.messages)
+    return handed_off or bare_end
+
+
+# a task whose expected actions change the database, as opposed to one the policy says to refuse
+def needs_write(task) -> bool:
+    actions = (task.evaluation_criteria.actions or []) if task.evaluation_criteria else []
+    return any(not action.name.startswith(("get_", "search_", "list_", "calculate", "think", "transfer")) for action in actions)
+
+
+# tau2 policies state their own present ("The current time is 2024-05-15 15:00:00 EST.")
+def policy_now(policy: str) -> int | None:
+    match = re.search(r"current time is (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) EST", policy)
+    if not match:
+        return None
+    est = datetime.timezone(datetime.timedelta(hours=-5))
+    return int(datetime.datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S").replace(tzinfo=est).timestamp() * 1000)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--domain", default="airline")
+    parser.add_argument("--model", default="Qwen3.5-9B-Q4_K_M")
+    parser.add_argument("--user-model", default="Qwen3.5-9B-Q4_K_M")
+    parser.add_argument("--tasks", type=int, default=None)
+    parser.add_argument("--task-ids", default=None, help="comma-separated ids, instead of a sample")
+    parser.add_argument("--max-steps", type=int, default=100)
+    parser.add_argument("--out", default=os.path.join(REPO, "output", "tau2"))
+    args = parser.parse_args()
+
+    os.makedirs(args.out, exist_ok=True)
+    out_file = os.path.join(args.out, f"{args.domain}-{args.model}-{int(time.time())}.jsonl")
+    endpoint = start_user_endpoint()
+    commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=REPO, capture_output=True, text=True).stdout.strip()
+    session = QvacSession(REPO, args.model, trace=out_file.replace(".jsonl", ".trace"))
+    user_llm_args = {"api_base": f"http://127.0.0.1:{PORT}/v1", "api_key": "local", "temperature": 0.0}
+    rewards = []
+    try:
+        chosen = args.task_ids.split(",") if args.task_ids else None
+        tasks = get_tasks(args.domain, task_ids=chosen) if chosen else sample(get_tasks(args.domain), args.tasks)
+        for task in tasks:
+            env = build_environment(args.domain)
+            agent = QvacTau2Agent(env.get_tools(), env.get_policy(), session, policy_now(env.get_policy()))
+            user = build_user("user_simulator", env, task, llm=f"openai/{args.user_model}", llm_args=user_llm_args)
+            orchestrator = Orchestrator(
+                domain=args.domain, agent=agent, user=user, environment=env, task=task,
+                max_steps=args.max_steps, max_errors=10, seed=42,
+            )
+            started = time.time()
+            try:
+                result = run_simulation(orchestrator, evaluation_type=EvaluationType.ALL)
+            finally:
+                session.finish()
+            reward = result.reward_info.reward if result.reward_info else 0.0
+            rewards.append(reward)
+            row = {
+                "task": task.id,
+                "commit": commit,
+                "writes": needs_write(task),
+                "reward": reward,
+                "termination": str(getattr(result, "termination_reason", "")),
+                "breakdown": {str(k): v for k, v in (result.reward_info.reward_breakdown or {}).items()} if result.reward_info else {},
+                "messages": len(result.messages),
+                "customer_never_ended": customer_never_ended(result),
+                "replies": agent.replies,
+                "failed_turns": agent.failed_turns,
+                "errors": agent.errors,
+                "speed": agent.speed,
+                "seconds": round(time.time() - started, 1),
+            }
+            with open(out_file, "a") as f:
+                f.write(json.dumps(row) + "\n")
+            print(f"{task.id}: reward {reward} in {row['seconds']}s ({row['messages']} messages)", flush=True)
+    finally:
+        session.close()
+        stop(endpoint)
+
+    print(f"{args.domain} {args.model}: pass^1 {sum(rewards) / max(1, len(rewards)):.3f} over {len(rewards)} tasks -> {out_file}")
+
+
+if __name__ == "__main__":
+    main()

package/bench/bridge/python/smoke.py

@@ -0,0 +1,47 @@
+"""Smoke the bridge without a benchmark: a real model, one bash tool, a throwaway local folder."""
+
+import asyncio
+import os
+import subprocess
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(__file__))
+from qvac_bench import QvacBridge  # noqa: E402
+from qvac_bench.tools import BASH  # noqa: E402
+
+REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+
+
+async def main(model: str) -> None:
+    workdir = os.path.realpath(tempfile.mkdtemp(prefix="qvac-bridge-"))
+    with open(os.path.join(workdir, "notes.txt"), "w") as f:
+        f.write("launch: 14 March\n")
+
+    async def execute(name: str, args: dict) -> dict:
+        # the model's commands run on this machine, so confine them: no network, writes only in workdir
+        profile = f'(version 1)(allow default)(deny network*)(deny file-write*)(allow file-write* (subpath "{workdir}") (literal "/dev/null"))'
+        done = subprocess.run(
+            ["sandbox-exec", "-p", profile, "/bin/sh", "-c", args["command"]],
+            cwd=workdir, capture_output=True, text=True, timeout=30,
+        )
+        return {"stdout": done.stdout, "stderr": done.stderr, "return_code": done.returncode}
+
+    bridge = QvacBridge(REPO, model)
+    await bridge.start()
+    try:
+        done = await bridge.run(
+            [{"role": "user", "content": "Create a file called answer.txt containing the launch date from notes.txt."}],
+            [BASH],
+            execute,
+            system="You work in a terminal. Use the bash tool.",
+        )
+    finally:
+        await bridge.close()
+
+    answer = os.path.join(workdir, "answer.txt")
+    print({"reply": done["reply"][:200], "calls": done["calls"], "error": done["error"]})
+    print("answer.txt:", open(answer).read().strip() if os.path.exists(answer) else None)
+
+
+asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else "Qwen3.5-4B-Q4_K_M"))

package/bench/bridge/python/test_airline.py

@@ -0,0 +1,28 @@
+"""python3 -m unittest test_airline, from this folder: tau2 is not needed."""
+
+import unittest
+
+from qvac_bench.airline import quote
+
+
+class Quote(unittest.TestCase):
+    def test_flights_that_are_not_objects_fall_back(self):
+        self.assertTrue(quote("book_reservation", {"flights": ["HAT1"]}).startswith("Call book_reservation with "))
+
+    def test_a_passenger_count_falls_back(self):
+        self.assertTrue(quote("book_reservation", {"passengers": 5}).startswith("Call book_reservation with "))
+
+    def test_a_string_amount_is_priced(self):
+        self.assertIn("total of $100.", quote("book_reservation", {"payment_methods": [{"amount": "100"}]}))
+        self.assertTrue(quote("book_reservation", {"payment_methods": [{"amount": "lots"}]}).startswith("Call "))
+
+    def test_a_string_bag_count_is_priced(self):
+        self.assertIn("2 paid at $50 each ($100)", quote("update_reservation_baggages", {"nonfree_baggages": "2"}))
+        self.assertIn("(2 paid)", quote("book_reservation", {"nonfree_baggages": "2"}))
+
+    def test_an_unknown_tool_is_summarised(self):
+        self.assertEqual(quote("get_user_details", {"user_id": "u1"}), 'Call get_user_details with {"user_id": "u1"}.')
+
+
+if __name__ == "__main__":
+    unittest.main()

package/bench/bridge/python/test_approval.py

@@ -0,0 +1,57 @@
+"""python3 -m unittest test_approval, from this folder: tau2 is not needed."""
+
+import unittest
+
+from qvac_bench.approval import confirms
+
+# replies a Qwen3.5 simulated user gave to a quoted tau2 write
+APPROVALS = [
+    "yes",
+    "Yes.",
+    "Yes, please proceed with the cabin upgrade.",
+    "Okay, I confirm the upgrade to business class for XEHM4B using the card ending in 2135.",
+    "I confirm.",
+    "I confirm the cabin upgrade to business class for both passengers on flights HAT235 and HAT298.",
+    "I already confirmed the name change from Mei Lee to Mei Garcia. Please proceed with updating it.",
+    "I’ve already confirmed that FDZ0T5 should be cancelled. Please go ahead and cancel it now.",
+    "I understand you're asking for confirmation. I already gave my verbal confirmation earlier.",
+    "Upgrade confirmed. Now cancel both reservations: XEHM4B and 59XX6W.",
+    "I understand. Please proceed with the upgrade and cancellation for XEHM4B. Could you please handle this?",
+    "I already confirmed the cancellation of FDZ0T5 earlier. Should I proceed with PUNERT as well?",
+]
+
+HOLDS = [
+    "",
+    "I",
+    "###STOP###",
+    "make it two bags",
+    "The total cost is $255, which doesn't match the flight price of $103 plus free baggage.",
+    "I see the confirmation request. However, there's a discrepancy in the date.",
+    "I need to clarify something before confirming. The payment method I prefer is a gift card.",
+    "Wait, before I confirm anything, let me clarify: you're asking to cancel all flights?",
+    "I'm sorry, but I cannot proceed with the cabin upgrade at this time.",
+    "I see the booking is confirmed with a total of $400.",
+    "I have successfully booked your flight!",
+    "Yes. But make it two bags.",
+    "Since I already confirmed yes verbally, could you please proceed with updating my reservation?",
+    "I don't have my date of birth in my user profile. Can you please proceed without it?",
+]
+
+
+class Confirms(unittest.TestCase):
+    def test_a_confirmation_approves(self):
+        for reply in APPROVALS:
+            with self.subTest(reply=reply):
+                self.assertTrue(confirms(reply))
+
+    def test_a_change_question_or_dispute_goes_back_to_the_model(self):
+        for reply in HOLDS:
+            with self.subTest(reply=reply):
+                self.assertFalse(confirms(reply))
+
+    def test_no_reply_holds(self):
+        self.assertFalse(confirms(None))
+
+
+if __name__ == "__main__":
+    unittest.main()