@tangle-network/agent-bench 0.4.0 → 0.4.1

package/pier_agents/candidate_contract.py

@@ -34,8 +34,13 @@ _RESERVED_PATH_PARTS = {".git", ".sidecar"}
 _PLAN_KIND = "agent-candidate-execution-plan-material"
 _RECEIPT_KIND = "agent-candidate-materialization"
 _PROFILE_PLAN_KIND = "agent-profile-workspace-plan"
+_PROFILE_ACTIVATION_KIND = "agent-candidate-profile-activation"
 _EXECUTION_EVIDENCE_KIND = "agent-candidate-execution-plan"
+_RUN_CELL_KIND = "agent-candidate-run-cell"
+_BENCHMARK_SUITE_KIND = "agent-candidate-benchmark-suite"
+_BENCHMARK_TASK_KIND = "agent-candidate-benchmark-task"
 _STDIN_TASK_PATH = "/tangle/input/stdin.txt"
+_MAX_SAFE_INTEGER = 9_007_199_254_740_991
 
 
 class CandidateContractError(ValueError):
@@ -130,6 +135,30 @@ def _integer(value: Any, label: str, *, minimum: int = 0) -> int:
     return value
 
 
+def _safe_integer(value: Any, label: str) -> int:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, int)
+        or value < -_MAX_SAFE_INTEGER
+        or value > _MAX_SAFE_INTEGER
+    ):
+        raise CandidateContractError(f"{label} must be a safe integer")
+    return value
+
+
+def _canonical_bytes(value: Any, label: str) -> bytes:
+    try:
+        return json.dumps(
+            value,
+            ensure_ascii=False,
+            allow_nan=False,
+            separators=(",", ":"),
+            sort_keys=True,
+        ).encode("utf-8")
+    except (TypeError, ValueError, UnicodeEncodeError) as exc:
+        raise CandidateContractError(f"{label} is not canonical JSON") from exc
+
+
 def _number(value: Any, label: str, *, minimum: float = 0) -> float:
     if (
         isinstance(value, bool)
@@ -246,6 +275,21 @@ def _embedded_bytes(value: Any, label: str) -> bytes:
     return raw
 
 
+def _embedded_contract(
+    value: Any, label: str, *, kind: str
+) -> tuple[dict[str, Any], bytes, str]:
+    evidence = _object(value, label)
+    digest = _digest(evidence.get("digest"), f"{label}.digest")
+    raw = _embedded_bytes(evidence.get("material"), f"{label}.material")
+    if sha256_bytes(raw) != digest:
+        raise CandidateContractError(f"{label} material does not match its digest")
+    try:
+        material = json.loads(raw)
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise CandidateContractError(f"{label} material is not UTF-8 JSON") from exc
+    return _contract_object(material, f"{label}.material", kind=kind), raw, digest
+
+
 def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]:
     snapshot = _contract_object(
         value, label, kind="agent-candidate-workspace-snapshot"
@@ -318,11 +362,10 @@ def _profile_files(value: Any) -> tuple[ProfileFile, ...]:
     return tuple(files)
 
 
-def _instruction(value: Any) -> InstructionEvidence:
-    obj = _object(value, "task.instruction")
-    if obj.get("encoding") != "utf8":
-        raise CandidateContractError("task.instruction.encoding must equal utf8")
-    delivery = _object(obj.get("delivery"), "task.instruction.delivery")
+def _instruction(value: Any, delivery_value: Any) -> InstructionEvidence:
+    text = _string(value, "benchmark task instruction")
+    raw = text.encode("utf-8")
+    delivery = _object(delivery_value, "plan.instructionDelivery")
     kind = delivery.get("kind")
     if kind == "argv-append" or kind == "stdin-utf8":
         if set(delivery) != {"kind"}:
@@ -342,10 +385,8 @@ def _instruction(value: Any) -> InstructionEvidence:
     else:
         raise CandidateContractError("unsupported task instruction delivery")
     return InstructionEvidence(
-        sha256=_digest(obj.get("sha256"), "task.instruction.sha256"),
-        byte_length=_integer(
-            obj.get("byteLength"), "task.instruction.byteLength", minimum=1
-        ),
+        sha256=sha256_bytes(raw),
+        byte_length=len(raw),
         delivery_kind=kind,
         delivery_env=env,
         delivery_path=path,
@@ -400,12 +441,87 @@ def load_prepared_candidate_contract(
             "receipt execution-plan artifact differs from the executed bytes"
         )
 
-    bundle_digest = _digest(plan.get("bundleDigest"), "plan.bundleDigest")
+    run_cell = _contract_object(
+        plan.get("runCell"), "plan.runCell", kind=_RUN_CELL_KIND
+    )
+    _digest(run_cell.get("experimentDigest"), "plan.runCell.experimentDigest")
+    run_cell_digest = _digest(run_cell.get("digest"), "plan.runCell.digest")
+    if run_cell.get("arm") not in {"baseline", "candidate"}:
+        raise CandidateContractError("plan.runCell.arm is unsupported")
+    run_cell_seed = _safe_integer(run_cell.get("seed"), "plan.runCell.seed")
+    if sha256_bytes(
+        _canonical_bytes(
+            {key: value for key, value in run_cell.items() if key != "digest"},
+            "plan.runCell",
+        )
+    ) != run_cell_digest:
+        raise CandidateContractError("plan.runCell digest does not match")
+    bundle_digest = _digest(
+        run_cell.get("bundleDigest"), "plan.runCell.bundleDigest"
+    )
     if receipt.get("bundleDigest") != bundle_digest:
         raise CandidateContractError("receipt and plan bundle digests differ")
     execution_id = _string(plan.get("executionId"), "plan.executionId")
 
-    task = _object(plan.get("task"), "plan.task")
+    benchmark = _object(receipt.get("benchmark"), "receipt.benchmark")
+    suite, _, suite_digest = _embedded_contract(
+        benchmark.get("suite"),
+        "receipt.benchmark.suite",
+        kind=_BENCHMARK_SUITE_KIND,
+    )
+    task, _, task_digest = _embedded_contract(
+        benchmark.get("task"),
+        "receipt.benchmark.task",
+        kind=_BENCHMARK_TASK_KIND,
+    )
+    if (
+        run_cell.get("suiteDigest") != suite_digest
+        or run_cell.get("taskDigest") != task_digest
+    ):
+        raise CandidateContractError(
+            "run cell does not bind the receipt benchmark suite and task"
+        )
+    task_index = _integer(run_cell.get("taskIndex"), "plan.runCell.taskIndex")
+    repetition = _integer(run_cell.get("repetition"), "plan.runCell.repetition")
+    if suite.get("digestAlgorithm") != "rfc8785-sha256":
+        raise CandidateContractError(
+            "benchmark suite digestAlgorithm must equal rfc8785-sha256"
+        )
+    reps = _integer(suite.get("reps"), "benchmark suite reps", minimum=1)
+    task_digests = suite.get("taskDigests")
+    seeds = suite.get("seeds")
+    if (
+        not isinstance(task_digests, list)
+        or not task_digests
+        or len(set(task_digests)) != len(task_digests)
+    ):
+        raise CandidateContractError(
+            "benchmark suite taskDigests must be a non-empty unique array"
+        )
+    for index, value in enumerate(task_digests):
+        _digest(value, f"benchmark suite taskDigests[{index}]")
+    if (
+        not isinstance(seeds, list)
+        or len(seeds) != len(task_digests) * reps
+    ):
+        raise CandidateContractError(
+            "benchmark suite must provide one seed per task repetition"
+        )
+    for index, value in enumerate(seeds):
+        _safe_integer(value, f"benchmark suite seeds[{index}]")
+    if (
+        task_index >= len(task_digests)
+        or task_digests[task_index] != task_digest
+        or repetition >= reps
+        or seeds[task_index * reps + repetition] != run_cell_seed
+    ):
+        raise CandidateContractError(
+            "run cell coordinates do not match the receipt benchmark suite"
+        )
+    if task.get("digestAlgorithm") != "rfc8785-sha256":
+        raise CandidateContractError(
+            "benchmark task digestAlgorithm must equal rfc8785-sha256"
+        )
     outcome = _object(task.get("outcome"), "plan.task.outcome")
     if outcome.get("kind") != "workspace":
         raise CandidateContractError("Pier requires a workspace task outcome")
@@ -418,7 +534,9 @@ def load_prepared_candidate_contract(
     )
     if len(base_commit) != len(base_tree):
         raise CandidateContractError("task Git objects use different hash formats")
-    instruction = _instruction(task.get("instruction"))
+    instruction = _instruction(
+        task.get("instruction"), plan.get("instructionDelivery")
+    )
     task_files = _workspace_files(task.get("workspace"), "plan.task.workspace")
     if not task_files:
         raise CandidateContractError("task workspace cannot be empty")
@@ -480,16 +598,40 @@ def load_prepared_candidate_contract(
     if receipt.get("candidateWorkspace") != candidate_snapshot:
         raise CandidateContractError("receipt and plan candidate workspaces differ")
 
+    profile_activation = _contract_object(
+        receipt.get("profileActivation"),
+        "receipt.profileActivation",
+        kind=_PROFILE_ACTIVATION_KIND,
+    )
+    profile_activation_digest = _digest(
+        profile_activation.get("digest"),
+        "receipt.profileActivation.digest",
+    )
+    if sha256_bytes(
+        _canonical_bytes(
+            {
+                key: value
+                for key, value in profile_activation.items()
+                if key != "digest"
+            },
+            "receipt.profileActivation",
+        )
+    ) != profile_activation_digest:
+        raise CandidateContractError(
+            "profile activation digest does not match"
+        )
     profile_evidence = _contract_object(
-        receipt.get("profilePlan"),
-        "receipt.profilePlan",
+        profile_activation.get("profilePlan"),
+        "receipt.profileActivation.profilePlan",
         kind=_PROFILE_PLAN_KIND,
     )
     profile_digest = _digest(
-        profile_evidence.get("digest"), "receipt.profilePlan.digest"
+        profile_evidence.get("digest"),
+        "receipt.profileActivation.profilePlan.digest",
     )
     profile_raw = _embedded_bytes(
-        profile_evidence.get("artifact"), "receipt.profilePlan.artifact"
+        profile_evidence.get("artifact"),
+        "receipt.profileActivation.profilePlan.artifact",
     )
     if sha256_bytes(profile_raw) != profile_digest:
         raise CandidateContractError("profile artifact does not match its digest")
@@ -498,11 +640,50 @@ def load_prepared_candidate_contract(
     except (UnicodeDecodeError, json.JSONDecodeError) as exc:
         raise CandidateContractError("profile artifact is not UTF-8 JSON") from exc
     profile_material = _contract_object(
-        profile_evidence.get("material"), "receipt.profilePlan.material"
+        profile_evidence.get("material"),
+        "receipt.profileActivation.profilePlan.material",
     )
     if profile_artifact_material != profile_material:
         raise CandidateContractError("profile artifact and material differ")
+    if profile_material.get("harness") != plan.get("harness"):
+        raise CandidateContractError(
+            "profile plan harness differs from the execution plan"
+        )
     profile_files = _profile_files(profile_material)
+    activation_files = profile_activation.get("files")
+    if not isinstance(activation_files, list):
+        raise CandidateContractError(
+            "receipt.profileActivation.files must be an array"
+        )
+    activated: list[ProfileFile] = []
+    for index, value in enumerate(activation_files):
+        item = _object(value, f"receipt.profileActivation.files[{index}]")
+        path = _safe_relative(
+            item.get("path"), f"receipt.profileActivation.files[{index}].path"
+        )
+        mode = _integer(
+            item.get("mode"), f"receipt.profileActivation.files[{index}].mode"
+        )
+        content = item.get("content")
+        if (
+            mode > 0o777
+            or not isinstance(content, str)
+            or "\0" in content
+        ):
+            raise CandidateContractError(
+                f"receipt.profileActivation.files[{index}] is invalid"
+            )
+        activated.append(
+            ProfileFile(
+                path=path,
+                mode=mode,
+                sha256=sha256_bytes(content.encode("utf-8")),
+            )
+        )
+    if activated != list(profile_files):
+        raise CandidateContractError(
+            "profile activation files differ from the signed profile plan"
+        )
     profile_application = _object(plan.get("profile"), "plan.profile")
     if profile_application.get("planDigest") != profile_digest:
         raise CandidateContractError("plan does not bind the profile digest")
@@ -563,6 +744,10 @@ def load_prepared_candidate_contract(
         raise CandidateContractError("launch cwd names an unavailable workspace")
     cwd_path = _safe_relative(cwd.get("path"), "plan.launch.cwd.path", allow_dot=True)
     limits = _object(plan.get("limits"), "plan.limits")
+    if limits != task.get("limits"):
+        raise CandidateContractError(
+            "execution plan limits differ from the benchmark task"
+        )
     timeout_ms = _integer(limits.get("timeoutMs"), "plan.limits.timeoutMs", minimum=1)
     _integer(limits.get("maxSteps"), "plan.limits.maxSteps", minimum=1)
     max_model_calls = _integer(
@@ -571,11 +756,27 @@ def load_prepared_candidate_contract(
     _integer(limits.get("maxInputTokens"), "plan.limits.maxInputTokens")
     _integer(limits.get("maxOutputTokens"), "plan.limits.maxOutputTokens")
     _number(limits.get("maxCostUsd"), "plan.limits.maxCostUsd")
-    attempt = _object(plan.get("attempt"), "plan.attempt")
-    _integer(attempt.get("number"), "plan.attempt.number", minimum=1)
-    _integer(attempt.get("maxAttempts"), "plan.attempt.maxAttempts", minimum=1)
-    if attempt.get("retryPolicy") != "none":
-        raise CandidateContractError("Pier execution requires runtime-owned retries")
+    attempt = _object(task.get("attempt"), "benchmark task attempt")
+    attempt_number = _integer(
+        run_cell.get("attempt"), "plan.runCell.attempt", minimum=1
+    )
+    max_attempts = _integer(
+        attempt.get("maxAttempts"), "benchmark task attempt.maxAttempts", minimum=1
+    )
+    if attempt_number > max_attempts:
+        raise CandidateContractError(
+            "run cell attempt exceeds the benchmark task maximum"
+        )
+    retry_policy = attempt.get("retryPolicy")
+    if retry_policy not in {
+        "none",
+        "pre-model-infrastructure-only",
+    }:
+        raise CandidateContractError("benchmark task retry policy is unsupported")
+    if retry_policy == "none" and max_attempts != 1:
+        raise CandidateContractError(
+            "a no-retry benchmark task must allow exactly one attempt"
+        )
     container = _object(plan.get("container"), "plan.container")
     if receipt.get("container") != container:
         raise CandidateContractError("receipt and plan container identities differ")
@@ -586,11 +787,21 @@ def load_prepared_candidate_contract(
         raise CandidateContractError(
             "container image must be an unpinned OCI reference without credentials"
         )
-    if container.get("source") not in {
+    container_source = container.get("source")
+    if container_source not in {
         "pinned-container",
         "evaluator-task-container",
     }:
         raise CandidateContractError("plan container source is unsupported")
+    task_container = task.get("evaluatorTaskContainer")
+    if container_source == "evaluator-task-container" and task_container != container:
+        raise CandidateContractError(
+            "execution plan container differs from the benchmark task container"
+        )
+    if container_source == "pinned-container" and task_container is not None:
+        raise CandidateContractError(
+            "pinned execution cannot override a benchmark task container"
+        )
     platform = _object(container.get("platform"), "plan.container.platform")
     if platform.get("os") != "linux" or platform.get("architecture") not in {
         "amd64",
@@ -605,7 +816,10 @@ def load_prepared_candidate_contract(
     _string(resolved_model.get("provider"), "plan.model.resolved.provider")
     _string(resolved_model.get("model"), "plan.model.resolved.model")
     _string(resolved_model.get("snapshot"), "plan.model.resolved.snapshot")
-    if receipt.get("resolvedModel") != resolved_model:
+    if (
+        receipt.get("resolvedModel") != resolved_model
+        or task.get("model") != resolved_model
+    ):
         raise CandidateContractError("receipt and plan resolved models differ")
     model_access = _object(model.get("access"), "plan.model.access")
     if set(model_access) != {"kind", "grantDigest", "network"}:

package/pier_agents/tangle_candidate.py

@@ -55,6 +55,7 @@ _CANDIDATE_HOME = "/tangle/home"
 _CANDIDATE_TMP = "/tangle/tmp"
 _CONTROL_ROOT = "/tangle/control"
 _FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+_FIXED_PATH_ENTRIES = frozenset(_FIXED_PATH.split(":"))
 _RESERVED_PROCESS_ENV = {
     "GIT_CONFIG_COUNT",
     "GIT_CONFIG_KEY_0",
@@ -124,6 +125,33 @@ def _trace_env(
     }
 
 
+def _evaluator_git(root: str) -> str:
+    return shlex.join(
+        [
+            "git",
+            "-c",
+            "core.hooksPath=/dev/null",
+            "-c",
+            "core.fsmonitor=false",
+            "-c",
+            "core.untrackedCache=false",
+            "-c",
+            "protocol.file.allow=never",
+            "-C",
+            root,
+        ]
+    )
+
+
+def _evaluator_git_config(repository_base_commit: str) -> bytes:
+    if len(repository_base_commit) == 64:
+        return (
+            b"[core]\n\trepositoryformatversion = 1\n"
+            b"[extensions]\n\tobjectformat = sha256\n"
+        )
+    return b"[core]\n\trepositoryformatversion = 0\n"
+
+
 class TangleCandidateAgent(BaseAgent):
     """Pier adapter for a branded ``PreparedAgentCandidateExecution``."""
 
@@ -241,12 +269,25 @@ class TangleCandidateAgent(BaseAgent):
     ) -> dict[str, str]:
         protected: dict[str, str] = {}
         public_names = set(self._contract.env)
-        reserved_public = sorted(public_names & _RESERVED_PROCESS_ENV)
+        reserved_public = sorted(
+            (public_names & _RESERVED_PROCESS_ENV) - {"PATH"}
+        )
         if reserved_public:
             raise PierCandidateError(
                 "signed public env collides with evaluator Git safety fields: "
                 + ", ".join(reserved_public)
             )
+        public_path = self._contract.env.get("PATH")
+        if public_path is not None:
+            entries = public_path.split(":")
+            if (
+                not entries
+                or len(entries) != len(set(entries))
+                or any(entry not in _FIXED_PATH_ENTRIES for entry in entries)
+            ):
+                raise PierCandidateError(
+                    "signed public PATH contains an empty, duplicate, or untrusted directory"
+                )
         expected_trace = _trace_env(self._contract, trace_run_id)
         for name, expected in expected_trace.items():
             if values.get(name) != expected:
@@ -455,7 +496,7 @@ class TangleCandidateAgent(BaseAgent):
             return ()
         originals: list[OriginalProfileFile] = []
         root = self._contract.task_root
-        git = f"git -c core.hooksPath=/dev/null -C {shlex.quote(root)}"
+        git = _evaluator_git(root)
         for file in self._contract.profile_files:
             target = f"{root}/{file.path}"
             quoted = shlex.quote(target)
@@ -674,14 +715,38 @@ class TangleCandidateAgent(BaseAgent):
             user="root",
         )
 
+    async def _install_evaluator_git_config(
+        self, environment: BaseEnvironment
+    ) -> None:
+        config = _evaluator_git_config(self._contract.repository_base_commit)
+        target = f"{self._contract.task_root}/.git/config"
+        with tempfile.NamedTemporaryFile(delete=False) as handle:
+            handle.write(config)
+            local_path = Path(handle.name)
+        try:
+            await self._write_verified_file(
+                environment,
+                local_path,
+                target,
+                anchor=self._contract.task_root,
+                mode=0o600,
+                digest=sha256_bytes(config),
+                byte_length=len(config),
+            )
+        finally:
+            local_path.unlink(missing_ok=True)
+
     async def _restore_profile_and_capture_solution(
         self, environment: BaseEnvironment
     ) -> None:
         root = self._contract.task_root
-        quoted_root = shlex.quote(root)
-        git = f"git -c core.hooksPath=/dev/null -C {quoted_root}"
+        git = _evaluator_git(root)
         git_env = candidate_git_env(root)
 
+        # Repository-local config is candidate-owned after execution. Replace it
+        # before root invokes Git so arbitrary filters and other programs cannot run.
+        await self._install_evaluator_git_config(environment)
+
         # Ignore candidate-authored commits and derive one evaluator-owned final
         # tree from the actual working files relative to the signed base commit.
         await self._exec(
@@ -718,7 +783,12 @@ class TangleCandidateAgent(BaseAgent):
                 await self._ensure_real_directory(environment, parent, anchor=root)
                 await self._exec(environment, f"rm -rf -- {quoted}", user="root")
 
-        await self._exec(environment, f"{git} add -A -- .", env=git_env, user="root")
+        await self._exec(
+            environment,
+            f"{git} add -A -- . {shlex.quote(':(exclude).sidecar')}",
+            env=git_env,
+            user="root",
+        )
         for original in self._original_profile_files:
             if original.tracked:
                 continue

package/scripts/verify-packed-consumer.mjs

@@ -54,9 +54,54 @@ try {
   const packDir = path.join(scratch, 'pack')
   const runtimePackDir = path.join(scratch, 'runtime-pack')
   const consumerDir = path.join(scratch, 'consumer')
+  const terminalBenchVenv = path.join(scratch, 'terminal-bench-venv')
+  const terminalBenchBinDir = path.join(terminalBenchVenv, 'bin')
   await mkdir(packDir)
   await mkdir(runtimePackDir)
   await mkdir(consumerDir)
+  await mkdir(terminalBenchBinDir, { recursive: true })
+  await writeFile(path.join(terminalBenchVenv, 'package.json'), '{"type":"module"}\n')
+  await writeFile(
+    path.join(terminalBenchBinDir, 'python'),
+    String.raw`#!/usr/bin/env node
+const rows = [{
+  id: 'installed-absolute-venv',
+  instruction: 'prove the installed path',
+  task_dir: '/tmp/terminal-bench-task',
+  solution: 'echo ok\n',
+}]
+process.stdout.write(JSON.stringify(rows) + '\n')
+`,
+    { mode: 0o755 },
+  )
+  await writeFile(
+    path.join(terminalBenchBinDir, 'tb'),
+    String.raw`#!/usr/bin/env node
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+const args = process.argv.slice(2)
+function value(flag) {
+  const index = args.indexOf(flag)
+  if (index === -1 || index + 1 >= args.length) throw new Error('missing ' + flag)
+  return args[index + 1]
+}
+
+const outputPath = value('--output-path')
+const runId = value('--run-id')
+const taskId = value('-t')
+const reportDir = join(outputPath, runId)
+mkdirSync(reportDir, { recursive: true })
+writeFileSync(
+  join(reportDir, 'results.json'),
+  JSON.stringify({
+    resolved_ids: [taskId],
+    results: [{ task_id: taskId, is_resolved: true, parser_results: { installedVenv: true } }],
+  }),
+)
+`,
+    { mode: 0o755 },
+  )
 
   // Build explicitly so verification cannot inherit a machine-level
   // ignore-scripts setting and accidentally pack stale or missing output.
@@ -122,6 +167,21 @@ try {
     path.join(consumerDir, 'index.mjs'),
     "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
   )
+  await writeFile(
+    path.join(consumerDir, 'terminal-bench-absolute-venv.mjs'),
+    `import { createTerminalBenchAdapter } from '@tangle-network/agent-bench/benchmarks/terminal-bench'
+
+const adapter = createTerminalBenchAdapter()
+const tasks = await adapter.loadTasks({ ids: ['installed-absolute-venv'] })
+if (tasks.length !== 1 || tasks[0]?.id !== 'installed-absolute-venv') {
+  throw new Error('packed Terminal-Bench adapter did not load through the absolute venv')
+}
+const score = await adapter.judge(tasks[0], 'echo installed-package-path')
+if (!score.resolved || score.score !== 1) {
+  throw new Error(\`packed Terminal-Bench adapter did not judge through the absolute venv: \${JSON.stringify(score)}\`)
+}
+`,
+  )
   await writeFile(
     path.join(consumerDir, 'tsconfig.json'),
     `${JSON.stringify(
@@ -165,6 +225,10 @@ for name in sorted(expected):
     consumerDir,
   )
   await run('node', ['index.mjs'], consumerDir)
+  await run('node', ['terminal-bench-absolute-venv.mjs'], consumerDir, {
+    ...process.env,
+    TERMINAL_BENCH_VENV: terminalBenchVenv,
+  })
   await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir)
   const typescript5 = await run('npm', ['exec', '--', 'tsc', '--version'], consumerDir)
   if (typescript5.stdout.trim() !== `Version ${TYPESCRIPT_5}`) {
@@ -228,7 +292,7 @@ for name in sorted(expected):
     })
   }
   console.log(
-    `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}; prepared ${prepareProof.executionPlanDigest}`,
+    `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}, absolute Terminal-Bench venv; prepared ${prepareProof.executionPlanDigest}`,
   )
 } finally {
   await rm(scratch, { recursive: true, force: true })

package/scripts/verify-pier-agent.mts

@@ -435,7 +435,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
     workspace: taskWorkspace,
     evaluatorTaskContainer: container,
     limits: {
-      timeoutMs: 60_000,
+      timeoutMs: 180_000,
       maxSteps: 8,
       maxModelCalls: 0,
       maxInputTokens: 0,
@@ -671,12 +671,14 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
     )
   }
   assertTreeOmits(scratch, 'zero-model-proof')
-  const usage = finalized.receipt.value.usage
+  const usage = finalized.receipt.value.modelSettlement.material.usage
   if (
     usage.modelCalls !== 0 ||
     usage.inputTokens !== 0 ||
     usage.outputTokens !== 0 ||
-    usage.costUsd !== 0 ||
+    usage.cachedInputTokens !== 0 ||
+    usage.reasoningTokens !== 0 ||
+    usage.costUsdNanos !== 0 ||
     finalized.receipt.value.trace.modelCallCount !== 0
   ) {
     throw new Error(`runtime minted nonzero usage for a zero-model run: ${JSON.stringify(usage)}`)
@@ -691,7 +693,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
         modelCalls: usage.modelCalls,
         inputTokens: usage.inputTokens,
         outputTokens: usage.outputTokens,
-        costUsd: usage.costUsd,
+        costUsd: usage.costUsdNanos / 1_000_000_000,
         pierContextUsage: null,
         profileExcludedByVerifier: true,
         container: {

package/src/benchmarks/_harness.test.mts

@@ -4,7 +4,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
 import test from 'node:test'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import { resolveBenchPython, runStagedJudge } from './_harness'
+import {
+  benchRoot,
+  resolveBenchPython,
+  runStagedJudge,
+  venvBin,
+  venvBinAt,
+} from './_harness'
 
 const digest = (bytes: Uint8Array): `sha256:${string}` =>
   `sha256:${createHash('sha256').update(bytes).digest('hex')}`
@@ -31,6 +37,15 @@ test('resolveBenchPython rejects relative and empty interpreter paths', () => {
   )
 })
 
+test('venv executable paths support package-owned and external environments', () => {
+  assert.equal(venvBinAt('/srv/terminal-bench', 'tb'), join('/srv/terminal-bench', 'bin', 'tb'))
+  assert.equal(
+    venvBinAt('.venv-commit0', 'python'),
+    join(benchRoot, '.venv-commit0', 'bin', 'python'),
+  )
+  assert.equal(venvBin('python'), venvBinAt('.venv', 'python'))
+})
+
 test('runStagedJudge terminates a hung evaluator at its explicit timeout', async () => {
   const started = Date.now()
   await assert.rejects(