@tangle-network/agent-bench 0.1.0 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/HARNESS.md +302 -0
- package/README.md +39 -2
- package/dist/adapters.d.ts +15 -0
- package/dist/adapters.js +43 -0
- package/dist/adapters.js.map +1 -0
- package/dist/benchmarks/_harness.d.ts +125 -0
- package/dist/benchmarks/_harness.js +33 -0
- package/dist/benchmarks/_harness.js.map +1 -0
- package/dist/benchmarks/aec-bench.d.ts +27 -0
- package/dist/benchmarks/aec-bench.js +8 -0
- package/dist/benchmarks/aec-bench.js.map +1 -0
- package/dist/benchmarks/agentbench.d.ts +16 -0
- package/dist/benchmarks/agentbench.js +10 -0
- package/dist/benchmarks/agentbench.js.map +1 -0
- package/dist/benchmarks/appworld.d.ts +37 -0
- package/dist/benchmarks/appworld.js +14 -0
- package/dist/benchmarks/appworld.js.map +1 -0
- package/dist/benchmarks/bfcl.d.ts +18 -0
- package/dist/benchmarks/bfcl.js +10 -0
- package/dist/benchmarks/bfcl.js.map +1 -0
- package/dist/benchmarks/cad-design.d.ts +45 -0
- package/dist/benchmarks/cad-design.js +7 -0
- package/dist/benchmarks/cad-design.js.map +1 -0
- package/dist/benchmarks/cadbench.d.ts +19 -0
- package/dist/benchmarks/cadbench.js +8 -0
- package/dist/benchmarks/cadbench.js.map +1 -0
- package/dist/benchmarks/cadgenbench.d.ts +22 -0
- package/dist/benchmarks/cadgenbench.js +8 -0
- package/dist/benchmarks/cadgenbench.js.map +1 -0
- package/dist/benchmarks/commit0.d.ts +31 -0
- package/dist/benchmarks/commit0.js +10 -0
- package/dist/benchmarks/commit0.js.map +1 -0
- package/dist/benchmarks/crag.d.ts +14 -0
- package/dist/benchmarks/crag.js +9 -0
- package/dist/benchmarks/crag.js.map +1 -0
- package/dist/benchmarks/dabstep.d.ts +18 -0
- package/dist/benchmarks/dabstep.js +10 -0
- package/dist/benchmarks/dabstep.js.map +1 -0
- package/dist/benchmarks/enterpriseops-gym.d.ts +38 -0
- package/dist/benchmarks/enterpriseops-gym.js +10 -0
- package/dist/benchmarks/enterpriseops-gym.js.map +1 -0
- package/dist/benchmarks/finresearchbench.d.ts +15 -0
- package/dist/benchmarks/finresearchbench.js +8 -0
- package/dist/benchmarks/finresearchbench.js.map +1 -0
- package/dist/benchmarks/finsearchcomp.d.ts +49 -0
- package/dist/benchmarks/finsearchcomp.js +7 -0
- package/dist/benchmarks/finsearchcomp.js.map +1 -0
- package/dist/benchmarks/frames.d.ts +59 -0
- package/dist/benchmarks/frames.js +13 -0
- package/dist/benchmarks/frames.js.map +1 -0
- package/dist/benchmarks/hotpotqa.d.ts +48 -0
- package/dist/benchmarks/hotpotqa.js +15 -0
- package/dist/benchmarks/hotpotqa.js.map +1 -0
- package/dist/benchmarks/humaneval.d.ts +53 -0
- package/dist/benchmarks/humaneval.js +15 -0
- package/dist/benchmarks/humaneval.js.map +1 -0
- package/dist/benchmarks/mind2web.d.ts +41 -0
- package/dist/benchmarks/mind2web.js +9 -0
- package/dist/benchmarks/mind2web.js.map +1 -0
- package/dist/benchmarks/nomiracl.d.ts +15 -0
- package/dist/benchmarks/nomiracl.js +9 -0
- package/dist/benchmarks/nomiracl.js.map +1 -0
- package/dist/benchmarks/open-rag-bench.d.ts +14 -0
- package/dist/benchmarks/open-rag-bench.js +9 -0
- package/dist/benchmarks/open-rag-bench.js.map +1 -0
- package/dist/benchmarks/programbench.d.ts +38 -0
- package/dist/benchmarks/programbench.js +10 -0
- package/dist/benchmarks/programbench.js.map +1 -0
- package/dist/benchmarks/rag-shared.d.ts +42 -0
- package/dist/benchmarks/rag-shared.js +39 -0
- package/dist/benchmarks/rag-shared.js.map +1 -0
- package/dist/benchmarks/ragbench.d.ts +16 -0
- package/dist/benchmarks/ragbench.js +9 -0
- package/dist/benchmarks/ragbench.js.map +1 -0
- package/dist/benchmarks/simpleqa.d.ts +64 -0
- package/dist/benchmarks/simpleqa.js +11 -0
- package/dist/benchmarks/simpleqa.js.map +1 -0
- package/dist/benchmarks/swe-bench.d.ts +56 -0
- package/dist/benchmarks/swe-bench.js +14 -0
- package/dist/benchmarks/swe-bench.js.map +1 -0
- package/dist/benchmarks/t2-ragbench.d.ts +14 -0
- package/dist/benchmarks/t2-ragbench.js +9 -0
- package/dist/benchmarks/t2-ragbench.js.map +1 -0
- package/dist/benchmarks/tau-bench-shared.d.ts +26 -0
- package/dist/benchmarks/tau-bench-shared.js +10 -0
- package/dist/benchmarks/tau-bench-shared.js.map +1 -0
- package/dist/benchmarks/tau2-bench.d.ts +7 -0
- package/dist/benchmarks/tau2-bench.js +11 -0
- package/dist/benchmarks/tau2-bench.js.map +1 -0
- package/dist/benchmarks/tau3-banking.d.ts +15 -0
- package/dist/benchmarks/tau3-banking.js +9 -0
- package/dist/benchmarks/tau3-banking.js.map +1 -0
- package/dist/benchmarks/terminal-bench.d.ts +24 -0
- package/dist/benchmarks/terminal-bench.js +8 -0
- package/dist/benchmarks/terminal-bench.js.map +1 -0
- package/dist/benchmarks/toollm.d.ts +16 -0
- package/dist/benchmarks/toollm.js +10 -0
- package/dist/benchmarks/toollm.js.map +1 -0
- package/dist/benchmarks/trata-hedge.d.ts +32 -0
- package/dist/benchmarks/trata-hedge.js +7 -0
- package/dist/benchmarks/trata-hedge.js.map +1 -0
- package/dist/benchmarks/types.d.ts +107 -0
- package/dist/benchmarks/types.js +1 -0
- package/dist/benchmarks/types.js.map +1 -0
- package/dist/benchmarks/webarena-verified.d.ts +16 -0
- package/dist/benchmarks/webarena-verified.js +10 -0
- package/dist/benchmarks/webarena-verified.js.map +1 -0
- package/dist/chunk-2PVVP7GN.js +197 -0
- package/dist/chunk-2PVVP7GN.js.map +1 -0
- package/dist/chunk-2XU6OGEN.js +170 -0
- package/dist/chunk-2XU6OGEN.js.map +1 -0
- package/dist/chunk-53UPUNBZ.js +325 -0
- package/dist/chunk-53UPUNBZ.js.map +1 -0
- package/dist/chunk-5SBJCB6W.js +144 -0
- package/dist/chunk-5SBJCB6W.js.map +1 -0
- package/dist/chunk-7WSD27QQ.js +118 -0
- package/dist/chunk-7WSD27QQ.js.map +1 -0
- package/dist/chunk-C7T7WEK2.js +103 -0
- package/dist/chunk-C7T7WEK2.js.map +1 -0
- package/dist/chunk-CKUVRZ2T.js +251 -0
- package/dist/chunk-CKUVRZ2T.js.map +1 -0
- package/dist/chunk-HBSWHQNJ.js +30 -0
- package/dist/chunk-HBSWHQNJ.js.map +1 -0
- package/dist/chunk-HHXFIHXC.js +116 -0
- package/dist/chunk-HHXFIHXC.js.map +1 -0
- package/dist/chunk-IFAV6KEM.js +276 -0
- package/dist/chunk-IFAV6KEM.js.map +1 -0
- package/dist/chunk-INNOYXCP.js +387 -0
- package/dist/chunk-INNOYXCP.js.map +1 -0
- package/dist/chunk-J3KDJNX2.js +182 -0
- package/dist/chunk-J3KDJNX2.js.map +1 -0
- package/dist/chunk-JRWWGMK7.js +148 -0
- package/dist/chunk-JRWWGMK7.js.map +1 -0
- package/dist/chunk-JTHWEDEW.js +32 -0
- package/dist/chunk-JTHWEDEW.js.map +1 -0
- package/dist/chunk-KDIKRJGB.js +120 -0
- package/dist/chunk-KDIKRJGB.js.map +1 -0
- package/dist/chunk-LRRD7NAG.js +301 -0
- package/dist/chunk-LRRD7NAG.js.map +1 -0
- package/dist/chunk-ODT47UAY.js +221 -0
- package/dist/chunk-ODT47UAY.js.map +1 -0
- package/dist/chunk-PA2ZKHJC.js +230 -0
- package/dist/chunk-PA2ZKHJC.js.map +1 -0
- package/dist/chunk-PPYSEKFM.js +182 -0
- package/dist/chunk-PPYSEKFM.js.map +1 -0
- package/dist/chunk-PUIRNYI7.js +189 -0
- package/dist/chunk-PUIRNYI7.js.map +1 -0
- package/dist/chunk-R36V2VP7.js +169 -0
- package/dist/chunk-R36V2VP7.js.map +1 -0
- package/dist/chunk-R67DFVLO.js +142 -0
- package/dist/chunk-R67DFVLO.js.map +1 -0
- package/dist/chunk-SEVJPLZC.js +260 -0
- package/dist/chunk-SEVJPLZC.js.map +1 -0
- package/dist/chunk-SYDW647C.js +318 -0
- package/dist/chunk-SYDW647C.js.map +1 -0
- package/dist/chunk-TBKU5XQI.js +228 -0
- package/dist/chunk-TBKU5XQI.js.map +1 -0
- package/dist/chunk-TSWPNOYM.js +147 -0
- package/dist/chunk-TSWPNOYM.js.map +1 -0
- package/dist/chunk-UAIOHCUK.js +27 -0
- package/dist/chunk-UAIOHCUK.js.map +1 -0
- package/dist/chunk-UPAMRDX4.js +233 -0
- package/dist/chunk-UPAMRDX4.js.map +1 -0
- package/dist/chunk-VQRS7VUC.js +342 -0
- package/dist/chunk-VQRS7VUC.js.map +1 -0
- package/dist/chunk-X3BTXCJ4.js +262 -0
- package/dist/chunk-X3BTXCJ4.js.map +1 -0
- package/dist/chunk-X5YKXC6V.js +211 -0
- package/dist/chunk-X5YKXC6V.js.map +1 -0
- package/dist/chunk-Y6O2OCUO.js +130 -0
- package/dist/chunk-Y6O2OCUO.js.map +1 -0
- package/dist/chunk-YCGY7UIZ.js +208 -0
- package/dist/chunk-YCGY7UIZ.js.map +1 -0
- package/dist/chunk-Z7ML6L77.js +162 -0
- package/dist/chunk-Z7ML6L77.js.map +1 -0
- package/dist/chunk-ZEWMTR5M.js +136 -0
- package/dist/chunk-ZEWMTR5M.js.map +1 -0
- package/dist/index.d.ts +355 -0
- package/dist/index.js +1908 -0
- package/dist/index.js.map +1 -0
- package/fixtures/aec-bench.json +18 -0
- package/fixtures/agentbench-dbbench.json +22 -0
- package/fixtures/bfcl.json +45 -0
- package/fixtures/commit0.json +72 -0
- package/fixtures/crag.json +10 -0
- package/fixtures/dabstep.json +22 -0
- package/fixtures/enterpriseops-gym.json +103 -0
- package/fixtures/finresearchbench.json +21 -0
- package/fixtures/finsearchcomp.json +66 -0
- package/fixtures/frames.json +26 -0
- package/fixtures/hotpotqa.json +182 -0
- package/fixtures/nomiracl.json +26 -0
- package/fixtures/open-rag-bench.json +16 -0
- package/fixtures/pier-agent/no-model-task/environment/Dockerfile +16 -0
- package/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt +1 -0
- package/fixtures/pier-agent/no-model-task/instruction.md +6 -0
- package/fixtures/pier-agent/no-model-task/pre_artifacts.sh +6 -0
- package/fixtures/pier-agent/no-model-task/task.toml +35 -0
- package/fixtures/pier-agent/no-model-task/tests/Dockerfile +17 -0
- package/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt +1 -0
- package/fixtures/pier-agent/no-model-task/tests/test.sh +19 -0
- package/fixtures/programbench.json +17 -0
- package/fixtures/ragbench.json +21 -0
- package/fixtures/simpleqa.json +121 -0
- package/fixtures/t2-ragbench.json +13 -0
- package/fixtures/tau2-bench.json +16 -0
- package/fixtures/tau3-banking.json +16 -0
- package/fixtures/toollm.json +28 -0
- package/fixtures/webarena-verified.json +20 -0
- package/package.json +60 -20
- package/pier_agents/__init__.py +18 -0
- package/pier_agents/candidate_contract.py +755 -0
- package/pier_agents/process_boundary.py +321 -0
- package/pier_agents/tangle_candidate.py +907 -0
- package/pier_agents/workspace_boundary.py +368 -0
- package/scripts/appworld_driver.py +359 -0
- package/scripts/cadbench_prepare.py +22 -0
- package/scripts/cadgenbench_hard_parts.py +48 -0
- package/scripts/clbench_codebase_judge.py +73 -0
- package/scripts/commit0_judge.py +170 -0
- package/scripts/dabstep_judge.py +42 -0
- package/scripts/enterpriseops_gym_judge.py +281 -0
- package/scripts/programbench_judge.py +120 -0
- package/scripts/render-gate-chart.mjs +176 -0
- package/scripts/run-package-tests.mjs +56 -0
- package/scripts/terminate-pier-trial.mts +66 -0
- package/scripts/trata-hedge/README.md +56 -0
- package/scripts/trata-hedge/run.sh +60 -0
- package/scripts/trata-hedge/solve.py +83 -0
- package/scripts/verify-packed-consumer.mjs +235 -0
- package/scripts/verify-pier-agent.mts +715 -0
- package/scripts/verify-pier-pair.mts +74 -0
- package/scripts/verify-pier-recovery.mts +139 -0
- package/src/adapters.ts +26 -0
- package/src/benchmarks/_harness.test.mts +178 -0
- package/src/benchmarks/_harness.ts +239 -16
- package/src/benchmarks/agentbench.ts +163 -0
- package/src/benchmarks/appworld.test.mts +15 -9
- package/src/benchmarks/bfcl.ts +346 -0
- package/src/benchmarks/crag.ts +137 -0
- package/src/benchmarks/dabstep.test.mts +70 -0
- package/src/benchmarks/dabstep.ts +212 -0
- package/src/benchmarks/external-adapters.test.mts +150 -0
- package/src/benchmarks/finresearchbench.ts +269 -0
- package/src/benchmarks/humaneval.ts +20 -8
- package/src/benchmarks/nomiracl.ts +180 -0
- package/src/benchmarks/open-rag-bench.ts +153 -0
- package/src/benchmarks/rag-benchmarks.test.mts +138 -0
- package/src/benchmarks/rag-shared.ts +327 -0
- package/src/benchmarks/ragbench.ts +171 -0
- package/src/benchmarks/swe-bench.test.mts +61 -0
- package/src/benchmarks/swe-bench.ts +201 -19
- package/src/benchmarks/t2-ragbench.ts +166 -0
- package/src/benchmarks/tau-bench-shared.ts +214 -0
- package/src/benchmarks/tau2-bench.ts +30 -0
- package/src/benchmarks/tau3-banking.ts +29 -0
- package/src/benchmarks/terminal-bench.test.mts +33 -0
- package/src/benchmarks/terminal-bench.ts +23 -8
- package/src/benchmarks/toollm.ts +254 -0
- package/src/benchmarks/types.ts +42 -0
- package/src/benchmarks/webarena-verified.ts +200 -0
- package/src/commit0-prereqs.sh +0 -0
- package/src/coordination-mcp-container-reach.mts +181 -0
- package/src/decoder-live.mts +1 -1
- package/src/examples/README.md +103 -39
- package/src/examples/benchmark-matrix.mts +101 -0
- package/src/examples/lean-proof-gate.README.md +77 -0
- package/src/examples/lean-proof-gate.mts +162 -0
- package/src/examples/lean-verify.ts +95 -0
- package/src/examples/lean.Dockerfile +12 -0
- package/src/examples/math-demo.mts +9 -7
- package/src/examples/strategy-demo.mts +10 -12
- package/src/gate.ts +3 -2
- package/src/hev-eval.mts +69 -0
- package/src/hev-improve.mts +169 -0
- package/src/hev-structural.mts +688 -0
- package/src/index.ts +73 -0
- package/src/mbpp-structural.mts +662 -0
- package/src/pier-agent.test-fixtures.mts +19 -0
- package/src/pier-agent.test.mts +363 -0
- package/src/pier-agent.ts +657 -0
- package/src/pier-result-grader.mjs +30 -0
- package/src/pier-result-grader.test.mts +62 -0
- package/src/pier-result-grader.ts +108 -0
- package/src/pier-task-outcome.test.mts +117 -0
- package/src/pier-task-outcome.ts +240 -0
- package/src/pier-trial-controller.test.mts +412 -0
- package/src/pier-trial-controller.ts +858 -0
- package/src/pier-trial-supervisor.mjs +352 -0
- package/src/resolve-client.ts +25 -2
- package/src/run-benchmarks-cli.mts +72 -0
- package/src/run-benchmarks-report.ts +66 -0
- package/src/run-benchmarks.test.mts +231 -0
- package/src/run-benchmarks.ts +589 -0
- package/src/smoke-structural-rollout.mts +393 -0
- package/src/swe-bench-env.test.ts +207 -0
- package/src/swe-bench-env.ts +554 -0
- package/src/swe-jail.ts +293 -0
- package/src/swe-self-improve.mts +84 -0
- package/src/swe-structural-judge-policy.test.ts +117 -0
- package/src/swe-structural-judge-policy.ts +133 -0
- package/src/swe-structural-policy.test.ts +124 -0
- package/src/swe-structural-policy.ts +132 -0
- package/src/swe-structural-provenance.test.ts +93 -0
- package/src/swe-structural-provenance.ts +138 -0
- package/src/swe-structural.mts +1260 -0
- package/src/swe-temp.ts +14 -0
- package/src/tb-container-executor.mts +234 -0
- package/src/tb-container-executor.test.mts +99 -0
- package/src/tb-supervisor-sidecar.mts +222 -0
- package/src/trata-gepa.mts +1 -1
- package/steerers/eops-itsm-population.json +1 -0
- package/tb_agents/opencode_refine_agent.py +117 -0
- package/tb_agents/opencode_router_agent.py +406 -0
- package/tb_agents/opencode_supervisor_agent.py +239 -0
- package/tb_agents/script_agent.py +66 -0
|
@@ -0,0 +1,907 @@
|
|
|
1
|
+
"""Execute one runtime-prepared candidate inside a Pier task container.
|
|
2
|
+
|
|
3
|
+
Pier owns the task container, patch transfer, and verifier. Agent Runtime owns
|
|
4
|
+
candidate verification, exact execution bytes, model access, and trace usage.
|
|
5
|
+
This bridge only rechecks the prepared bytes, copies their signed workspaces,
|
|
6
|
+
delivers the exact task instruction, and reports truthful process termination.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import base64
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import secrets
|
|
18
|
+
import shlex
|
|
19
|
+
import tempfile
|
|
20
|
+
import time
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from importlib.metadata import PackageNotFoundError, version as package_version
|
|
23
|
+
from pathlib import Path, PurePosixPath
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from pier.agents.base import BaseAgent
|
|
27
|
+
from pier.agents.installed.base import NonZeroAgentExitCodeError
|
|
28
|
+
from pier.environments.base import BaseEnvironment
|
|
29
|
+
from pier.models.agent.context import AgentContext
|
|
30
|
+
from pier.models.agent.network import NetworkAllowlist
|
|
31
|
+
|
|
32
|
+
from .candidate_contract import (
|
|
33
|
+
CandidateContractError,
|
|
34
|
+
PreparedCandidateContract,
|
|
35
|
+
immutable_snapshot,
|
|
36
|
+
load_prepared_candidate_contract,
|
|
37
|
+
sha256_bytes,
|
|
38
|
+
)
|
|
39
|
+
from .process_boundary import CandidateProcessBoundary, ProcessBoundaryConfig
|
|
40
|
+
from .workspace_boundary import (
|
|
41
|
+
CandidateWorkspaceBoundary,
|
|
42
|
+
WorkspaceBoundaryConfig,
|
|
43
|
+
candidate_git_env,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
_ADAPTER_VERSION = "2.0.0"
|
|
48
|
+
_ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
49
|
+
_PROTECTED_TASK_PATH = "/tangle/input/stdin.txt"
|
|
50
|
+
_PIER_PROTECTED_ROOTS = ("/logs", "/tests", "/solution", "/tangle")
|
|
51
|
+
_CANDIDATE_UID = 65532
|
|
52
|
+
_CANDIDATE_GID = 65532
|
|
53
|
+
_EVALUATOR_ROOT = "/tangle"
|
|
54
|
+
_CANDIDATE_HOME = "/tangle/home"
|
|
55
|
+
_CANDIDATE_TMP = "/tangle/tmp"
|
|
56
|
+
_CONTROL_ROOT = "/tangle/control"
|
|
57
|
+
_FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
58
|
+
_RESERVED_PROCESS_ENV = {
|
|
59
|
+
"GIT_CONFIG_COUNT",
|
|
60
|
+
"GIT_CONFIG_KEY_0",
|
|
61
|
+
"GIT_CONFIG_VALUE_0",
|
|
62
|
+
"GIT_CONFIG_NOSYSTEM",
|
|
63
|
+
"GIT_CONFIG_GLOBAL",
|
|
64
|
+
"GIT_CONFIG_SYSTEM",
|
|
65
|
+
"GIT_NO_REPLACE_OBJECTS",
|
|
66
|
+
"GIT_TERMINAL_PROMPT",
|
|
67
|
+
"HOME",
|
|
68
|
+
"LC_ALL",
|
|
69
|
+
"PATH",
|
|
70
|
+
"TMPDIR",
|
|
71
|
+
}
|
|
72
|
+
_TRACE_ENV_NAMES = {
|
|
73
|
+
"TANGLE_CANDIDATE_EXECUTION_ID",
|
|
74
|
+
"TANGLE_CANDIDATE_BUNDLE_DIGEST",
|
|
75
|
+
"TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST",
|
|
76
|
+
"TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST",
|
|
77
|
+
"TANGLE_TRACE_RUN_ID",
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class OriginalProfileFile:
|
|
83
|
+
path: str
|
|
84
|
+
existed: bool
|
|
85
|
+
tracked: bool
|
|
86
|
+
mode: int | None
|
|
87
|
+
content: bytes | None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class PierCandidateError(CandidateContractError):
|
|
91
|
+
"""The prepared candidate cannot be replayed safely by Pier."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _runtime_pier_version() -> str:
|
|
95
|
+
try:
|
|
96
|
+
return package_version("datacurve-pier")
|
|
97
|
+
except PackageNotFoundError as exc:
|
|
98
|
+
raise PierCandidateError(
|
|
99
|
+
"datacurve-pier package metadata is unavailable"
|
|
100
|
+
) from exc
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _same_or_descendant(path: str, root: str) -> bool:
|
|
104
|
+
return path == root or path.startswith(f"{root}/")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _join_workspace(root: str, relative: str) -> str:
|
|
108
|
+
root_path = PurePosixPath(root)
|
|
109
|
+
joined = root_path if relative == "." else root_path / PurePosixPath(relative)
|
|
110
|
+
if joined != root_path and root_path not in joined.parents:
|
|
111
|
+
raise PierCandidateError(f"cwd escapes its workspace: {relative}")
|
|
112
|
+
return str(joined)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _trace_env(
|
|
116
|
+
contract: PreparedCandidateContract, trace_run_id: str
|
|
117
|
+
) -> dict[str, str]:
|
|
118
|
+
return {
|
|
119
|
+
"TANGLE_CANDIDATE_EXECUTION_ID": contract.execution_id,
|
|
120
|
+
"TANGLE_CANDIDATE_BUNDLE_DIGEST": contract.bundle_digest,
|
|
121
|
+
"TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": contract.execution_plan_digest,
|
|
122
|
+
"TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": contract.materialization_receipt_digest,
|
|
123
|
+
"TANGLE_TRACE_RUN_ID": trace_run_id,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class TangleCandidateAgent(BaseAgent):
|
|
128
|
+
"""Pier adapter for a branded ``PreparedAgentCandidateExecution``."""
|
|
129
|
+
|
|
130
|
+
SUPPORTS_ATIF = False
|
|
131
|
+
SUPPORTS_WINDOWS = False
|
|
132
|
+
ISOLATE_CANDIDATE_USER = True
|
|
133
|
+
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
logs_dir: Path,
|
|
137
|
+
model_name: str | None = None,
|
|
138
|
+
plan_path: str | None = None,
|
|
139
|
+
receipt_path: str | None = None,
|
|
140
|
+
expected_receipt_digest: str | None = None,
|
|
141
|
+
trace_run_id: str | None = None,
|
|
142
|
+
task_dir: str | None = None,
|
|
143
|
+
profile_dir: str | None = None,
|
|
144
|
+
candidate_dir: str | None = None,
|
|
145
|
+
pier_version: str | None = None,
|
|
146
|
+
extra_env: dict[str, str] | None = None,
|
|
147
|
+
*args: Any,
|
|
148
|
+
**kwargs: Any,
|
|
149
|
+
) -> None:
|
|
150
|
+
super().__init__(logs_dir=logs_dir, model_name=model_name, *args, **kwargs)
|
|
151
|
+
if (
|
|
152
|
+
not plan_path
|
|
153
|
+
or not receipt_path
|
|
154
|
+
or not expected_receipt_digest
|
|
155
|
+
or not trace_run_id
|
|
156
|
+
):
|
|
157
|
+
raise PierCandidateError(
|
|
158
|
+
"plan_path, receipt_path, expected_receipt_digest, and trace_run_id are required"
|
|
159
|
+
)
|
|
160
|
+
if not task_dir or not profile_dir or not pier_version:
|
|
161
|
+
raise PierCandidateError("task_dir, profile_dir, and pier_version are required")
|
|
162
|
+
|
|
163
|
+
self._expected_pier_version = pier_version
|
|
164
|
+
self._contract = load_prepared_candidate_contract(
|
|
165
|
+
Path(plan_path).expanduser().resolve(),
|
|
166
|
+
Path(receipt_path).expanduser().resolve(),
|
|
167
|
+
expected_receipt_digest,
|
|
168
|
+
)
|
|
169
|
+
if model_name != self._contract.requested_model:
|
|
170
|
+
raise PierCandidateError(
|
|
171
|
+
"Pier model does not match the runtime-resolved request: "
|
|
172
|
+
f"expected={self._contract.requested_model} observed={model_name}"
|
|
173
|
+
)
|
|
174
|
+
self._extra_env = self._validate_protected_env(
|
|
175
|
+
extra_env or {}, trace_run_id
|
|
176
|
+
)
|
|
177
|
+
self._validate_execution_roots()
|
|
178
|
+
|
|
179
|
+
self._snapshots = tempfile.TemporaryDirectory(
|
|
180
|
+
prefix="agent-bench-pier-candidate-"
|
|
181
|
+
)
|
|
182
|
+
snapshot_root = Path(self._snapshots.name)
|
|
183
|
+
self._task_snapshot = snapshot_root / "task"
|
|
184
|
+
self._profile_snapshot = snapshot_root / "profile"
|
|
185
|
+
self._candidate_snapshot: Path | None = None
|
|
186
|
+
try:
|
|
187
|
+
immutable_snapshot(
|
|
188
|
+
Path(os.path.abspath(Path(task_dir).expanduser())),
|
|
189
|
+
self._task_snapshot,
|
|
190
|
+
self._contract.task_files,
|
|
191
|
+
"prepared task directory",
|
|
192
|
+
)
|
|
193
|
+
immutable_snapshot(
|
|
194
|
+
Path(os.path.abspath(Path(profile_dir).expanduser())),
|
|
195
|
+
self._profile_snapshot,
|
|
196
|
+
self._contract.profile_files,
|
|
197
|
+
"prepared profile directory",
|
|
198
|
+
)
|
|
199
|
+
if self._contract.candidate_root is not None:
|
|
200
|
+
if not candidate_dir:
|
|
201
|
+
raise PierCandidateError("active code requires candidate_dir")
|
|
202
|
+
self._candidate_snapshot = snapshot_root / "candidate"
|
|
203
|
+
immutable_snapshot(
|
|
204
|
+
Path(os.path.abspath(Path(candidate_dir).expanduser())),
|
|
205
|
+
self._candidate_snapshot,
|
|
206
|
+
self._contract.candidate_files,
|
|
207
|
+
"prepared candidate directory",
|
|
208
|
+
)
|
|
209
|
+
elif candidate_dir:
|
|
210
|
+
raise PierCandidateError("disabled code cannot receive candidate_dir")
|
|
211
|
+
except Exception:
|
|
212
|
+
self._snapshots.cleanup()
|
|
213
|
+
raise
|
|
214
|
+
|
|
215
|
+
self._original_profile_files: tuple[OriginalProfileFile, ...] = ()
|
|
216
|
+
self._process_boundary = CandidateProcessBoundary(
|
|
217
|
+
self,
|
|
218
|
+
ProcessBoundaryConfig(
|
|
219
|
+
uid=_CANDIDATE_UID,
|
|
220
|
+
gid=_CANDIDATE_GID,
|
|
221
|
+
evaluator_root=_EVALUATOR_ROOT,
|
|
222
|
+
control_root=_CONTROL_ROOT,
|
|
223
|
+
home=_CANDIDATE_HOME,
|
|
224
|
+
temporary=_CANDIDATE_TMP,
|
|
225
|
+
isolate_user=self.ISOLATE_CANDIDATE_USER,
|
|
226
|
+
),
|
|
227
|
+
)
|
|
228
|
+
self._workspace_boundary = CandidateWorkspaceBoundary(
|
|
229
|
+
self,
|
|
230
|
+
WorkspaceBoundaryConfig(
|
|
231
|
+
contract=self._contract,
|
|
232
|
+
task_snapshot=self._task_snapshot,
|
|
233
|
+
control_root=_CONTROL_ROOT,
|
|
234
|
+
protected_roots=_PIER_PROTECTED_ROOTS,
|
|
235
|
+
error_type=PierCandidateError,
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
def _validate_protected_env(
|
|
240
|
+
self, values: dict[str, str], trace_run_id: str
|
|
241
|
+
) -> dict[str, str]:
|
|
242
|
+
protected: dict[str, str] = {}
|
|
243
|
+
public_names = set(self._contract.env)
|
|
244
|
+
reserved_public = sorted(public_names & _RESERVED_PROCESS_ENV)
|
|
245
|
+
if reserved_public:
|
|
246
|
+
raise PierCandidateError(
|
|
247
|
+
"signed public env collides with evaluator Git safety fields: "
|
|
248
|
+
+ ", ".join(reserved_public)
|
|
249
|
+
)
|
|
250
|
+
expected_trace = _trace_env(self._contract, trace_run_id)
|
|
251
|
+
for name, expected in expected_trace.items():
|
|
252
|
+
if values.get(name) != expected:
|
|
253
|
+
raise PierCandidateError(
|
|
254
|
+
f"evaluator trace environment is missing or mismatched: {name}"
|
|
255
|
+
)
|
|
256
|
+
for name, value in values.items():
|
|
257
|
+
if not _ENV_RE.fullmatch(name) or name in _RESERVED_PROCESS_ENV:
|
|
258
|
+
raise PierCandidateError(
|
|
259
|
+
f"invalid protected model environment name: {name}"
|
|
260
|
+
)
|
|
261
|
+
if name in public_names:
|
|
262
|
+
raise PierCandidateError(
|
|
263
|
+
f"protected model environment overlaps signed public env: {name}"
|
|
264
|
+
)
|
|
265
|
+
if name.startswith("TANGLE_CANDIDATE_") and name not in _TRACE_ENV_NAMES:
|
|
266
|
+
raise PierCandidateError(
|
|
267
|
+
f"unknown evaluator candidate environment: {name}"
|
|
268
|
+
)
|
|
269
|
+
if not isinstance(value, str) or not value or "\0" in value:
|
|
270
|
+
raise PierCandidateError(
|
|
271
|
+
f"protected model environment value is empty or invalid: {name}"
|
|
272
|
+
)
|
|
273
|
+
protected[name] = value
|
|
274
|
+
return protected
|
|
275
|
+
|
|
276
|
+
def _validate_execution_roots(self) -> None:
|
|
277
|
+
roots = [self._contract.task_root]
|
|
278
|
+
if self._contract.candidate_root is not None:
|
|
279
|
+
roots.append(self._contract.candidate_root)
|
|
280
|
+
for root in roots:
|
|
281
|
+
protected = next(
|
|
282
|
+
(
|
|
283
|
+
path
|
|
284
|
+
for path in _PIER_PROTECTED_ROOTS
|
|
285
|
+
if _same_or_descendant(root, path)
|
|
286
|
+
),
|
|
287
|
+
None,
|
|
288
|
+
)
|
|
289
|
+
if protected is not None:
|
|
290
|
+
raise PierCandidateError(
|
|
291
|
+
f"execution workspace cannot be below Pier-owned path {protected}"
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def name(self) -> str:
|
|
295
|
+
short = self._contract.bundle_digest.removeprefix("sha256:")[:16]
|
|
296
|
+
return f"tangle-candidate-{short}"
|
|
297
|
+
|
|
298
|
+
def version(self) -> str:
|
|
299
|
+
return _ADAPTER_VERSION
|
|
300
|
+
|
|
301
|
+
def network_allowlist(self) -> NetworkAllowlist:
|
|
302
|
+
return NetworkAllowlist(domains=list(self._contract.model_network_domains))
|
|
303
|
+
|
|
304
|
+
async def _exec(
|
|
305
|
+
self,
|
|
306
|
+
environment: BaseEnvironment,
|
|
307
|
+
command: str,
|
|
308
|
+
*,
|
|
309
|
+
cwd: str | None = None,
|
|
310
|
+
env: dict[str, str] | None = None,
|
|
311
|
+
user: str | int | None = None,
|
|
312
|
+
timeout_sec: float | None = None,
|
|
313
|
+
check: bool = True,
|
|
314
|
+
) -> Any:
|
|
315
|
+
result = await environment.exec(
|
|
316
|
+
command=command,
|
|
317
|
+
cwd=cwd,
|
|
318
|
+
env=environment.agent_process_env(env),
|
|
319
|
+
user=user,
|
|
320
|
+
timeout_sec=timeout_sec,
|
|
321
|
+
)
|
|
322
|
+
if check and result.return_code != 0:
|
|
323
|
+
raise PierCandidateError(
|
|
324
|
+
f"candidate bridge command failed ({result.return_code}): {command}; "
|
|
325
|
+
f"stderr={(result.stderr or '')[:500]}"
|
|
326
|
+
)
|
|
327
|
+
return result
|
|
328
|
+
|
|
329
|
+
async def _assert_real_directory(
|
|
330
|
+
self,
|
|
331
|
+
environment: BaseEnvironment,
|
|
332
|
+
path: str,
|
|
333
|
+
*,
|
|
334
|
+
label: str,
|
|
335
|
+
user: str | int | None = "root",
|
|
336
|
+
) -> None:
|
|
337
|
+
quoted = shlex.quote(path)
|
|
338
|
+
await self._exec(
|
|
339
|
+
environment,
|
|
340
|
+
"\n".join(
|
|
341
|
+
[
|
|
342
|
+
"set -eu",
|
|
343
|
+
f"test -d {quoted}",
|
|
344
|
+
f"test ! -L {quoted}",
|
|
345
|
+
f'test "$(realpath -e -- {quoted})" = {quoted}',
|
|
346
|
+
]
|
|
347
|
+
),
|
|
348
|
+
user=user,
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
async def _ensure_real_directory(
|
|
352
|
+
self,
|
|
353
|
+
environment: BaseEnvironment,
|
|
354
|
+
path: str,
|
|
355
|
+
*,
|
|
356
|
+
anchor: str,
|
|
357
|
+
mode: int = 0o755,
|
|
358
|
+
) -> None:
|
|
359
|
+
parsed = PurePosixPath(path)
|
|
360
|
+
parsed_anchor = PurePosixPath(anchor)
|
|
361
|
+
if parsed != parsed_anchor and parsed_anchor not in parsed.parents:
|
|
362
|
+
raise PierCandidateError(f"directory escapes its trusted anchor: {path}")
|
|
363
|
+
await self._assert_real_directory(
|
|
364
|
+
environment, anchor, label="trusted directory anchor"
|
|
365
|
+
)
|
|
366
|
+
current = parsed_anchor
|
|
367
|
+
relative_parts = parsed.relative_to(parsed_anchor).parts
|
|
368
|
+
for part in relative_parts:
|
|
369
|
+
current /= part
|
|
370
|
+
value = str(current)
|
|
371
|
+
quoted = shlex.quote(value)
|
|
372
|
+
await self._exec(
|
|
373
|
+
environment,
|
|
374
|
+
"\n".join(
|
|
375
|
+
[
|
|
376
|
+
"set -eu",
|
|
377
|
+
f"if test -e {quoted} || test -L {quoted}; then",
|
|
378
|
+
f" test -d {quoted}",
|
|
379
|
+
f" test ! -L {quoted}",
|
|
380
|
+
"else",
|
|
381
|
+
f" mkdir -- {quoted}",
|
|
382
|
+
f" chmod {mode:o} -- {quoted}",
|
|
383
|
+
"fi",
|
|
384
|
+
f'test "$(realpath -e -- {quoted})" = {quoted}',
|
|
385
|
+
]
|
|
386
|
+
),
|
|
387
|
+
user="root",
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
async def _write_verified_file(
|
|
391
|
+
self,
|
|
392
|
+
environment: BaseEnvironment,
|
|
393
|
+
source: Path,
|
|
394
|
+
target: str,
|
|
395
|
+
*,
|
|
396
|
+
anchor: str,
|
|
397
|
+
mode: int,
|
|
398
|
+
digest: str,
|
|
399
|
+
byte_length: int,
|
|
400
|
+
) -> None:
|
|
401
|
+
parent = str(PurePosixPath(target).parent)
|
|
402
|
+
await self._ensure_real_directory(environment, parent, anchor=anchor)
|
|
403
|
+
temporary = f"{parent}/.tangle-{secrets.token_hex(24)}"
|
|
404
|
+
quoted_temporary = shlex.quote(temporary)
|
|
405
|
+
quoted_target = shlex.quote(target)
|
|
406
|
+
expected_hash = digest.removeprefix("sha256:")
|
|
407
|
+
await self._exec(
|
|
408
|
+
environment,
|
|
409
|
+
f"test ! -e {quoted_temporary} && test ! -L {quoted_temporary}",
|
|
410
|
+
user="root",
|
|
411
|
+
)
|
|
412
|
+
try:
|
|
413
|
+
await environment.upload_file(source, temporary)
|
|
414
|
+
await self._exec(
|
|
415
|
+
environment,
|
|
416
|
+
"\n".join(
|
|
417
|
+
[
|
|
418
|
+
"set -eu",
|
|
419
|
+
f"test -f {quoted_temporary}",
|
|
420
|
+
f"test ! -L {quoted_temporary}",
|
|
421
|
+
f'test "$(stat -c %h -- {quoted_temporary})" = 1',
|
|
422
|
+
f'test "$(wc -c < {quoted_temporary})" = {byte_length}',
|
|
423
|
+
f"test \"$(sha256sum {quoted_temporary} | cut -d' ' -f1)\" = {expected_hash}",
|
|
424
|
+
f"chmod {mode:o} -- {quoted_temporary}",
|
|
425
|
+
f'test "$(stat -c %a -- {quoted_temporary})" = {mode:o}',
|
|
426
|
+
f"rm -rf -- {quoted_target}",
|
|
427
|
+
f"mv -fT -- {quoted_temporary} {quoted_target}",
|
|
428
|
+
f"test -f {quoted_target}",
|
|
429
|
+
f"test ! -L {quoted_target}",
|
|
430
|
+
f'test "$(stat -c %h -- {quoted_target})" = 1',
|
|
431
|
+
f'test "$(stat -c %a -- {quoted_target})" = {mode:o}',
|
|
432
|
+
f'test "$(wc -c < {quoted_target})" = {byte_length}',
|
|
433
|
+
f"test \"$(sha256sum {quoted_target} | cut -d' ' -f1)\" = {expected_hash}",
|
|
434
|
+
]
|
|
435
|
+
),
|
|
436
|
+
user="root",
|
|
437
|
+
)
|
|
438
|
+
finally:
|
|
439
|
+
await self._exec(
|
|
440
|
+
environment,
|
|
441
|
+
f"rm -f -- {quoted_temporary}",
|
|
442
|
+
user="root",
|
|
443
|
+
check=False,
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
async def _verify_pier_execution_identity(
|
|
447
|
+
self, environment: BaseEnvironment
|
|
448
|
+
) -> None:
|
|
449
|
+
await self._workspace_boundary.verify_pier_execution_identity(environment)
|
|
450
|
+
|
|
451
|
+
async def _capture_original_profile_files(
|
|
452
|
+
self, environment: BaseEnvironment
|
|
453
|
+
) -> tuple[OriginalProfileFile, ...]:
|
|
454
|
+
if self._contract.profile_target != "task":
|
|
455
|
+
return ()
|
|
456
|
+
originals: list[OriginalProfileFile] = []
|
|
457
|
+
root = self._contract.task_root
|
|
458
|
+
git = f"git -c core.hooksPath=/dev/null -C {shlex.quote(root)}"
|
|
459
|
+
for file in self._contract.profile_files:
|
|
460
|
+
target = f"{root}/{file.path}"
|
|
461
|
+
quoted = shlex.quote(target)
|
|
462
|
+
parent = shlex.quote(str(PurePosixPath(target).parent))
|
|
463
|
+
presence = await self._exec(
|
|
464
|
+
environment,
|
|
465
|
+
"\n".join(
|
|
466
|
+
[
|
|
467
|
+
"set -eu",
|
|
468
|
+
f'test "$(realpath -m -- {parent})" = {parent}',
|
|
469
|
+
f"if test -e {quoted} || test -L {quoted}; then",
|
|
470
|
+
f" test -f {quoted}",
|
|
471
|
+
f" test ! -L {quoted}",
|
|
472
|
+
f' test "$(stat -c %h -- {quoted})" = 1',
|
|
473
|
+
"else",
|
|
474
|
+
" exit 2",
|
|
475
|
+
"fi",
|
|
476
|
+
]
|
|
477
|
+
),
|
|
478
|
+
user="root",
|
|
479
|
+
check=False,
|
|
480
|
+
)
|
|
481
|
+
tracked = (
|
|
482
|
+
await self._exec(
|
|
483
|
+
environment,
|
|
484
|
+
f"{git} ls-files --error-unmatch -- {shlex.quote(file.path)}",
|
|
485
|
+
env=candidate_git_env(root),
|
|
486
|
+
user="root",
|
|
487
|
+
check=False,
|
|
488
|
+
)
|
|
489
|
+
).return_code == 0
|
|
490
|
+
if presence.return_code == 2:
|
|
491
|
+
originals.append(
|
|
492
|
+
OriginalProfileFile(file.path, False, tracked, None, None)
|
|
493
|
+
)
|
|
494
|
+
continue
|
|
495
|
+
if presence.return_code != 0:
|
|
496
|
+
raise PierCandidateError(
|
|
497
|
+
f"profile mount target is not one regular file: {file.path}"
|
|
498
|
+
)
|
|
499
|
+
encoded = await self._exec(
|
|
500
|
+
environment, f"base64 -w0 -- {quoted}", user="root"
|
|
501
|
+
)
|
|
502
|
+
mode = await self._exec(environment, f"stat -c %a -- {quoted}", user="root")
|
|
503
|
+
try:
|
|
504
|
+
content = base64.b64decode(
|
|
505
|
+
(encoded.stdout or "").strip(), validate=True
|
|
506
|
+
)
|
|
507
|
+
parsed_mode = int((mode.stdout or "").strip(), 8)
|
|
508
|
+
except (ValueError, TypeError) as exc:
|
|
509
|
+
raise PierCandidateError(
|
|
510
|
+
f"could not capture original profile target: {file.path}"
|
|
511
|
+
) from exc
|
|
512
|
+
originals.append(
|
|
513
|
+
OriginalProfileFile(file.path, True, tracked, parsed_mode, content)
|
|
514
|
+
)
|
|
515
|
+
return tuple(originals)
|
|
516
|
+
|
|
517
|
+
async def _apply_profile(self, environment: BaseEnvironment) -> None:
|
|
518
|
+
target_root = (
|
|
519
|
+
self._contract.task_root
|
|
520
|
+
if self._contract.profile_target == "task"
|
|
521
|
+
else self._contract.candidate_root
|
|
522
|
+
)
|
|
523
|
+
if target_root is None:
|
|
524
|
+
raise PierCandidateError("profile target workspace is unavailable")
|
|
525
|
+
self._original_profile_files = await self._capture_original_profile_files(
|
|
526
|
+
environment
|
|
527
|
+
)
|
|
528
|
+
for file in self._contract.profile_files:
|
|
529
|
+
target = f"{target_root}/{file.path}"
|
|
530
|
+
source = self._profile_snapshot / file.path
|
|
531
|
+
await self._write_verified_file(
|
|
532
|
+
environment,
|
|
533
|
+
source,
|
|
534
|
+
target,
|
|
535
|
+
anchor=target_root,
|
|
536
|
+
mode=file.mode,
|
|
537
|
+
digest=file.sha256,
|
|
538
|
+
byte_length=source.stat().st_size,
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
async def setup(self, environment: BaseEnvironment) -> None:
|
|
542
|
+
try:
|
|
543
|
+
observed_pier = _runtime_pier_version()
|
|
544
|
+
if observed_pier != self._expected_pier_version:
|
|
545
|
+
raise PierCandidateError(
|
|
546
|
+
f"Pier version mismatch: expected={self._expected_pier_version} "
|
|
547
|
+
f"installed={observed_pier}"
|
|
548
|
+
)
|
|
549
|
+
await self._verify_pier_execution_identity(environment)
|
|
550
|
+
await self._workspace_boundary.materialize_task_workspace(environment)
|
|
551
|
+
await self._workspace_boundary.verify_repository(environment)
|
|
552
|
+
await self._workspace_boundary.verify_container_workspace(
|
|
553
|
+
environment,
|
|
554
|
+
self._contract.task_root,
|
|
555
|
+
self._contract.task_files,
|
|
556
|
+
allow_task_metadata=True,
|
|
557
|
+
)
|
|
558
|
+
if self._contract.candidate_root is not None:
|
|
559
|
+
if self._candidate_snapshot is None:
|
|
560
|
+
raise PierCandidateError("candidate snapshot is unavailable")
|
|
561
|
+
quoted = shlex.quote(self._contract.candidate_root)
|
|
562
|
+
candidate_parent = str(
|
|
563
|
+
PurePosixPath(self._contract.candidate_root).parent
|
|
564
|
+
)
|
|
565
|
+
await self._ensure_real_directory(
|
|
566
|
+
environment, candidate_parent, anchor="/"
|
|
567
|
+
)
|
|
568
|
+
await self._exec(
|
|
569
|
+
environment,
|
|
570
|
+
"\n".join(
|
|
571
|
+
[
|
|
572
|
+
"set -eu",
|
|
573
|
+
f"test ! -e {quoted}",
|
|
574
|
+
f"test ! -L {quoted}",
|
|
575
|
+
f"mkdir -- {quoted}",
|
|
576
|
+
f'test "$(realpath -e -- {quoted})" = {quoted}',
|
|
577
|
+
]
|
|
578
|
+
),
|
|
579
|
+
user="root",
|
|
580
|
+
)
|
|
581
|
+
await self._workspace_boundary.verify_resolved_root_isolation(
|
|
582
|
+
environment
|
|
583
|
+
)
|
|
584
|
+
await environment.upload_dir(
|
|
585
|
+
self._candidate_snapshot, self._contract.candidate_root
|
|
586
|
+
)
|
|
587
|
+
await self._workspace_boundary.verify_container_workspace(
|
|
588
|
+
environment,
|
|
589
|
+
self._contract.candidate_root,
|
|
590
|
+
self._contract.candidate_files,
|
|
591
|
+
allow_task_metadata=False,
|
|
592
|
+
)
|
|
593
|
+
else:
|
|
594
|
+
await self._workspace_boundary.verify_resolved_root_isolation(
|
|
595
|
+
environment
|
|
596
|
+
)
|
|
597
|
+
workspace_roots = (self._contract.task_root,) + (
|
|
598
|
+
(self._contract.candidate_root,)
|
|
599
|
+
if self._contract.candidate_root is not None
|
|
600
|
+
else ()
|
|
601
|
+
)
|
|
602
|
+
await self._process_boundary.prepare_identity(environment, workspace_roots)
|
|
603
|
+
await self._apply_profile(environment)
|
|
604
|
+
|
|
605
|
+
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
606
|
+
(self.logs_dir / "execution-plan.json").write_bytes(self._contract.plan_raw)
|
|
607
|
+
(self.logs_dir / "materialization-receipt.json").write_bytes(
|
|
608
|
+
self._contract.receipt_raw
|
|
609
|
+
)
|
|
610
|
+
finally:
|
|
611
|
+
self._snapshots.cleanup()
|
|
612
|
+
|
|
613
|
+
def _identity_metadata(self) -> dict[str, Any]:
|
|
614
|
+
return {
|
|
615
|
+
"executionId": self._contract.execution_id,
|
|
616
|
+
"bundleDigest": self._contract.bundle_digest,
|
|
617
|
+
"executionPlanDigest": self._contract.execution_plan_digest,
|
|
618
|
+
"materializationReceiptDigest": self._contract.materialization_receipt_digest,
|
|
619
|
+
"usageSource": "protected-agent-eval-trace",
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
async def _upload_instruction(
|
|
623
|
+
self, environment: BaseEnvironment, instruction: str
|
|
624
|
+
) -> tuple[list[str], str | None, str | None]:
|
|
625
|
+
try:
|
|
626
|
+
raw = instruction.encode("utf-8")
|
|
627
|
+
except UnicodeEncodeError as exc:
|
|
628
|
+
raise PierCandidateError("Pier instruction is not valid UTF-8") from exc
|
|
629
|
+
evidence = self._contract.instruction
|
|
630
|
+
if len(raw) != evidence.byte_length or sha256_bytes(raw) != evidence.sha256:
|
|
631
|
+
raise PierCandidateError(
|
|
632
|
+
"Pier instruction bytes do not match the signed task instruction"
|
|
633
|
+
)
|
|
634
|
+
|
|
635
|
+
argv = [self._contract.executable, *self._contract.args]
|
|
636
|
+
if evidence.delivery_kind == "argv-append":
|
|
637
|
+
argv.append(instruction)
|
|
638
|
+
return argv, None, None
|
|
639
|
+
|
|
640
|
+
target = evidence.delivery_path or _PROTECTED_TASK_PATH
|
|
641
|
+
with tempfile.NamedTemporaryFile(delete=False) as handle:
|
|
642
|
+
handle.write(raw)
|
|
643
|
+
local_path = Path(handle.name)
|
|
644
|
+
try:
|
|
645
|
+
await self._write_verified_file(
|
|
646
|
+
environment,
|
|
647
|
+
local_path,
|
|
648
|
+
target,
|
|
649
|
+
anchor="/",
|
|
650
|
+
mode=0o644,
|
|
651
|
+
digest=evidence.sha256,
|
|
652
|
+
byte_length=len(raw),
|
|
653
|
+
)
|
|
654
|
+
finally:
|
|
655
|
+
local_path.unlink(missing_ok=True)
|
|
656
|
+
return (
|
|
657
|
+
argv,
|
|
658
|
+
target if evidence.delivery_kind == "stdin-utf8" else None,
|
|
659
|
+
target,
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
async def _remove_instruction_file(
|
|
663
|
+
self, environment: BaseEnvironment, path: str | None
|
|
664
|
+
) -> None:
|
|
665
|
+
if path is None:
|
|
666
|
+
return
|
|
667
|
+
parent = str(PurePosixPath(path).parent)
|
|
668
|
+
await self._assert_real_directory(
|
|
669
|
+
environment, parent, label="protected instruction directory"
|
|
670
|
+
)
|
|
671
|
+
await self._exec(
|
|
672
|
+
environment,
|
|
673
|
+
f"rm -f -- {shlex.quote(path)}",
|
|
674
|
+
user="root",
|
|
675
|
+
)
|
|
676
|
+
|
|
677
|
+
async def _restore_profile_and_capture_solution(
|
|
678
|
+
self, environment: BaseEnvironment
|
|
679
|
+
) -> None:
|
|
680
|
+
root = self._contract.task_root
|
|
681
|
+
quoted_root = shlex.quote(root)
|
|
682
|
+
git = f"git -c core.hooksPath=/dev/null -C {quoted_root}"
|
|
683
|
+
git_env = candidate_git_env(root)
|
|
684
|
+
|
|
685
|
+
# Ignore candidate-authored commits and derive one evaluator-owned final
|
|
686
|
+
# tree from the actual working files relative to the signed base commit.
|
|
687
|
+
await self._exec(
|
|
688
|
+
environment,
|
|
689
|
+
f"{git} reset --mixed {shlex.quote(self._contract.repository_base_commit)}",
|
|
690
|
+
env=git_env,
|
|
691
|
+
user="root",
|
|
692
|
+
)
|
|
693
|
+
for original in self._original_profile_files:
|
|
694
|
+
target = f"{root}/{original.path}"
|
|
695
|
+
quoted = shlex.quote(target)
|
|
696
|
+
if original.existed:
|
|
697
|
+
if original.content is None or original.mode is None:
|
|
698
|
+
raise PierCandidateError(
|
|
699
|
+
f"profile backup is incomplete: {original.path}"
|
|
700
|
+
)
|
|
701
|
+
with tempfile.NamedTemporaryFile(delete=False) as handle:
|
|
702
|
+
handle.write(original.content)
|
|
703
|
+
local_path = Path(handle.name)
|
|
704
|
+
try:
|
|
705
|
+
await self._write_verified_file(
|
|
706
|
+
environment,
|
|
707
|
+
local_path,
|
|
708
|
+
target,
|
|
709
|
+
anchor=root,
|
|
710
|
+
mode=original.mode,
|
|
711
|
+
digest=sha256_bytes(original.content),
|
|
712
|
+
byte_length=len(original.content),
|
|
713
|
+
)
|
|
714
|
+
finally:
|
|
715
|
+
local_path.unlink(missing_ok=True)
|
|
716
|
+
else:
|
|
717
|
+
parent = str(PurePosixPath(target).parent)
|
|
718
|
+
await self._ensure_real_directory(environment, parent, anchor=root)
|
|
719
|
+
await self._exec(environment, f"rm -rf -- {quoted}", user="root")
|
|
720
|
+
|
|
721
|
+
await self._exec(environment, f"{git} add -A -- .", env=git_env, user="root")
|
|
722
|
+
for original in self._original_profile_files:
|
|
723
|
+
if original.tracked:
|
|
724
|
+
continue
|
|
725
|
+
await self._exec(
|
|
726
|
+
environment,
|
|
727
|
+
f"{git} rm -r -f --cached --ignore-unmatch -- {shlex.quote(original.path)}",
|
|
728
|
+
env=git_env,
|
|
729
|
+
user="root",
|
|
730
|
+
)
|
|
731
|
+
staged = await self._exec(
|
|
732
|
+
environment,
|
|
733
|
+
f"{git} diff --cached --quiet",
|
|
734
|
+
env=git_env,
|
|
735
|
+
user="root",
|
|
736
|
+
check=False,
|
|
737
|
+
)
|
|
738
|
+
if staged.return_code not in {0, 1}:
|
|
739
|
+
raise PierCandidateError("could not inspect the sanitized solution tree")
|
|
740
|
+
if staged.return_code == 1:
|
|
741
|
+
commit_env = {
|
|
742
|
+
**git_env,
|
|
743
|
+
"GIT_AUTHOR_NAME": "Tangle Evaluator",
|
|
744
|
+
"GIT_AUTHOR_EMAIL": "evaluator@tangle.tools",
|
|
745
|
+
"GIT_COMMITTER_NAME": "Tangle Evaluator",
|
|
746
|
+
"GIT_COMMITTER_EMAIL": "evaluator@tangle.tools",
|
|
747
|
+
}
|
|
748
|
+
await self._exec(
|
|
749
|
+
environment,
|
|
750
|
+
f"{git} commit -m 'capture candidate solution'",
|
|
751
|
+
env=commit_env,
|
|
752
|
+
user="root",
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
async def run(
|
|
756
|
+
self,
|
|
757
|
+
instruction: str,
|
|
758
|
+
environment: BaseEnvironment,
|
|
759
|
+
context: AgentContext,
|
|
760
|
+
) -> None:
|
|
761
|
+
argv, stdin_path, instruction_path = await self._upload_instruction(
|
|
762
|
+
environment, instruction
|
|
763
|
+
)
|
|
764
|
+
cwd = _join_workspace(self._contract.cwd_root, self._contract.cwd_path)
|
|
765
|
+
await self._exec(environment, f"test -d {shlex.quote(cwd)}")
|
|
766
|
+
|
|
767
|
+
process_env = {
|
|
768
|
+
**self._extra_env,
|
|
769
|
+
**candidate_git_env(self._contract.task_root),
|
|
770
|
+
"HOME": _CANDIDATE_HOME,
|
|
771
|
+
"TMPDIR": _CANDIDATE_TMP,
|
|
772
|
+
"PATH": _FIXED_PATH,
|
|
773
|
+
}
|
|
774
|
+
wrapper, timeout_marker = await self._process_boundary.stage_wrapper(
|
|
775
|
+
environment
|
|
776
|
+
)
|
|
777
|
+
timeout_seconds = self._contract.timeout_ms / 1000
|
|
778
|
+
command = shlex.join(
|
|
779
|
+
[
|
|
780
|
+
wrapper,
|
|
781
|
+
timeout_marker,
|
|
782
|
+
str(_CANDIDATE_UID),
|
|
783
|
+
str(_CANDIDATE_GID),
|
|
784
|
+
f"{timeout_seconds:.3f}",
|
|
785
|
+
"2",
|
|
786
|
+
stdin_path or "-",
|
|
787
|
+
*(f"{name}={value}" for name, value in self._contract.env.items()),
|
|
788
|
+
*argv,
|
|
789
|
+
]
|
|
790
|
+
)
|
|
791
|
+
context.metadata = {
|
|
792
|
+
**self._identity_metadata(),
|
|
793
|
+
"termination": {"kind": "running"},
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
started = time.monotonic()
|
|
797
|
+
try:
|
|
798
|
+
result = await environment.exec(
|
|
799
|
+
command=command,
|
|
800
|
+
cwd=cwd,
|
|
801
|
+
env=environment.agent_process_env(process_env),
|
|
802
|
+
user="root",
|
|
803
|
+
timeout_sec=math.ceil(timeout_seconds) + 12,
|
|
804
|
+
)
|
|
805
|
+
except (asyncio.CancelledError, asyncio.TimeoutError):
|
|
806
|
+
elapsed = int((time.monotonic() - started) * 1000)
|
|
807
|
+
cleanup_errors: list[str] = []
|
|
808
|
+
try:
|
|
809
|
+
await asyncio.shield(
|
|
810
|
+
self._process_boundary.kill_and_wait(environment)
|
|
811
|
+
)
|
|
812
|
+
except Exception as cleanup_exc:
|
|
813
|
+
cleanup_errors.append(str(cleanup_exc))
|
|
814
|
+
try:
|
|
815
|
+
await asyncio.shield(
|
|
816
|
+
self._remove_instruction_file(environment, instruction_path)
|
|
817
|
+
)
|
|
818
|
+
except Exception as cleanup_exc:
|
|
819
|
+
cleanup_errors.append(str(cleanup_exc))
|
|
820
|
+
try:
|
|
821
|
+
await asyncio.shield(
|
|
822
|
+
self._restore_profile_and_capture_solution(environment)
|
|
823
|
+
)
|
|
824
|
+
except (
|
|
825
|
+
Exception
|
|
826
|
+
) as cleanup_exc: # cleanup failure is evidence, never hidden
|
|
827
|
+
cleanup_errors.append(str(cleanup_exc))
|
|
828
|
+
try:
|
|
829
|
+
await asyncio.shield(
|
|
830
|
+
self._process_boundary.remove_control_files(
|
|
831
|
+
environment, wrapper, timeout_marker
|
|
832
|
+
)
|
|
833
|
+
)
|
|
834
|
+
except Exception as cleanup_exc:
|
|
835
|
+
cleanup_errors.append(str(cleanup_exc))
|
|
836
|
+
context.metadata = {
|
|
837
|
+
**self._identity_metadata(),
|
|
838
|
+
"termination": {"kind": "cancelled", "observedElapsedMs": elapsed},
|
|
839
|
+
**(
|
|
840
|
+
{"cleanupError": "; ".join(cleanup_errors)}
|
|
841
|
+
if cleanup_errors
|
|
842
|
+
else {}
|
|
843
|
+
),
|
|
844
|
+
}
|
|
845
|
+
raise
|
|
846
|
+
except Exception:
|
|
847
|
+
await asyncio.shield(self._process_boundary.kill_and_wait(environment))
|
|
848
|
+
await asyncio.shield(
|
|
849
|
+
self._remove_instruction_file(environment, instruction_path)
|
|
850
|
+
)
|
|
851
|
+
await asyncio.shield(
|
|
852
|
+
self._restore_profile_and_capture_solution(environment)
|
|
853
|
+
)
|
|
854
|
+
await asyncio.shield(
|
|
855
|
+
self._process_boundary.remove_control_files(
|
|
856
|
+
environment, wrapper, timeout_marker
|
|
857
|
+
)
|
|
858
|
+
)
|
|
859
|
+
raise
|
|
860
|
+
|
|
861
|
+
elapsed = int((time.monotonic() - started) * 1000)
|
|
862
|
+
timed_out = await self._process_boundary.timeout_marker_exists(
|
|
863
|
+
environment, timeout_marker
|
|
864
|
+
)
|
|
865
|
+
await self._process_boundary.kill_and_wait(environment)
|
|
866
|
+
await self._remove_instruction_file(environment, instruction_path)
|
|
867
|
+
await self._restore_profile_and_capture_solution(environment)
|
|
868
|
+
await self._process_boundary.remove_control_files(
|
|
869
|
+
environment, wrapper, timeout_marker
|
|
870
|
+
)
|
|
871
|
+
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
872
|
+
(self.logs_dir / "stdout.txt").write_text(result.stdout or "", encoding="utf-8")
|
|
873
|
+
(self.logs_dir / "stderr.txt").write_text(result.stderr or "", encoding="utf-8")
|
|
874
|
+
(self.logs_dir / "process.json").write_text(
|
|
875
|
+
json.dumps(
|
|
876
|
+
{
|
|
877
|
+
"executionId": self._contract.execution_id,
|
|
878
|
+
"exitCode": result.return_code,
|
|
879
|
+
"elapsedMs": elapsed,
|
|
880
|
+
"cwd": cwd,
|
|
881
|
+
"argv": argv,
|
|
882
|
+
"instructionDelivery": self._contract.instruction.delivery_kind,
|
|
883
|
+
},
|
|
884
|
+
indent=2,
|
|
885
|
+
)
|
|
886
|
+
+ "\n",
|
|
887
|
+
encoding="utf-8",
|
|
888
|
+
)
|
|
889
|
+
context.metadata = {
|
|
890
|
+
**self._identity_metadata(),
|
|
891
|
+
"termination": (
|
|
892
|
+
{"kind": "timeout", "timeoutMs": self._contract.timeout_ms}
|
|
893
|
+
if timed_out
|
|
894
|
+
else {"kind": "exit", "exitCode": result.return_code}
|
|
895
|
+
),
|
|
896
|
+
"observedElapsedMs": elapsed,
|
|
897
|
+
}
|
|
898
|
+
if timed_out:
|
|
899
|
+
raise asyncio.TimeoutError(
|
|
900
|
+
f"candidate exceeded signed timeout {self._contract.timeout_ms}ms"
|
|
901
|
+
)
|
|
902
|
+
if result.return_code != 0:
|
|
903
|
+
raise NonZeroAgentExitCodeError(
|
|
904
|
+
f"candidate exited {result.return_code}: "
|
|
905
|
+
f"stdout={(result.stdout or '')[:500]} "
|
|
906
|
+
f"stderr={(result.stderr or '')[:500]}"
|
|
907
|
+
)
|