@tangle-network/agent-bench 0.1.0 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/HARNESS.md +302 -0
  3. package/README.md +26 -1
  4. package/fixtures/aec-bench.json +18 -0
  5. package/fixtures/agentbench-dbbench.json +22 -0
  6. package/fixtures/bfcl.json +45 -0
  7. package/fixtures/commit0.json +72 -0
  8. package/fixtures/crag.json +10 -0
  9. package/fixtures/dabstep.json +22 -0
  10. package/fixtures/enterpriseops-gym.json +103 -0
  11. package/fixtures/finresearchbench.json +21 -0
  12. package/fixtures/finsearchcomp.json +66 -0
  13. package/fixtures/frames.json +26 -0
  14. package/fixtures/hotpotqa.json +182 -0
  15. package/fixtures/nomiracl.json +26 -0
  16. package/fixtures/open-rag-bench.json +16 -0
  17. package/fixtures/pier-agent/no-model-task/environment/Dockerfile +16 -0
  18. package/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt +1 -0
  19. package/fixtures/pier-agent/no-model-task/instruction.md +6 -0
  20. package/fixtures/pier-agent/no-model-task/pre_artifacts.sh +6 -0
  21. package/fixtures/pier-agent/no-model-task/task.toml +35 -0
  22. package/fixtures/pier-agent/no-model-task/tests/Dockerfile +17 -0
  23. package/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt +1 -0
  24. package/fixtures/pier-agent/no-model-task/tests/test.sh +19 -0
  25. package/fixtures/programbench.json +17 -0
  26. package/fixtures/ragbench.json +21 -0
  27. package/fixtures/simpleqa.json +121 -0
  28. package/fixtures/t2-ragbench.json +13 -0
  29. package/fixtures/tau2-bench.json +16 -0
  30. package/fixtures/tau3-banking.json +16 -0
  31. package/fixtures/toollm.json +28 -0
  32. package/fixtures/webarena-verified.json +20 -0
  33. package/package.json +39 -15
  34. package/pier_agents/__init__.py +18 -0
  35. package/pier_agents/candidate_contract.py +755 -0
  36. package/pier_agents/process_boundary.py +321 -0
  37. package/pier_agents/tangle_candidate.py +907 -0
  38. package/pier_agents/workspace_boundary.py +368 -0
  39. package/scripts/appworld_driver.py +359 -0
  40. package/scripts/cadbench_prepare.py +22 -0
  41. package/scripts/cadgenbench_hard_parts.py +48 -0
  42. package/scripts/clbench_codebase_judge.py +73 -0
  43. package/scripts/commit0_judge.py +170 -0
  44. package/scripts/dabstep_judge.py +42 -0
  45. package/scripts/enterpriseops_gym_judge.py +281 -0
  46. package/scripts/programbench_judge.py +120 -0
  47. package/scripts/render-gate-chart.mjs +176 -0
  48. package/scripts/run-package-tests.mjs +56 -0
  49. package/scripts/terminate-pier-trial.mts +66 -0
  50. package/scripts/trata-hedge/README.md +56 -0
  51. package/scripts/trata-hedge/run.sh +60 -0
  52. package/scripts/trata-hedge/solve.py +83 -0
  53. package/scripts/verify-packed-consumer.mjs +224 -0
  54. package/scripts/verify-pier-agent.mts +715 -0
  55. package/scripts/verify-pier-pair.mts +74 -0
  56. package/scripts/verify-pier-recovery.mts +139 -0
  57. package/src/adapters.ts +26 -0
  58. package/src/benchmarks/_harness.test.mts +178 -0
  59. package/src/benchmarks/_harness.ts +239 -16
  60. package/src/benchmarks/agentbench.ts +163 -0
  61. package/src/benchmarks/appworld.test.mts +15 -9
  62. package/src/benchmarks/bfcl.ts +346 -0
  63. package/src/benchmarks/crag.ts +137 -0
  64. package/src/benchmarks/dabstep.test.mts +70 -0
  65. package/src/benchmarks/dabstep.ts +212 -0
  66. package/src/benchmarks/external-adapters.test.mts +150 -0
  67. package/src/benchmarks/finresearchbench.ts +269 -0
  68. package/src/benchmarks/humaneval.ts +20 -8
  69. package/src/benchmarks/nomiracl.ts +180 -0
  70. package/src/benchmarks/open-rag-bench.ts +153 -0
  71. package/src/benchmarks/rag-benchmarks.test.mts +138 -0
  72. package/src/benchmarks/rag-shared.ts +327 -0
  73. package/src/benchmarks/ragbench.ts +171 -0
  74. package/src/benchmarks/swe-bench.test.mts +61 -0
  75. package/src/benchmarks/swe-bench.ts +201 -19
  76. package/src/benchmarks/t2-ragbench.ts +166 -0
  77. package/src/benchmarks/tau-bench-shared.ts +214 -0
  78. package/src/benchmarks/tau2-bench.ts +30 -0
  79. package/src/benchmarks/tau3-banking.ts +29 -0
  80. package/src/benchmarks/terminal-bench.test.mts +33 -0
  81. package/src/benchmarks/terminal-bench.ts +23 -8
  82. package/src/benchmarks/toollm.ts +254 -0
  83. package/src/benchmarks/types.ts +42 -0
  84. package/src/benchmarks/webarena-verified.ts +200 -0
  85. package/src/commit0-prereqs.sh +0 -0
  86. package/src/coordination-mcp-container-reach.mts +181 -0
  87. package/src/decoder-live.mts +1 -1
  88. package/src/examples/README.md +103 -39
  89. package/src/examples/benchmark-matrix.mts +101 -0
  90. package/src/examples/lean-proof-gate.README.md +77 -0
  91. package/src/examples/lean-proof-gate.mts +162 -0
  92. package/src/examples/lean-verify.ts +95 -0
  93. package/src/examples/lean.Dockerfile +12 -0
  94. package/src/examples/math-demo.mts +9 -7
  95. package/src/examples/strategy-demo.mts +10 -12
  96. package/src/gate.ts +3 -2
  97. package/src/hev-eval.mts +69 -0
  98. package/src/hev-improve.mts +169 -0
  99. package/src/hev-structural.mts +688 -0
  100. package/src/index.ts +73 -0
  101. package/src/mbpp-structural.mts +662 -0
  102. package/src/pier-agent.test-fixtures.mts +19 -0
  103. package/src/pier-agent.test.mts +363 -0
  104. package/src/pier-agent.ts +657 -0
  105. package/src/pier-result-grader.mjs +30 -0
  106. package/src/pier-result-grader.test.mts +62 -0
  107. package/src/pier-result-grader.ts +108 -0
  108. package/src/pier-task-outcome.test.mts +117 -0
  109. package/src/pier-task-outcome.ts +240 -0
  110. package/src/pier-trial-controller.test.mts +412 -0
  111. package/src/pier-trial-controller.ts +858 -0
  112. package/src/pier-trial-supervisor.mjs +352 -0
  113. package/src/resolve-client.ts +25 -2
  114. package/src/run-benchmarks-cli.mts +72 -0
  115. package/src/run-benchmarks-report.ts +66 -0
  116. package/src/run-benchmarks.test.mts +231 -0
  117. package/src/run-benchmarks.ts +589 -0
  118. package/src/smoke-structural-rollout.mts +393 -0
  119. package/src/swe-bench-env.test.ts +207 -0
  120. package/src/swe-bench-env.ts +554 -0
  121. package/src/swe-jail.ts +293 -0
  122. package/src/swe-self-improve.mts +84 -0
  123. package/src/swe-structural-judge-policy.test.ts +117 -0
  124. package/src/swe-structural-judge-policy.ts +133 -0
  125. package/src/swe-structural-policy.test.ts +124 -0
  126. package/src/swe-structural-policy.ts +132 -0
  127. package/src/swe-structural-provenance.test.ts +93 -0
  128. package/src/swe-structural-provenance.ts +138 -0
  129. package/src/swe-structural.mts +1260 -0
  130. package/src/swe-temp.ts +14 -0
  131. package/src/tb-container-executor.mts +234 -0
  132. package/src/tb-container-executor.test.mts +99 -0
  133. package/src/tb-supervisor-sidecar.mts +222 -0
  134. package/src/trata-gepa.mts +1 -1
  135. package/steerers/eops-itsm-population.json +1 -0
  136. package/tb_agents/opencode_refine_agent.py +117 -0
  137. package/tb_agents/opencode_router_agent.py +406 -0
  138. package/tb_agents/opencode_supervisor_agent.py +239 -0
  139. package/tb_agents/script_agent.py +66 -0
@@ -0,0 +1,907 @@
1
+ """Execute one runtime-prepared candidate inside a Pier task container.
2
+
3
+ Pier owns the task container, patch transfer, and verifier. Agent Runtime owns
4
+ candidate verification, exact execution bytes, model access, and trace usage.
5
+ This bridge only rechecks the prepared bytes, copies their signed workspaces,
6
+ delivers the exact task instruction, and reports truthful process termination.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import base64
13
+ import json
14
+ import math
15
+ import os
16
+ import re
17
+ import secrets
18
+ import shlex
19
+ import tempfile
20
+ import time
21
+ from dataclasses import dataclass
22
+ from importlib.metadata import PackageNotFoundError, version as package_version
23
+ from pathlib import Path, PurePosixPath
24
+ from typing import Any
25
+
26
+ from pier.agents.base import BaseAgent
27
+ from pier.agents.installed.base import NonZeroAgentExitCodeError
28
+ from pier.environments.base import BaseEnvironment
29
+ from pier.models.agent.context import AgentContext
30
+ from pier.models.agent.network import NetworkAllowlist
31
+
32
+ from .candidate_contract import (
33
+ CandidateContractError,
34
+ PreparedCandidateContract,
35
+ immutable_snapshot,
36
+ load_prepared_candidate_contract,
37
+ sha256_bytes,
38
+ )
39
+ from .process_boundary import CandidateProcessBoundary, ProcessBoundaryConfig
40
+ from .workspace_boundary import (
41
+ CandidateWorkspaceBoundary,
42
+ WorkspaceBoundaryConfig,
43
+ candidate_git_env,
44
+ )
45
+
46
+
47
+ _ADAPTER_VERSION = "2.0.0"
48
+ _ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
49
+ _PROTECTED_TASK_PATH = "/tangle/input/stdin.txt"
50
+ _PIER_PROTECTED_ROOTS = ("/logs", "/tests", "/solution", "/tangle")
51
+ _CANDIDATE_UID = 65532
52
+ _CANDIDATE_GID = 65532
53
+ _EVALUATOR_ROOT = "/tangle"
54
+ _CANDIDATE_HOME = "/tangle/home"
55
+ _CANDIDATE_TMP = "/tangle/tmp"
56
+ _CONTROL_ROOT = "/tangle/control"
57
+ _FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
58
+ _RESERVED_PROCESS_ENV = {
59
+ "GIT_CONFIG_COUNT",
60
+ "GIT_CONFIG_KEY_0",
61
+ "GIT_CONFIG_VALUE_0",
62
+ "GIT_CONFIG_NOSYSTEM",
63
+ "GIT_CONFIG_GLOBAL",
64
+ "GIT_CONFIG_SYSTEM",
65
+ "GIT_NO_REPLACE_OBJECTS",
66
+ "GIT_TERMINAL_PROMPT",
67
+ "HOME",
68
+ "LC_ALL",
69
+ "PATH",
70
+ "TMPDIR",
71
+ }
72
+ _TRACE_ENV_NAMES = {
73
+ "TANGLE_CANDIDATE_EXECUTION_ID",
74
+ "TANGLE_CANDIDATE_BUNDLE_DIGEST",
75
+ "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST",
76
+ "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST",
77
+ "TANGLE_TRACE_RUN_ID",
78
+ }
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class OriginalProfileFile:
83
+ path: str
84
+ existed: bool
85
+ tracked: bool
86
+ mode: int | None
87
+ content: bytes | None
88
+
89
+
90
+ class PierCandidateError(CandidateContractError):
91
+ """The prepared candidate cannot be replayed safely by Pier."""
92
+
93
+
94
+ def _runtime_pier_version() -> str:
95
+ try:
96
+ return package_version("datacurve-pier")
97
+ except PackageNotFoundError as exc:
98
+ raise PierCandidateError(
99
+ "datacurve-pier package metadata is unavailable"
100
+ ) from exc
101
+
102
+
103
+ def _same_or_descendant(path: str, root: str) -> bool:
104
+ return path == root or path.startswith(f"{root}/")
105
+
106
+
107
+ def _join_workspace(root: str, relative: str) -> str:
108
+ root_path = PurePosixPath(root)
109
+ joined = root_path if relative == "." else root_path / PurePosixPath(relative)
110
+ if joined != root_path and root_path not in joined.parents:
111
+ raise PierCandidateError(f"cwd escapes its workspace: {relative}")
112
+ return str(joined)
113
+
114
+
115
+ def _trace_env(
116
+ contract: PreparedCandidateContract, trace_run_id: str
117
+ ) -> dict[str, str]:
118
+ return {
119
+ "TANGLE_CANDIDATE_EXECUTION_ID": contract.execution_id,
120
+ "TANGLE_CANDIDATE_BUNDLE_DIGEST": contract.bundle_digest,
121
+ "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": contract.execution_plan_digest,
122
+ "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": contract.materialization_receipt_digest,
123
+ "TANGLE_TRACE_RUN_ID": trace_run_id,
124
+ }
125
+
126
+
127
+ class TangleCandidateAgent(BaseAgent):
128
+ """Pier adapter for a branded ``PreparedAgentCandidateExecution``."""
129
+
130
+ SUPPORTS_ATIF = False
131
+ SUPPORTS_WINDOWS = False
132
+ ISOLATE_CANDIDATE_USER = True
133
+
134
+ def __init__(
135
+ self,
136
+ logs_dir: Path,
137
+ model_name: str | None = None,
138
+ plan_path: str | None = None,
139
+ receipt_path: str | None = None,
140
+ expected_receipt_digest: str | None = None,
141
+ trace_run_id: str | None = None,
142
+ task_dir: str | None = None,
143
+ profile_dir: str | None = None,
144
+ candidate_dir: str | None = None,
145
+ pier_version: str | None = None,
146
+ extra_env: dict[str, str] | None = None,
147
+ *args: Any,
148
+ **kwargs: Any,
149
+ ) -> None:
150
+ super().__init__(logs_dir=logs_dir, model_name=model_name, *args, **kwargs)
151
+ if (
152
+ not plan_path
153
+ or not receipt_path
154
+ or not expected_receipt_digest
155
+ or not trace_run_id
156
+ ):
157
+ raise PierCandidateError(
158
+ "plan_path, receipt_path, expected_receipt_digest, and trace_run_id are required"
159
+ )
160
+ if not task_dir or not profile_dir or not pier_version:
161
+ raise PierCandidateError("task_dir, profile_dir, and pier_version are required")
162
+
163
+ self._expected_pier_version = pier_version
164
+ self._contract = load_prepared_candidate_contract(
165
+ Path(plan_path).expanduser().resolve(),
166
+ Path(receipt_path).expanduser().resolve(),
167
+ expected_receipt_digest,
168
+ )
169
+ if model_name != self._contract.requested_model:
170
+ raise PierCandidateError(
171
+ "Pier model does not match the runtime-resolved request: "
172
+ f"expected={self._contract.requested_model} observed={model_name}"
173
+ )
174
+ self._extra_env = self._validate_protected_env(
175
+ extra_env or {}, trace_run_id
176
+ )
177
+ self._validate_execution_roots()
178
+
179
+ self._snapshots = tempfile.TemporaryDirectory(
180
+ prefix="agent-bench-pier-candidate-"
181
+ )
182
+ snapshot_root = Path(self._snapshots.name)
183
+ self._task_snapshot = snapshot_root / "task"
184
+ self._profile_snapshot = snapshot_root / "profile"
185
+ self._candidate_snapshot: Path | None = None
186
+ try:
187
+ immutable_snapshot(
188
+ Path(os.path.abspath(Path(task_dir).expanduser())),
189
+ self._task_snapshot,
190
+ self._contract.task_files,
191
+ "prepared task directory",
192
+ )
193
+ immutable_snapshot(
194
+ Path(os.path.abspath(Path(profile_dir).expanduser())),
195
+ self._profile_snapshot,
196
+ self._contract.profile_files,
197
+ "prepared profile directory",
198
+ )
199
+ if self._contract.candidate_root is not None:
200
+ if not candidate_dir:
201
+ raise PierCandidateError("active code requires candidate_dir")
202
+ self._candidate_snapshot = snapshot_root / "candidate"
203
+ immutable_snapshot(
204
+ Path(os.path.abspath(Path(candidate_dir).expanduser())),
205
+ self._candidate_snapshot,
206
+ self._contract.candidate_files,
207
+ "prepared candidate directory",
208
+ )
209
+ elif candidate_dir:
210
+ raise PierCandidateError("disabled code cannot receive candidate_dir")
211
+ except Exception:
212
+ self._snapshots.cleanup()
213
+ raise
214
+
215
+ self._original_profile_files: tuple[OriginalProfileFile, ...] = ()
216
+ self._process_boundary = CandidateProcessBoundary(
217
+ self,
218
+ ProcessBoundaryConfig(
219
+ uid=_CANDIDATE_UID,
220
+ gid=_CANDIDATE_GID,
221
+ evaluator_root=_EVALUATOR_ROOT,
222
+ control_root=_CONTROL_ROOT,
223
+ home=_CANDIDATE_HOME,
224
+ temporary=_CANDIDATE_TMP,
225
+ isolate_user=self.ISOLATE_CANDIDATE_USER,
226
+ ),
227
+ )
228
+ self._workspace_boundary = CandidateWorkspaceBoundary(
229
+ self,
230
+ WorkspaceBoundaryConfig(
231
+ contract=self._contract,
232
+ task_snapshot=self._task_snapshot,
233
+ control_root=_CONTROL_ROOT,
234
+ protected_roots=_PIER_PROTECTED_ROOTS,
235
+ error_type=PierCandidateError,
236
+ ),
237
+ )
238
+
239
+ def _validate_protected_env(
240
+ self, values: dict[str, str], trace_run_id: str
241
+ ) -> dict[str, str]:
242
+ protected: dict[str, str] = {}
243
+ public_names = set(self._contract.env)
244
+ reserved_public = sorted(public_names & _RESERVED_PROCESS_ENV)
245
+ if reserved_public:
246
+ raise PierCandidateError(
247
+ "signed public env collides with evaluator Git safety fields: "
248
+ + ", ".join(reserved_public)
249
+ )
250
+ expected_trace = _trace_env(self._contract, trace_run_id)
251
+ for name, expected in expected_trace.items():
252
+ if values.get(name) != expected:
253
+ raise PierCandidateError(
254
+ f"evaluator trace environment is missing or mismatched: {name}"
255
+ )
256
+ for name, value in values.items():
257
+ if not _ENV_RE.fullmatch(name) or name in _RESERVED_PROCESS_ENV:
258
+ raise PierCandidateError(
259
+ f"invalid protected model environment name: {name}"
260
+ )
261
+ if name in public_names:
262
+ raise PierCandidateError(
263
+ f"protected model environment overlaps signed public env: {name}"
264
+ )
265
+ if name.startswith("TANGLE_CANDIDATE_") and name not in _TRACE_ENV_NAMES:
266
+ raise PierCandidateError(
267
+ f"unknown evaluator candidate environment: {name}"
268
+ )
269
+ if not isinstance(value, str) or not value or "\0" in value:
270
+ raise PierCandidateError(
271
+ f"protected model environment value is empty or invalid: {name}"
272
+ )
273
+ protected[name] = value
274
+ return protected
275
+
276
+ def _validate_execution_roots(self) -> None:
277
+ roots = [self._contract.task_root]
278
+ if self._contract.candidate_root is not None:
279
+ roots.append(self._contract.candidate_root)
280
+ for root in roots:
281
+ protected = next(
282
+ (
283
+ path
284
+ for path in _PIER_PROTECTED_ROOTS
285
+ if _same_or_descendant(root, path)
286
+ ),
287
+ None,
288
+ )
289
+ if protected is not None:
290
+ raise PierCandidateError(
291
+ f"execution workspace cannot be below Pier-owned path {protected}"
292
+ )
293
+
294
+ def name(self) -> str:
295
+ short = self._contract.bundle_digest.removeprefix("sha256:")[:16]
296
+ return f"tangle-candidate-{short}"
297
+
298
+ def version(self) -> str:
299
+ return _ADAPTER_VERSION
300
+
301
+ def network_allowlist(self) -> NetworkAllowlist:
302
+ return NetworkAllowlist(domains=list(self._contract.model_network_domains))
303
+
304
+ async def _exec(
305
+ self,
306
+ environment: BaseEnvironment,
307
+ command: str,
308
+ *,
309
+ cwd: str | None = None,
310
+ env: dict[str, str] | None = None,
311
+ user: str | int | None = None,
312
+ timeout_sec: float | None = None,
313
+ check: bool = True,
314
+ ) -> Any:
315
+ result = await environment.exec(
316
+ command=command,
317
+ cwd=cwd,
318
+ env=environment.agent_process_env(env),
319
+ user=user,
320
+ timeout_sec=timeout_sec,
321
+ )
322
+ if check and result.return_code != 0:
323
+ raise PierCandidateError(
324
+ f"candidate bridge command failed ({result.return_code}): {command}; "
325
+ f"stderr={(result.stderr or '')[:500]}"
326
+ )
327
+ return result
328
+
329
+ async def _assert_real_directory(
330
+ self,
331
+ environment: BaseEnvironment,
332
+ path: str,
333
+ *,
334
+ label: str,
335
+ user: str | int | None = "root",
336
+ ) -> None:
337
+ quoted = shlex.quote(path)
338
+ await self._exec(
339
+ environment,
340
+ "\n".join(
341
+ [
342
+ "set -eu",
343
+ f"test -d {quoted}",
344
+ f"test ! -L {quoted}",
345
+ f'test "$(realpath -e -- {quoted})" = {quoted}',
346
+ ]
347
+ ),
348
+ user=user,
349
+ )
350
+
351
+ async def _ensure_real_directory(
352
+ self,
353
+ environment: BaseEnvironment,
354
+ path: str,
355
+ *,
356
+ anchor: str,
357
+ mode: int = 0o755,
358
+ ) -> None:
359
+ parsed = PurePosixPath(path)
360
+ parsed_anchor = PurePosixPath(anchor)
361
+ if parsed != parsed_anchor and parsed_anchor not in parsed.parents:
362
+ raise PierCandidateError(f"directory escapes its trusted anchor: {path}")
363
+ await self._assert_real_directory(
364
+ environment, anchor, label="trusted directory anchor"
365
+ )
366
+ current = parsed_anchor
367
+ relative_parts = parsed.relative_to(parsed_anchor).parts
368
+ for part in relative_parts:
369
+ current /= part
370
+ value = str(current)
371
+ quoted = shlex.quote(value)
372
+ await self._exec(
373
+ environment,
374
+ "\n".join(
375
+ [
376
+ "set -eu",
377
+ f"if test -e {quoted} || test -L {quoted}; then",
378
+ f" test -d {quoted}",
379
+ f" test ! -L {quoted}",
380
+ "else",
381
+ f" mkdir -- {quoted}",
382
+ f" chmod {mode:o} -- {quoted}",
383
+ "fi",
384
+ f'test "$(realpath -e -- {quoted})" = {quoted}',
385
+ ]
386
+ ),
387
+ user="root",
388
+ )
389
+
390
+ async def _write_verified_file(
391
+ self,
392
+ environment: BaseEnvironment,
393
+ source: Path,
394
+ target: str,
395
+ *,
396
+ anchor: str,
397
+ mode: int,
398
+ digest: str,
399
+ byte_length: int,
400
+ ) -> None:
401
+ parent = str(PurePosixPath(target).parent)
402
+ await self._ensure_real_directory(environment, parent, anchor=anchor)
403
+ temporary = f"{parent}/.tangle-{secrets.token_hex(24)}"
404
+ quoted_temporary = shlex.quote(temporary)
405
+ quoted_target = shlex.quote(target)
406
+ expected_hash = digest.removeprefix("sha256:")
407
+ await self._exec(
408
+ environment,
409
+ f"test ! -e {quoted_temporary} && test ! -L {quoted_temporary}",
410
+ user="root",
411
+ )
412
+ try:
413
+ await environment.upload_file(source, temporary)
414
+ await self._exec(
415
+ environment,
416
+ "\n".join(
417
+ [
418
+ "set -eu",
419
+ f"test -f {quoted_temporary}",
420
+ f"test ! -L {quoted_temporary}",
421
+ f'test "$(stat -c %h -- {quoted_temporary})" = 1',
422
+ f'test "$(wc -c < {quoted_temporary})" = {byte_length}',
423
+ f"test \"$(sha256sum {quoted_temporary} | cut -d' ' -f1)\" = {expected_hash}",
424
+ f"chmod {mode:o} -- {quoted_temporary}",
425
+ f'test "$(stat -c %a -- {quoted_temporary})" = {mode:o}',
426
+ f"rm -rf -- {quoted_target}",
427
+ f"mv -fT -- {quoted_temporary} {quoted_target}",
428
+ f"test -f {quoted_target}",
429
+ f"test ! -L {quoted_target}",
430
+ f'test "$(stat -c %h -- {quoted_target})" = 1',
431
+ f'test "$(stat -c %a -- {quoted_target})" = {mode:o}',
432
+ f'test "$(wc -c < {quoted_target})" = {byte_length}',
433
+ f"test \"$(sha256sum {quoted_target} | cut -d' ' -f1)\" = {expected_hash}",
434
+ ]
435
+ ),
436
+ user="root",
437
+ )
438
+ finally:
439
+ await self._exec(
440
+ environment,
441
+ f"rm -f -- {quoted_temporary}",
442
+ user="root",
443
+ check=False,
444
+ )
445
+
446
+ async def _verify_pier_execution_identity(
447
+ self, environment: BaseEnvironment
448
+ ) -> None:
449
+ await self._workspace_boundary.verify_pier_execution_identity(environment)
450
+
451
+ async def _capture_original_profile_files(
452
+ self, environment: BaseEnvironment
453
+ ) -> tuple[OriginalProfileFile, ...]:
454
+ if self._contract.profile_target != "task":
455
+ return ()
456
+ originals: list[OriginalProfileFile] = []
457
+ root = self._contract.task_root
458
+ git = f"git -c core.hooksPath=/dev/null -C {shlex.quote(root)}"
459
+ for file in self._contract.profile_files:
460
+ target = f"{root}/{file.path}"
461
+ quoted = shlex.quote(target)
462
+ parent = shlex.quote(str(PurePosixPath(target).parent))
463
+ presence = await self._exec(
464
+ environment,
465
+ "\n".join(
466
+ [
467
+ "set -eu",
468
+ f'test "$(realpath -m -- {parent})" = {parent}',
469
+ f"if test -e {quoted} || test -L {quoted}; then",
470
+ f" test -f {quoted}",
471
+ f" test ! -L {quoted}",
472
+ f' test "$(stat -c %h -- {quoted})" = 1',
473
+ "else",
474
+ " exit 2",
475
+ "fi",
476
+ ]
477
+ ),
478
+ user="root",
479
+ check=False,
480
+ )
481
+ tracked = (
482
+ await self._exec(
483
+ environment,
484
+ f"{git} ls-files --error-unmatch -- {shlex.quote(file.path)}",
485
+ env=candidate_git_env(root),
486
+ user="root",
487
+ check=False,
488
+ )
489
+ ).return_code == 0
490
+ if presence.return_code == 2:
491
+ originals.append(
492
+ OriginalProfileFile(file.path, False, tracked, None, None)
493
+ )
494
+ continue
495
+ if presence.return_code != 0:
496
+ raise PierCandidateError(
497
+ f"profile mount target is not one regular file: {file.path}"
498
+ )
499
+ encoded = await self._exec(
500
+ environment, f"base64 -w0 -- {quoted}", user="root"
501
+ )
502
+ mode = await self._exec(environment, f"stat -c %a -- {quoted}", user="root")
503
+ try:
504
+ content = base64.b64decode(
505
+ (encoded.stdout or "").strip(), validate=True
506
+ )
507
+ parsed_mode = int((mode.stdout or "").strip(), 8)
508
+ except (ValueError, TypeError) as exc:
509
+ raise PierCandidateError(
510
+ f"could not capture original profile target: {file.path}"
511
+ ) from exc
512
+ originals.append(
513
+ OriginalProfileFile(file.path, True, tracked, parsed_mode, content)
514
+ )
515
+ return tuple(originals)
516
+
517
+ async def _apply_profile(self, environment: BaseEnvironment) -> None:
518
+ target_root = (
519
+ self._contract.task_root
520
+ if self._contract.profile_target == "task"
521
+ else self._contract.candidate_root
522
+ )
523
+ if target_root is None:
524
+ raise PierCandidateError("profile target workspace is unavailable")
525
+ self._original_profile_files = await self._capture_original_profile_files(
526
+ environment
527
+ )
528
+ for file in self._contract.profile_files:
529
+ target = f"{target_root}/{file.path}"
530
+ source = self._profile_snapshot / file.path
531
+ await self._write_verified_file(
532
+ environment,
533
+ source,
534
+ target,
535
+ anchor=target_root,
536
+ mode=file.mode,
537
+ digest=file.sha256,
538
+ byte_length=source.stat().st_size,
539
+ )
540
+
541
+ async def setup(self, environment: BaseEnvironment) -> None:
542
+ try:
543
+ observed_pier = _runtime_pier_version()
544
+ if observed_pier != self._expected_pier_version:
545
+ raise PierCandidateError(
546
+ f"Pier version mismatch: expected={self._expected_pier_version} "
547
+ f"installed={observed_pier}"
548
+ )
549
+ await self._verify_pier_execution_identity(environment)
550
+ await self._workspace_boundary.materialize_task_workspace(environment)
551
+ await self._workspace_boundary.verify_repository(environment)
552
+ await self._workspace_boundary.verify_container_workspace(
553
+ environment,
554
+ self._contract.task_root,
555
+ self._contract.task_files,
556
+ allow_task_metadata=True,
557
+ )
558
+ if self._contract.candidate_root is not None:
559
+ if self._candidate_snapshot is None:
560
+ raise PierCandidateError("candidate snapshot is unavailable")
561
+ quoted = shlex.quote(self._contract.candidate_root)
562
+ candidate_parent = str(
563
+ PurePosixPath(self._contract.candidate_root).parent
564
+ )
565
+ await self._ensure_real_directory(
566
+ environment, candidate_parent, anchor="/"
567
+ )
568
+ await self._exec(
569
+ environment,
570
+ "\n".join(
571
+ [
572
+ "set -eu",
573
+ f"test ! -e {quoted}",
574
+ f"test ! -L {quoted}",
575
+ f"mkdir -- {quoted}",
576
+ f'test "$(realpath -e -- {quoted})" = {quoted}',
577
+ ]
578
+ ),
579
+ user="root",
580
+ )
581
+ await self._workspace_boundary.verify_resolved_root_isolation(
582
+ environment
583
+ )
584
+ await environment.upload_dir(
585
+ self._candidate_snapshot, self._contract.candidate_root
586
+ )
587
+ await self._workspace_boundary.verify_container_workspace(
588
+ environment,
589
+ self._contract.candidate_root,
590
+ self._contract.candidate_files,
591
+ allow_task_metadata=False,
592
+ )
593
+ else:
594
+ await self._workspace_boundary.verify_resolved_root_isolation(
595
+ environment
596
+ )
597
+ workspace_roots = (self._contract.task_root,) + (
598
+ (self._contract.candidate_root,)
599
+ if self._contract.candidate_root is not None
600
+ else ()
601
+ )
602
+ await self._process_boundary.prepare_identity(environment, workspace_roots)
603
+ await self._apply_profile(environment)
604
+
605
+ self.logs_dir.mkdir(parents=True, exist_ok=True)
606
+ (self.logs_dir / "execution-plan.json").write_bytes(self._contract.plan_raw)
607
+ (self.logs_dir / "materialization-receipt.json").write_bytes(
608
+ self._contract.receipt_raw
609
+ )
610
+ finally:
611
+ self._snapshots.cleanup()
612
+
613
+ def _identity_metadata(self) -> dict[str, Any]:
614
+ return {
615
+ "executionId": self._contract.execution_id,
616
+ "bundleDigest": self._contract.bundle_digest,
617
+ "executionPlanDigest": self._contract.execution_plan_digest,
618
+ "materializationReceiptDigest": self._contract.materialization_receipt_digest,
619
+ "usageSource": "protected-agent-eval-trace",
620
+ }
621
+
622
+ async def _upload_instruction(
623
+ self, environment: BaseEnvironment, instruction: str
624
+ ) -> tuple[list[str], str | None, str | None]:
625
+ try:
626
+ raw = instruction.encode("utf-8")
627
+ except UnicodeEncodeError as exc:
628
+ raise PierCandidateError("Pier instruction is not valid UTF-8") from exc
629
+ evidence = self._contract.instruction
630
+ if len(raw) != evidence.byte_length or sha256_bytes(raw) != evidence.sha256:
631
+ raise PierCandidateError(
632
+ "Pier instruction bytes do not match the signed task instruction"
633
+ )
634
+
635
+ argv = [self._contract.executable, *self._contract.args]
636
+ if evidence.delivery_kind == "argv-append":
637
+ argv.append(instruction)
638
+ return argv, None, None
639
+
640
+ target = evidence.delivery_path or _PROTECTED_TASK_PATH
641
+ with tempfile.NamedTemporaryFile(delete=False) as handle:
642
+ handle.write(raw)
643
+ local_path = Path(handle.name)
644
+ try:
645
+ await self._write_verified_file(
646
+ environment,
647
+ local_path,
648
+ target,
649
+ anchor="/",
650
+ mode=0o644,
651
+ digest=evidence.sha256,
652
+ byte_length=len(raw),
653
+ )
654
+ finally:
655
+ local_path.unlink(missing_ok=True)
656
+ return (
657
+ argv,
658
+ target if evidence.delivery_kind == "stdin-utf8" else None,
659
+ target,
660
+ )
661
+
662
+ async def _remove_instruction_file(
663
+ self, environment: BaseEnvironment, path: str | None
664
+ ) -> None:
665
+ if path is None:
666
+ return
667
+ parent = str(PurePosixPath(path).parent)
668
+ await self._assert_real_directory(
669
+ environment, parent, label="protected instruction directory"
670
+ )
671
+ await self._exec(
672
+ environment,
673
+ f"rm -f -- {shlex.quote(path)}",
674
+ user="root",
675
+ )
676
+
677
+ async def _restore_profile_and_capture_solution(
678
+ self, environment: BaseEnvironment
679
+ ) -> None:
680
+ root = self._contract.task_root
681
+ quoted_root = shlex.quote(root)
682
+ git = f"git -c core.hooksPath=/dev/null -C {quoted_root}"
683
+ git_env = candidate_git_env(root)
684
+
685
+ # Ignore candidate-authored commits and derive one evaluator-owned final
686
+ # tree from the actual working files relative to the signed base commit.
687
+ await self._exec(
688
+ environment,
689
+ f"{git} reset --mixed {shlex.quote(self._contract.repository_base_commit)}",
690
+ env=git_env,
691
+ user="root",
692
+ )
693
+ for original in self._original_profile_files:
694
+ target = f"{root}/{original.path}"
695
+ quoted = shlex.quote(target)
696
+ if original.existed:
697
+ if original.content is None or original.mode is None:
698
+ raise PierCandidateError(
699
+ f"profile backup is incomplete: {original.path}"
700
+ )
701
+ with tempfile.NamedTemporaryFile(delete=False) as handle:
702
+ handle.write(original.content)
703
+ local_path = Path(handle.name)
704
+ try:
705
+ await self._write_verified_file(
706
+ environment,
707
+ local_path,
708
+ target,
709
+ anchor=root,
710
+ mode=original.mode,
711
+ digest=sha256_bytes(original.content),
712
+ byte_length=len(original.content),
713
+ )
714
+ finally:
715
+ local_path.unlink(missing_ok=True)
716
+ else:
717
+ parent = str(PurePosixPath(target).parent)
718
+ await self._ensure_real_directory(environment, parent, anchor=root)
719
+ await self._exec(environment, f"rm -rf -- {quoted}", user="root")
720
+
721
+ await self._exec(environment, f"{git} add -A -- .", env=git_env, user="root")
722
+ for original in self._original_profile_files:
723
+ if original.tracked:
724
+ continue
725
+ await self._exec(
726
+ environment,
727
+ f"{git} rm -r -f --cached --ignore-unmatch -- {shlex.quote(original.path)}",
728
+ env=git_env,
729
+ user="root",
730
+ )
731
+ staged = await self._exec(
732
+ environment,
733
+ f"{git} diff --cached --quiet",
734
+ env=git_env,
735
+ user="root",
736
+ check=False,
737
+ )
738
+ if staged.return_code not in {0, 1}:
739
+ raise PierCandidateError("could not inspect the sanitized solution tree")
740
+ if staged.return_code == 1:
741
+ commit_env = {
742
+ **git_env,
743
+ "GIT_AUTHOR_NAME": "Tangle Evaluator",
744
+ "GIT_AUTHOR_EMAIL": "evaluator@tangle.tools",
745
+ "GIT_COMMITTER_NAME": "Tangle Evaluator",
746
+ "GIT_COMMITTER_EMAIL": "evaluator@tangle.tools",
747
+ }
748
+ await self._exec(
749
+ environment,
750
+ f"{git} commit -m 'capture candidate solution'",
751
+ env=commit_env,
752
+ user="root",
753
+ )
754
+
755
+ async def run(
756
+ self,
757
+ instruction: str,
758
+ environment: BaseEnvironment,
759
+ context: AgentContext,
760
+ ) -> None:
761
+ argv, stdin_path, instruction_path = await self._upload_instruction(
762
+ environment, instruction
763
+ )
764
+ cwd = _join_workspace(self._contract.cwd_root, self._contract.cwd_path)
765
+ await self._exec(environment, f"test -d {shlex.quote(cwd)}")
766
+
767
+ process_env = {
768
+ **self._extra_env,
769
+ **candidate_git_env(self._contract.task_root),
770
+ "HOME": _CANDIDATE_HOME,
771
+ "TMPDIR": _CANDIDATE_TMP,
772
+ "PATH": _FIXED_PATH,
773
+ }
774
+ wrapper, timeout_marker = await self._process_boundary.stage_wrapper(
775
+ environment
776
+ )
777
+ timeout_seconds = self._contract.timeout_ms / 1000
778
+ command = shlex.join(
779
+ [
780
+ wrapper,
781
+ timeout_marker,
782
+ str(_CANDIDATE_UID),
783
+ str(_CANDIDATE_GID),
784
+ f"{timeout_seconds:.3f}",
785
+ "2",
786
+ stdin_path or "-",
787
+ *(f"{name}={value}" for name, value in self._contract.env.items()),
788
+ *argv,
789
+ ]
790
+ )
791
+ context.metadata = {
792
+ **self._identity_metadata(),
793
+ "termination": {"kind": "running"},
794
+ }
795
+
796
+ started = time.monotonic()
797
+ try:
798
+ result = await environment.exec(
799
+ command=command,
800
+ cwd=cwd,
801
+ env=environment.agent_process_env(process_env),
802
+ user="root",
803
+ timeout_sec=math.ceil(timeout_seconds) + 12,
804
+ )
805
+ except (asyncio.CancelledError, asyncio.TimeoutError):
806
+ elapsed = int((time.monotonic() - started) * 1000)
807
+ cleanup_errors: list[str] = []
808
+ try:
809
+ await asyncio.shield(
810
+ self._process_boundary.kill_and_wait(environment)
811
+ )
812
+ except Exception as cleanup_exc:
813
+ cleanup_errors.append(str(cleanup_exc))
814
+ try:
815
+ await asyncio.shield(
816
+ self._remove_instruction_file(environment, instruction_path)
817
+ )
818
+ except Exception as cleanup_exc:
819
+ cleanup_errors.append(str(cleanup_exc))
820
+ try:
821
+ await asyncio.shield(
822
+ self._restore_profile_and_capture_solution(environment)
823
+ )
824
+ except (
825
+ Exception
826
+ ) as cleanup_exc: # cleanup failure is evidence, never hidden
827
+ cleanup_errors.append(str(cleanup_exc))
828
+ try:
829
+ await asyncio.shield(
830
+ self._process_boundary.remove_control_files(
831
+ environment, wrapper, timeout_marker
832
+ )
833
+ )
834
+ except Exception as cleanup_exc:
835
+ cleanup_errors.append(str(cleanup_exc))
836
+ context.metadata = {
837
+ **self._identity_metadata(),
838
+ "termination": {"kind": "cancelled", "observedElapsedMs": elapsed},
839
+ **(
840
+ {"cleanupError": "; ".join(cleanup_errors)}
841
+ if cleanup_errors
842
+ else {}
843
+ ),
844
+ }
845
+ raise
846
+ except Exception:
847
+ await asyncio.shield(self._process_boundary.kill_and_wait(environment))
848
+ await asyncio.shield(
849
+ self._remove_instruction_file(environment, instruction_path)
850
+ )
851
+ await asyncio.shield(
852
+ self._restore_profile_and_capture_solution(environment)
853
+ )
854
+ await asyncio.shield(
855
+ self._process_boundary.remove_control_files(
856
+ environment, wrapper, timeout_marker
857
+ )
858
+ )
859
+ raise
860
+
861
+ elapsed = int((time.monotonic() - started) * 1000)
862
+ timed_out = await self._process_boundary.timeout_marker_exists(
863
+ environment, timeout_marker
864
+ )
865
+ await self._process_boundary.kill_and_wait(environment)
866
+ await self._remove_instruction_file(environment, instruction_path)
867
+ await self._restore_profile_and_capture_solution(environment)
868
+ await self._process_boundary.remove_control_files(
869
+ environment, wrapper, timeout_marker
870
+ )
871
+ self.logs_dir.mkdir(parents=True, exist_ok=True)
872
+ (self.logs_dir / "stdout.txt").write_text(result.stdout or "", encoding="utf-8")
873
+ (self.logs_dir / "stderr.txt").write_text(result.stderr or "", encoding="utf-8")
874
+ (self.logs_dir / "process.json").write_text(
875
+ json.dumps(
876
+ {
877
+ "executionId": self._contract.execution_id,
878
+ "exitCode": result.return_code,
879
+ "elapsedMs": elapsed,
880
+ "cwd": cwd,
881
+ "argv": argv,
882
+ "instructionDelivery": self._contract.instruction.delivery_kind,
883
+ },
884
+ indent=2,
885
+ )
886
+ + "\n",
887
+ encoding="utf-8",
888
+ )
889
+ context.metadata = {
890
+ **self._identity_metadata(),
891
+ "termination": (
892
+ {"kind": "timeout", "timeoutMs": self._contract.timeout_ms}
893
+ if timed_out
894
+ else {"kind": "exit", "exitCode": result.return_code}
895
+ ),
896
+ "observedElapsedMs": elapsed,
897
+ }
898
+ if timed_out:
899
+ raise asyncio.TimeoutError(
900
+ f"candidate exceeded signed timeout {self._contract.timeout_ms}ms"
901
+ )
902
+ if result.return_code != 0:
903
+ raise NonZeroAgentExitCodeError(
904
+ f"candidate exited {result.return_code}: "
905
+ f"stdout={(result.stdout or '')[:500]} "
906
+ f"stderr={(result.stderr or '')[:500]}"
907
+ )