@tangle-network/agent-bench 0.1.0 → 0.3.5

package/scripts/terminate-pier-trial.mts

@@ -0,0 +1,66 @@
+import path from 'node:path'
+
+import { InMemoryTraceStore } from '@tangle-network/agent-eval'
+
+import { createPierCandidateRecoveryExecutor } from '../src/pier-agent'
+import { FilePierCandidateTrialController } from '../src/pier-trial-controller'
+
+const [directoryArg, executionId, executionPlanDigest, ...extra] = process.argv.slice(2)
+if (!directoryArg || !executionId || !executionPlanDigest || extra.length > 0) {
+  throw new Error(
+    'usage: terminate-pier-trial.mts <control-directory> <execution-id> <execution-plan-digest>',
+  )
+}
+if (!/^sha256:[a-f0-9]{64}$/.test(executionPlanDigest)) {
+  throw new Error('execution-plan-digest must be a SHA-256 digest')
+}
+
+const dockerConnectionId = process.env.PIER_DOCKER_CONNECTION_ID
+const dockerEnvironmentNames = process.env.PIER_DOCKER_ENV_NAMES
+if ((dockerConnectionId === undefined) !== (dockerEnvironmentNames === undefined)) {
+  throw new Error(
+    'PIER_DOCKER_CONNECTION_ID and PIER_DOCKER_ENV_NAMES must be set together',
+  )
+}
+const dockerConnection = (() => {
+  if (dockerConnectionId === undefined || dockerEnvironmentNames === undefined) return undefined
+  const names = dockerEnvironmentNames
+    .split(',')
+    .map((name) => name.trim())
+    .filter(Boolean)
+  if (new Set(names).size !== names.length) {
+    throw new Error('PIER_DOCKER_ENV_NAMES must not contain duplicates')
+  }
+  const env: Record<string, string> = {}
+  for (const name of names) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+      throw new Error('PIER_DOCKER_ENV_NAMES contains an invalid environment name')
+    }
+    const value = process.env[name]
+    if (value === undefined) throw new Error(`required Docker environment variable ${name} is unset`)
+    env[name] = value
+  }
+  return { id: dockerConnectionId, env }
+})()
+
+const controller = new FilePierCandidateTrialController({
+  directory: path.resolve(directoryArg),
+  ...(dockerConnection ? { dockerConnection } : {}),
+})
+const executor = createPierCandidateRecoveryExecutor(controller)
+const recovered = await executor.stop(
+  {
+    executionId,
+    executionPlanDigest: executionPlanDigest as `sha256:${string}`,
+  },
+  {
+    traceStore: new InMemoryTraceStore(),
+    reason: 'failed',
+    signal: new AbortController().signal,
+    deadlineAtMs: Date.now() + 30_000,
+  },
+)
+if (recovered.stopped !== true) throw new Error('Pier recovery executor did not stop the trial')
+process.stdout.write(
+  `${JSON.stringify({ processExited: true, containersRemoved: true })}\n`,
+)

package/scripts/trata-hedge/README.md

@@ -0,0 +1,56 @@
+# trata-hedge-bench — evaluating our system against an LLM-judge benchmark
+
+[trata-hedge-bench](https://github.com/Trata-Inc/trata-hedge-bench) is a 102-task
+financial-analyst benchmark (Harbor format): the agent reads a company's data corpus
+(earnings calls, 10-K/10-Q, financials, press releases) and writes an analysis; the
+verifier is a **Gemini-3.1-pro 3-task cascade** (hallucination-check → per-move-hit →
+synthesis) that grades *concept match* against the expert analyst's documented moves
+(`ground_truth.txt`). Graded `score` is 0–4 (themes fully covered); `reward.txt` is
+sparse (1 iff all themes). No deployable ground-truth checker — it's an **oracle judge**.
+
+## What's admissible here (and what isn't)
+
+- **NOT** the verifier-grounded selector gate (our HumanEval/commit0 headline). The only
+  checker is the judge itself, so selecting by it = selecting by the eval metric (an
+  oracle, a Goodhart trap). See `docs/results.md`.
+- **IS** admissible: (1) more-compute headroom (pass@k), (2) the **improvement loop**
+  (`selfImprove`/GEPA optimizing the analyst directive *against this judge*, held-out
+  gated — the legitimate use of a judge domain), with the honest caveat that an
+  LLM-judge can be *gamed*, so a held-out lift means "higher judge score," which without
+  ground truth we can't independently certify as "better analysis."
+
+## Status (2026-06-06): pipeline PROVEN end-to-end
+
+Our solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link
+works. The naive **single-shot** baseline (gpt-4o, ~3 of N corpus files in one context
+window) scores **0/4** — a floor: it hit only 1/3 moves on a few themes with
+hallucinations flagged, because it could not explore the full corpus. The bench is built
+for **agentic** exploration; a fair baseline needs our sandbox runtime as the solver
+(browse + cite selectively), like the commit0 gate — pending sandbox-gateway health.
+
+## Run it
+
+```bash
+# clone the bench once (102 envs, ~580MB; or sparse-checkout one env)
+git clone https://github.com/Trata-Inc/trata-hedge-bench /tmp/thb
+
+dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
+  bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/<env-name> gpt-4o
+```
+
+## Gotchas (each cost a debugging cycle)
+
+- **Use `GOOGLE_AI_KEY`, not `GEMINI_API_KEY`** — the latter in the secrets is
+  `API_KEY_INVALID`; `GOOGLE_AI_KEY` is the working Gemini key (`gemini-3.1-pro-preview`
+  returns 200). `run.sh` maps `GOOGLE_API_KEY=$GOOGLE_AI_KEY` for `grade.py`.
+- The router is behind Cloudflare bot-fight: a default urllib User-Agent → 403 (CF 1010)
+  on large bodies; the solver sends a browser UA.
+- `grade.py` hardcodes `/app/answer.txt` (only that path); `/app` is sudo-writable.
+  `DATA_DIR` / `GROUND_TRUTH_PATH` / `REWARD_PATH` / `DETAILS_PATH` are env-overridable, so
+  the grader runs **unmodified** (faithful).
+
+## Next steps
+
+1. Agentic solver: our sandbox runtime browses the corpus + cites (fair baseline).
+2. The improvement loop: `selfImprove` on the analyst directive vs this judge, held-out
+   gated, across the 102 tasks — with the judge-gaming caveat stated.

package/scripts/trata-hedge/run.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Evaluate our system against trata-hedge-bench's OWN Gemini-3.1-pro judge.
+#
+# Pipeline: our solver writes a cited analysis -> their grade.py (unmodified, a
+# 3-task Gemini cascade: hallucination-check -> per-move-hit -> synthesis) scores it
+# against the expert analyst's documented moves (ground_truth.txt).
+#
+# Prereqs (all present in this environment):
+#   - GOOGLE_AI_KEY  — a VALID Gemini key (NOTE: GEMINI_API_KEY in the secrets is
+#                      API_KEY_INVALID; GOOGLE_AI_KEY is the working one).
+#   - TANGLE_API_KEY — our router (the solver).
+#   - uv + Docker.  /app is sudo-writable (grade.py hardcodes /app/answer.txt).
+#
+# Usage (wrap in dotenvx so the keys resolve):
+#   dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
+#     bash bench/scripts/trata-hedge/run.sh <path-to-trata-env-dir> [WORKER_MODEL]
+set -euo pipefail
+
+ENV="${1:?usage: run.sh <trata-env-dir> [model]}"
+MODEL="${2:-${WORKER_MODEL:-gpt-4o}}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ANS=/tmp/thb-answer.txt
+REWARD=/tmp/thb-reward.txt
+DETAILS=/tmp/thb-details.json
+
+echo "[trata] solve: $ENV (model=$MODEL)"
+WORKER_MODEL="$MODEL" python3 "$HERE/solve.py" "$ENV" "$ANS"
+
+sudo -n mkdir -p /app && sudo -n cp "$ANS" /app/answer.txt && sudo -n chmod 644 /app/answer.txt
+
+echo "[trata] grade: their grade.py (gemini-3.1-pro)"
+GRADE_LOG=/tmp/thb-grade.log
+# Fail loud: capture the grader's output; if it errors (bad key, import, network) or
+# writes no details, surface ITS error rather than crashing the reporter cryptically.
+set +e
+( cd "$ENV/tests" && \
+  GOOGLE_API_KEY="$GOOGLE_AI_KEY" \
+  DATA_DIR="$ENV/environment/data" \
+  GROUND_TRUTH_PATH="$ENV/tests/ground_truth.txt" \
+  REWARD_PATH="$REWARD" DETAILS_PATH="$DETAILS" \
+  uvx --with google-genai python3 grade.py ) >"$GRADE_LOG" 2>&1
+grade_status=$?
+set -e
+if [ "$grade_status" -ne 0 ] || [ ! -f "$DETAILS" ]; then
+  echo "[trata] grade.py FAILED (exit $grade_status) — last output:" >&2
+  tail -n 15 "$GRADE_LOG" >&2
+  exit 1
+fi
+
+python3 - "$DETAILS" "$REWARD" <<'PY'
+import json, sys
+d = json.load(open(sys.argv[1]))
+reward = open(sys.argv[2]).read().strip() if __import__("os").path.exists(sys.argv[2]) else "?"
+print(f"  sparse reward (1 iff all themes): {reward}")
+print(f"  graded score (0-4): {d.get('score')} | themes_covered: {d.get('themes_covered')}/{d.get('num_themes')} | hallucinations: {d.get('hallucinations_detected')}")
+for t in d.get("themes_hit", []):
+    print(f"    HIT : {t.get('label')}")
+for t in d.get("themes_missed", []):
+    print(f"    MISS: {t.get('label')} ({len(t.get('moves_hit',[]))}/{t.get('n_moves')} moves)")
+PY

package/scripts/trata-hedge/solve.py

@@ -0,0 +1,83 @@
+"""trata-hedge-bench solver — our system's analyst, graded by THEIR Gemini judge.
+
+Reads a task's instruction.md + data corpus, writes a cited analysis to answer.txt.
+The corpus is large (~5MB/task), so a single-shot router call is a LOWER-BOUND
+baseline (it sees only what fits in context) — the bench is built for agentic
+exploration, which our sandbox runtime provides (see README; pending sandbox health).
+
+Notes learned the hard way:
+- router.tangle.tools is behind Cloudflare bot-fight: a default Python-urllib
+  User-Agent gets 403 (CF 1010) on large bodies → send a browser UA.
+- gpt-4.1 is NOT keyed for direct router calls (403); gpt-4o / claude-sonnet-4-6 /
+  gpt-4o-mini are. Very large bodies (~350KB) can 503 — keep DATA_BUDGET in range.
+
+  WORKER_MODEL=gpt-4o DATA_BUDGET=160000 python3 solve.py <env-dir> <out.txt>
+"""
+import json
+import os
+import sys
+import urllib.request
+from pathlib import Path
+
+env = Path(sys.argv[1])
+out = sys.argv[2] if len(sys.argv) > 2 else "/tmp/thb-answer.txt"
+model = os.environ.get("WORKER_MODEL", "gpt-4o")
+budget = int(os.environ.get("DATA_BUDGET", "160000"))
+instruction = (env / "instruction.md").read_text()
+data_dir = env / "environment" / "data"
+
+
+def rank(p: Path) -> int:
+    s = str(p)
+    return (0 if "earnings_call" in s else 1 if "financials" in s else 2 if "company_profiles" in s
+            else 3 if "press_releases" in s else 4)
+
+
+files = sorted((p for p in data_dir.rglob("*") if p.is_file()), key=rank)
+blocks, used, cited = [], 0, []
+for p in files:
+    rel = p.relative_to(data_dir)
+    try:
+        c = p.read_text(errors="replace")
+    except Exception:
+        continue
+    if used + len(c) > budget:
+        continue
+    used += len(c)
+    cited.append(str(rel))
+    blocks.append(f"\n=== FILE: data/{rel} ===\n{c}")
+print(f"[solve] {len(cited)}/{len(files)} files in context ({used} chars)", file=sys.stderr)
+
+prompt = (
+    instruction
+    + "\n\n--- AVAILABLE DATA (cite files by their `data/<path>` name inline) ---\n"
+    + "".join(blocks)
+    + "\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/<path>`."
+)
+
+key = os.environ["TANGLE_API_KEY"]
+base = os.environ.get("ROUTER_BASE", "https://router.tangle.tools/v1")
+body = json.dumps({
+    "model": model,
+    "messages": [{"role": "user", "content": prompt}],
+    "temperature": float(os.environ.get("TEMPERATURE", "0.5")),
+    "max_tokens": int(os.environ.get("MAX_TOKENS", "6000")),
+}).encode()
+req = urllib.request.Request(
+    f"{base}/chat/completions", data=body,
+    headers={
+        "Authorization": f"Bearer {key}",
+        "Content-Type": "application/json",
+        # browser UA: router is behind Cloudflare bot-fight, default urllib UA → 403 on big bodies
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+    },
+)
+try:
+    with urllib.request.urlopen(req, timeout=300) as r:
+        resp = json.load(r)
+except urllib.error.HTTPError as e:
+    print(f"[solve] HTTP {e.code}: {e.read().decode(errors='replace')[:400]}", file=sys.stderr)
+    raise
+answer = resp["choices"][0]["message"]["content"]
+Path(out).write_text(answer)
+print(f"[solve] wrote {len(answer)} chars -> {out} (model={model})", file=sys.stderr)

package/scripts/verify-packed-consumer.mjs

@@ -0,0 +1,224 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+
+const execFileAsync = promisify(execFile)
+const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const repoRoot = path.resolve(benchDir, '..')
+const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-consumer-'))
+const args = new Set(process.argv.slice(2))
+const useLocalRuntime = args.delete('--local-runtime')
+if (args.size > 0) throw new Error(`unknown arguments: ${[...args].join(', ')}`)
+
+const TYPESCRIPT_5 = '5.9.3'
+const TYPESCRIPT_6 = '6.0.3'
+
+async function run(command, args, cwd, env = process.env) {
+  try {
+    return await execFileAsync(command, args, {
+      cwd,
+      env,
+      maxBuffer: 10 * 1024 * 1024,
+      timeout: 120_000,
+    })
+  } catch (error) {
+    if (error?.stdout) process.stdout.write(error.stdout)
+    if (error?.stderr) process.stderr.write(error.stderr)
+    const invocation = [command, ...args].join(' ')
+    const message = error instanceof Error ? error.message : String(error)
+    throw new Error(`${invocation} failed: ${message}`, { cause: error })
+  }
+}
+
+async function resolveRuntimePackage(packDir) {
+  if (process.env.AGENT_RUNTIME_PACKAGE) {
+    return path.resolve(process.env.AGENT_RUNTIME_PACKAGE)
+  }
+  if (!useLocalRuntime) return undefined
+  const manifest = JSON.parse(await readFile(path.join(repoRoot, 'package.json'), 'utf8'))
+  if (manifest.name !== '@tangle-network/agent-runtime') {
+    throw new Error('--local-runtime requires an agent-runtime source workspace')
+  }
+  await run('pnpm', ['pack', '--pack-destination', packDir], repoRoot)
+  const tarballs = (await readdir(packDir)).filter((name) => name.endsWith('.tgz'))
+  if (tarballs.length !== 1) {
+    throw new Error(`expected one packed agent-runtime tarball, found ${tarballs.length}`)
+  }
+  return path.join(packDir, tarballs[0])
+}
+
+try {
+  const packDir = path.join(scratch, 'pack')
+  const runtimePackDir = path.join(scratch, 'runtime-pack')
+  const consumerDir = path.join(scratch, 'consumer')
+  await mkdir(packDir)
+  await mkdir(runtimePackDir)
+  await mkdir(consumerDir)
+
+  await run('pnpm', ['pack', '--pack-destination', packDir], benchDir)
+  const packedFiles = (await readdir(packDir)).filter((name) => name.endsWith('.tgz'))
+  if (packedFiles.length !== 1) {
+    throw new Error(`expected one packed agent-bench tarball, found ${packedFiles.length}`)
+  }
+  const tarball = path.join(packDir, packedFiles[0])
+  const runtimePackage = await resolveRuntimePackage(runtimePackDir)
+  const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8'))
+  const devDependencies = manifest.devDependencies
+  if (
+    typeof devDependencies?.['@types/node'] !== 'string' ||
+    typeof devDependencies.typescript !== 'string' ||
+    typeof devDependencies.tsx !== 'string' ||
+    !devDependencies['@types/node'] ||
+    !devDependencies.typescript ||
+    !devDependencies.tsx
+  ) {
+    throw new Error(
+      'package verification requires @types/node, typescript, and tsx devDependencies',
+    )
+  }
+  const publicTsconfig = JSON.parse(
+    await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8'),
+  )
+  if (!publicTsconfig.compilerOptions)
+    throw new Error('tsconfig.public.json must define compilerOptions')
+
+  await writeFile(
+    path.join(consumerDir, 'package.json'),
+    `${JSON.stringify(
+      {
+        name: 'agent-bench-package-verifier',
+        private: true,
+        type: 'module',
+        dependencies: {
+          '@tangle-network/agent-bench': `file:${tarball}`,
+          ...(runtimePackage
+            ? { '@tangle-network/agent-runtime': `file:${runtimePackage}` }
+            : {}),
+        },
+        devDependencies: {
+          '@types/node': devDependencies['@types/node'],
+          typescript: TYPESCRIPT_5,
+          tsx: devDependencies.tsx,
+        },
+      },
+      null,
+      2,
+    )}\n`,
+  )
+  await writeFile(
+    path.join(consumerDir, 'index.ts'),
+    "import { createSweBenchAdapter, executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, runStagedJudge, StagedJudgeError, type BenchmarkAdapter, type JudgeArtifactReceipt, type PierCandidateTrialController, type PierCandidateTrialHandle, type PierDockerConnection, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst captureAdapter: BenchmarkAdapter = createSweBenchAdapter({ captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ destination: `/tmp/${taskId}/${attemptSequence}` }) })\nconst receipt = undefined as JudgeArtifactReceipt | undefined\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nconst dockerConnection = undefined as PierDockerConnection | undefined\nvoid adapter\nvoid captureAdapter\nvoid receipt\nvoid staged\nvoid trial\nvoid controller\nvoid dockerConnection\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\nvoid runStagedJudge\nvoid StagedJudgeError\n",
+  )
+  await writeFile(
+    path.join(consumerDir, 'tsconfig.json'),
+    `${JSON.stringify(
+      {
+        compilerOptions: publicTsconfig.compilerOptions,
+        files: ['index.ts'],
+      },
+      null,
+      2,
+    )}\n`,
+  )
+  await writeFile(
+    path.join(consumerDir, 'verify_pier_payload.py'),
+    `import py_compile
+from pathlib import Path
+
+import pier_agents
+from pier_agents import candidate_contract
+
+root = Path(pier_agents.__file__).parent
+expected = {
+    "__init__.py",
+    "candidate_contract.py",
+    "process_boundary.py",
+    "tangle_candidate.py",
+    "workspace_boundary.py",
+}
+observed = {path.name for path in root.glob("*.py")}
+assert observed == expected, (observed, expected)
+assert candidate_contract.PreparedCandidateContract.__module__ == "pier_agents.candidate_contract"
+for name in sorted(expected):
+    py_compile.compile(root / name, doraise=True)
+`,
+  )
+
+  // Intentionally resolve the declared ranges like a brand-new registry consumer.
+  // The preceding frozen install + public typecheck cover the exact bench lockfile.
+  await run(
+    'npm',
+    ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'],
+    consumerDir,
+  )
+  await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir)
+  const typescript5 = await run('npm', ['exec', '--', 'tsc', '--version'], consumerDir)
+  if (typescript5.stdout.trim() !== `Version ${TYPESCRIPT_5}`) {
+    throw new Error(`expected TypeScript ${TYPESCRIPT_5}, received ${typescript5.stdout.trim()}`)
+  }
+  await run(
+    'npm',
+    [
+      'install',
+      '--ignore-scripts',
+      '--no-audit',
+      '--no-fund',
+      '--no-save',
+      '--package-lock=false',
+      `typescript@${TYPESCRIPT_6}`,
+    ],
+    consumerDir,
+  )
+  await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir)
+  const typescript6 = await run('npm', ['exec', '--', 'tsc', '--version'], consumerDir)
+  if (typescript6.stdout.trim() !== `Version ${TYPESCRIPT_6}`) {
+    throw new Error(`expected TypeScript ${TYPESCRIPT_6}, received ${typescript6.stdout.trim()}`)
+  }
+  await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir)
+  const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench')
+  const prepared = await run(
+    'npm',
+    [
+      'exec',
+      '--',
+      'tsx',
+      path.join(installedPackage, 'scripts', 'verify-pier-agent.mts'),
+    ],
+    consumerDir,
+    { ...process.env, PIER_PREPARE_ONLY: '1', PIER_PROOF_ARM: 'failure' },
+  )
+  const prepareProof = JSON.parse(prepared.stdout)
+  if (
+    prepareProof.prepared !== true ||
+    prepareProof.disposed !== true ||
+    !/^sha256:[a-f0-9]{64}$/.test(prepareProof.executionPlanDigest) ||
+    !/^sha256:[a-f0-9]{64}$/.test(prepareProof.graderDigest)
+  ) {
+    throw new Error(`packed consumer did not prepare a real candidate: ${prepared.stdout}`)
+  }
+  await run('python3', ['verify_pier_payload.py'], consumerDir, {
+    ...process.env,
+    PYTHONPATH: installedPackage,
+  })
+  let runtimeManifest
+  try {
+    runtimeManifest = JSON.parse(
+      await readFile(
+        path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'),
+        'utf8',
+      ),
+    )
+  } catch (error) {
+    throw new Error('packed consumer did not install @tangle-network/agent-runtime', {
+      cause: error,
+    })
+  }
+  console.log(
+    `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}; prepared ${prepareProof.executionPlanDigest}`,
+  )
+} finally {
+  await rm(scratch, { recursive: true, force: true })
+}