@tangle-network/agent-bench 0.1.0 → 0.3.6

package/fixtures/pier-agent/no-model-task/instruction.md

@@ -0,0 +1,6 @@
+Change src/status.txt so it contains exactly:
+
+ready
+owner=tangle
+
+Commit the change to git when finished.

package/fixtures/pier-agent/no-model-task/pre_artifacts.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir -p /logs/artifacts
+cd /app
+git -c safe.directory=/app diff --binary benchmark-base..HEAD > /logs/artifacts/model.patch

package/fixtures/pier-agent/no-model-task/task.toml

@@ -0,0 +1,35 @@
+version = "1.0"
+
+[task]
+name = "agent-bench/pier-candidate-no-model"
+authors = []
+keywords = ["pier", "candidate", "no-model"]
+
+[metadata]
+author_name = "Tangle Research"
+author_email = "research@tangle.tools"
+difficulty = "synthetic"
+category = "software-engineering"
+tags = ["no-model", "separate-verifier"]
+
+[agent]
+timeout_sec = 60.0
+
+[verifier]
+timeout_sec = 60.0
+environment_mode = "separate"
+
+[verifier.environment]
+
+[environment]
+build_timeout_sec = 300.0
+cpus = 1
+memory_mb = 1024
+storage_mb = 2048
+gpus = 0
+allow_internet = false
+mcp_servers = []
+
+[verifier.env]
+
+[solution.env]

package/fixtures/pier-agent/no-model-task/tests/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:24.04
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY seed/ /app/
+RUN git init \
+    && git config user.email fixture@tangle.tools \
+    && git config user.name "Tangle Fixture" \
+    && git add -A \
+    && GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \
+       GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \
+       git commit -m baseline \
+    && git tag benchmark-base
+COPY test.sh /tests/test.sh

package/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt

@@ -0,0 +1 @@
+not-ready

package/fixtures/pier-agent/no-model-task/tests/test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+reward=0
+patch_applied=0
+patch=/logs/artifacts/model.patch
+
+if [ -s "$patch" ] \
+    && git -c safe.directory=/app apply --check "$patch" \
+    && git -c safe.directory=/app apply "$patch"; then
+  patch_applied=1
+fi
+if [ "$(cat /app/src/status.txt 2>/dev/null || true)" = $'ready\nowner=tangle' ] \
+    && [ ! -e /app/AGENTS.md ]; then
+  reward=1
+fi
+
+printf '{"reward":%d,"patch_applied":%d}\n' "$reward" "$patch_applied" \
+  > /logs/verifier/reward.json

package/fixtures/programbench.json

@@ -0,0 +1,17 @@
+[
+  {
+    "instance_id": "abishekvashok__cmatrix.5c082c6",
+    "image_name": "abishekvashok/cmatrix",
+    "note": "hidden behavioral tests live in programbench/ProgramBench-Tests tests/<hash>.tar.gz; fetch via `programbench blob sync`."
+  },
+  {
+    "instance_id": "agourlay__zip-password-finder.704700d",
+    "image_name": "agourlay/zip-password-finder",
+    "note": "hidden behavioral tests live in programbench/ProgramBench-Tests tests/<hash>.tar.gz; fetch via `programbench blob sync`."
+  },
+  {
+    "instance_id": "ajeetdsouza__zoxide.67ca1bc",
+    "image_name": "ajeetdsouza/zoxide",
+    "note": "hidden behavioral tests live in programbench/ProgramBench-Tests tests/<hash>.tar.gz; fetch via `programbench blob sync`."
+  }
+]

package/fixtures/ragbench.json

@@ -0,0 +1,21 @@
+[
+  {
+    "id": "ragbench-fixture-0",
+    "dataset": "support",
+    "question": "How long is the refund window?",
+    "response": "30 days",
+    "documents": [
+      [
+        ["0a", "Title: Refund policy"],
+        ["0b", "Customers can request a refund within 30 days of purchase."]
+      ],
+      [
+        ["1a", "Title: Shipping policy"],
+        ["1b", "Shipping labels expire after 7 days."]
+      ]
+    ],
+    "adherence": 1,
+    "completeness": 1,
+    "relevance": 1
+  }
+]

package/fixtures/simpleqa.json

@@ -0,0 +1,121 @@
+[
+  {
+    "problem": "Who received the IEEE Frank Rosenblatt Award in 2010?",
+    "answer": "Michio Sugeno",
+    "topic": "Science and technology",
+    "answer_type": "Person",
+    "urls": [
+      "https://en.wikipedia.org/wiki/IEEE_Frank_Rosenblatt_Award",
+      "https://ieeexplore.ieee.org/author/37271220500",
+      "https://en.wikipedia.org/wiki/IEEE_Frank_Rosenblatt_Award",
+      "https://www.nxtbook.com/nxtbooks/ieee/awards_2010/index.php?startid=21#/p/20"
+    ]
+  },
+  {
+    "problem": "Who was awarded the Oceanography Society's Jerlov Award in 2018?",
+    "answer": "Annick Bricaud",
+    "topic": "Science and technology",
+    "answer_type": "Person",
+    "urls": [
+      "https://en.wikipedia.org/wiki/The_Oceanography_Society",
+      "https://en.wikipedia.org/wiki/The_Oceanography_Society",
+      "https://tos.org/jerlov-medal",
+      "https://www.eurekalert.org/news-releases/490504"
+    ]
+  },
+  {
+    "problem": "What's the name of the women's liberal arts college in Cambridge, Massachusetts?",
+    "answer": "Radcliffe College",
+    "topic": "Geography",
+    "answer_type": "Place",
+    "urls": [
+      "https://en.wikipedia.org/wiki/Radcliffe_College",
+      "https://en.wikipedia.org/wiki/Radcliffe_College",
+      "https://www.braingainmag.com/7-historic-liberal-arts-colleges-in-the-us.htm",
+      "https://thepeoplesarchive.dclibrary.org/repositories/2/resources/2228"
+    ]
+  },
+  {
+    "problem": "In whose honor was the Leipzig 1877 tournament organized?",
+    "answer": "Adolf Anderssen",
+    "topic": "Sports",
+    "answer_type": "Person",
+    "urls": [
+      "https://en.wikipedia.org/wiki/Adolf_Anderssen",
+      "https://www.chessgames.com/perl/chess.pl?tid=79429",
+      "https://en.wikipedia.org/wiki/Adolf_Anderssen"
+    ]
+  },
+  {
+    "problem": "According to Karl Küchler, what did Empress Elizabeth of Austria's favorite sculpture depict, which was made for her villa Achilleion at Corfu?",
+    "answer": "Poet Henrich Heine.",
+    "topic": "Art",
+    "answer_type": "Person",
+    "urls": [
+      "https://www.gutenberg.org/files/60408/60408-h/60408-h.htm\nhttps://en.wikipedia.org/wiki/Achilleion_(Corfu)",
+      "https://www.gutenberg.org/cache/epub/60408/pg60408-images.html",
+      "https://archive.org/stream/elizabethempres01burggoog/elizabethempres01burggoog_djvu.txt",
+      "https://www.habsburger.net/en/chapter/achilleion-corfu-elisabeths-flight-antiquity"
+    ]
+  },
+  {
+    "problem": "How much money, in euros, was the surgeon held responsible for Stella Obasanjo's death ordered to pay her son?",
+    "answer": "120,000",
+    "topic": "Politics",
+    "answer_type": "Number",
+    "urls": [
+      "https://en.wikipedia.org/wiki/Stella_Obasanjo#Death",
+      "https://en.wikipedia.org/wiki/Stella_Obasanjo",
+      "https://www.independent.co.uk/news/world/africa/surgeon-jailed-over-death-of-first-lady-1791712.html)",
+      "https://www.abc.net.au/news/2009-09-22/doctor-jailed-over-former-first-ladys-lipo-death/1437416)"
+    ]
+  },
+  {
+    "problem": "What were the month and year when Obama told Christianity Today, \"I am a Christian, and I am a devout Christian. I believe in the redemptive death and resurrection of Jesus Christ\"?",
+    "answer": "January 2008",
+    "topic": "Politics",
+    "answer_type": "Date",
+    "urls": [
+      "https://en.wikipedia.org/wiki/Barack_Obama",
+      "https://will-lover-32-wikia.fandom.com/wiki/Barack_obama",
+      "https://people.wikimedia.org/~ori/mod_pagespeed_tests/obama-modpagespeed.html",
+      "https://www.dreame.com/story/2723094784-beyond-the-crust/0196694272-a-new-passenger.html"
+    ]
+  },
+  {
+    "problem": "Who appointed the Chief Justice of India, Mirza Hameedullah Beg, in 1977?",
+    "answer": "Fakhruddin Ali Ahmed",
+    "topic": "Politics",
+    "answer_type": "Person",
+    "urls": [
+      "https://en.wikipedia.org/wiki/Mirza_Hameedullah_Beg",
+      "https://en.wikipedia.org/wiki/Mirza_Hameedullah_Beg",
+      "https://www.tutorialspoint.com/mirza-hameedullah-beg-former-chief-justice-of-india",
+      "https://en.wikipedia.org/wiki/List_of_chief_justices_of_India"
+    ]
+  },
+  {
+    "problem": "What is the name of the former Prime Minister of Iceland who worked as a cabin crew member until 1971?",
+    "answer": "Jóhanna Sigurðardóttir",
+    "topic": "Politics",
+    "answer_type": "Person",
+    "urls": [
+      "https://en.wikipedia.org/wiki/J%C3%B3hanna_Sigur%C3%B0ard%C3%B3ttir",
+      "https://en.wikipedia.org/wiki/J%C3%B3hanna_Sigur%C3%B0ard%C3%B3ttir",
+      "https://www.britannica.com/biography/Johanna-Sigurdardottir",
+      "https://kids.kiddle.co/J%C3%B3hanna_Sigur%C3%B0ard%C3%B3ttir"
+    ]
+  },
+  {
+    "problem": "To whom did Mehbooba Mufti Sayed contest the 2019 Lok Sabha elections and lose?",
+    "answer": "Hasnain Masoodi",
+    "topic": "Politics",
+    "answer_type": "Person",
+    "urls": [
+      "https://en.wikipedia.org/wiki/Mehbooba_Mufti#References",
+      "https://www.indiatoday.in/elections/lok-sabha-2019/story/j-k-lok-sabha-results-2019-pdp-chief-mehbooba-mufti-loses-anantnag-seat-to-nc-hasnain-masoodi-1533245-2019-05-23",
+      "https://en.wikipedia.org/wiki/Mehbooba_Mufti#Political_career",
+      "https://timesofindia.indiatimes.com/elections/lok-sabha-constituencies/jammu-kashmir/anantnag"
+    ]
+  }
+]

package/fixtures/t2-ragbench.json

@@ -0,0 +1,13 @@
+[
+  {
+    "id": "t2-ragbench-fixture-0",
+    "subset": "finqa",
+    "context_id": "10k-0001",
+    "question": "What is the percentage increase from 100 to 112?",
+    "program_answer": "12%",
+    "original_answer": "12.0 percent",
+    "context": "Revenue increased from 100 in 2024 to 112 in 2025.",
+    "table": "Year | Revenue\n2024 | 100\n2025 | 112",
+    "file_name": "10k-0001.pdf"
+  }
+]

package/fixtures/tau2-bench.json

@@ -0,0 +1,16 @@
+[
+  {
+    "id": "retail-fixture-0",
+    "domain": "retail",
+    "user_scenario": {
+      "persona": "A customer contacting support.",
+      "instructions": "Ask the agent to check an order status and follow the retail policy."
+    },
+    "description": {
+      "purpose": "Fixture row for adapter plumbing only."
+    },
+    "evaluation_criteria": {
+      "reward_basis": ["DB", "COMMUNICATE"]
+    }
+  }
+]

package/fixtures/tau3-banking.json

@@ -0,0 +1,16 @@
+[
+  {
+    "id": "banking-knowledge-fixture-0",
+    "domain": "banking_knowledge",
+    "user_scenario": {
+      "instructions": "The customer asks whether they can dispute a cash-back charge and needs the correct policy path before any account mutation."
+    },
+    "description": "Adapter fixture for tau3 banking knowledge task loading.",
+    "evaluation_criteria": {
+      "actions": [
+        "retrieve the relevant banking policy",
+        "only mutate account state after confirming the policy path"
+      ]
+    }
+  }
+]

package/fixtures/toollm.json

@@ -0,0 +1,28 @@
+[
+  {
+    "query_id": 1,
+    "query": "Verify API health by calling the Checkhealth endpoint, then retrieve the list of projects using the Projects endpoint.",
+    "api_list": [
+      {
+        "category_name": "Logistics",
+        "tool_name": "SQUAKE",
+        "api_name": "Checkhealth",
+        "required_parameters": [],
+        "optional_parameters": [],
+        "method": "GET"
+      },
+      {
+        "category_name": "Logistics",
+        "tool_name": "SQUAKE",
+        "api_name": "Projects",
+        "required_parameters": [],
+        "optional_parameters": [],
+        "method": "GET"
+      }
+    ],
+    "relevant APIs": [
+      ["SQUAKE", "Checkhealth"],
+      ["SQUAKE", "Projects"]
+    ]
+  }
+]

package/fixtures/webarena-verified.json

@@ -0,0 +1,20 @@
+[
+  {
+    "sites": ["shopping_admin"],
+    "task_id": 0,
+    "intent_template_id": 279,
+    "start_urls": ["__SHOPPING_ADMIN__"],
+    "intent": "Get the top-1 best-selling product name(s) in 2022",
+    "revision": 2,
+    "eval": [
+      {
+        "evaluator": "AgentResponseEvaluator",
+        "expected": {
+          "task_type": "retrieve",
+          "status": "SUCCESS",
+          "retrieved_data": ["Quest Lumaflex Band"]
+        }
+      }
+    ]
+  }
+]

package/package.json

@@ -1,35 +1,75 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.1.0",
+  "version": "0.3.6",
   "type": "module",
-  "description": "The unified benchmark suite for agent-runtime agents: 18 adapters (commit0, enterpriseops-gym, trata-hedge, finsearchcomp, swe-bench, humaneval, …) behind one resolveAdapter registry, each with a real deterministic judge. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "exports": {
-    ".": "./src/index.ts",
-    "./adapters": "./src/adapters.ts",
-    "./benchmarks/*": "./src/benchmarks/*.ts"
+  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tangle-network/agent-runtime.git",
+    "directory": "bench"
   },
-  "scripts": {
-    "gate": "tsx src/corpus-replay.mts corpus/finsearch.jsonl --selector",
-    "gate-cli": "tsx src/gate-cli.mts",
-    "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl",
-    "terminal-compare": "tsx src/terminal-compare.ts"
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./adapters": {
+      "types": "./dist/adapters.d.ts",
+      "import": "./dist/adapters.js"
+    },
+    "./benchmarks/*": {
+      "types": "./dist/benchmarks/*.d.ts",
+      "import": "./dist/benchmarks/*.js"
+    }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.100.0",
-    "@tangle-network/agent-runtime": "^0.78.0",
-    "@tangle-network/sandbox": "^0.9.3"
+    "@tangle-network/agent-eval": "0.122.8",
+    "@tangle-network/agent-interface": "0.31.0",
+    "@tangle-network/sandbox": "^0.11.1",
+    "@tangle-network/agent-runtime": "0.97.0"
   },
   "devDependencies": {
-    "tsx": "^4.19.0",
-    "typescript": "^6.0.3"
+    "@types/node": "^25.9.3",
+    "tsx": "^4.22.4",
+    "tsup": "^8.5.1",
+    "typescript": "^5.7.0"
+  },
+  "engines": {
+    "node": ">=20.19.0"
   },
   "files": [
+    "dist",
     "src",
-    "README.md"
+    "fixtures",
+    "scripts",
+    "pier_agents/__init__.py",
+    "pier_agents/candidate_contract.py",
+    "pier_agents/process_boundary.py",
+    "pier_agents/tangle_candidate.py",
+    "pier_agents/workspace_boundary.py",
+    "tb_agents/*.py",
+    "steerers",
+    "CHANGELOG.md",
+    "README.md",
+    "HARNESS.md"
   ],
   "publishConfig": {
     "access": "public"
+  },
+  "scripts": {
+    "prebuild": "pnpm --dir .. build",
+    "build": "tsup",
+    "gate": "tsx src/corpus-replay.mts corpus/finsearch.jsonl --selector",
+    "gate-cli": "tsx src/gate-cli.mts",
+    "run-benchmarks": "tsx src/run-benchmarks-cli.mts",
+    "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl",
+    "terminal-compare": "tsx src/terminal-compare.ts",
+    "test": "node scripts/run-package-tests.mjs",
+    "typecheck:public": "tsc -p tsconfig.public.json",
+    "verify:package": "node scripts/verify-packed-consumer.mjs",
+    "verify:package:local-runtime": "node scripts/verify-packed-consumer.mjs --local-runtime",
+    "verify:pier": "tsx scripts/verify-pier-pair.mts"
   }
-}
+}

package/pier_agents/__init__.py

@@ -0,0 +1,18 @@
+"""Pier custom agents shipped by ``@tangle-network/agent-bench``."""
+
+from typing import TYPE_CHECKING, Any
+
+
+if TYPE_CHECKING:
+    from .tangle_candidate import TangleCandidateAgent
+
+__all__ = ["TangleCandidateAgent"]
+
+
+def __getattr__(name: str) -> Any:
+    """Load Pier only when callers request the custom agent."""
+    if name != "TangleCandidateAgent":
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    from .tangle_candidate import TangleCandidateAgent
+
+    return TangleCandidateAgent