@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/benchmarks/rag-shared.js

@@ -0,0 +1,39 @@
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringArrayFrom,
+  stringFrom,
+  tokenF1
+} from "../chunk-X3BTXCJ4.js";
+export {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringArrayFrom,
+  stringFrom,
+  tokenF1
+};
+//# sourceMappingURL=rag-shared.js.map

package/dist/benchmarks/rag-shared.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/ragbench.d.ts

@@ -0,0 +1,16 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * RAGBench-compatible adapter.
+ *
+ * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a
+ * compatible table. Rows must carry a query and at least one reference answer.
+ * Contexts, TRACe labels, and source metadata are preserved in task metadata
+ * for diagnostics; the deterministic judge scores the worker's final answer
+ * against the reference answer(s).
+ */
+
+declare function createRagBenchAdapter(): BenchmarkAdapter;
+
+export { createRagBenchAdapter };

package/dist/benchmarks/ragbench.js

@@ -0,0 +1,9 @@
+import {
+  createRagBenchAdapter
+} from "../chunk-R67DFVLO.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createRagBenchAdapter
+};
+//# sourceMappingURL=ragbench.js.map

package/dist/benchmarks/ragbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/simpleqa.d.ts

@@ -0,0 +1,64 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact
+ * = a single free-text final answer string (optionally preceded by a CITATIONS:
+ * block). Each item ships a short gold `answer` plus metadata (topic,
+ * answer_type, source urls).
+ *
+ * Judge is the official SimpleQA grader — an LLM classifier that maps
+ * (question, gold target, predicted answer) to exactly one of:
+ *   A = CORRECT       — fully contains the gold, no contradiction
+ *   B = INCORRECT     — contradicts / contains a different factual value
+ *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given
+ * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter
+ * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from
+ * INCORRECT so the scorecard can separate abstention from error.
+ *
+ * There is no deterministic tier: SimpleQA's rubric (containment + abstention)
+ * is the grader's job by design, so judge() always calls the pinned grader model
+ * (temperature 0) and fails loud on unparseable grader output. The final-answer
+ * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').
+ *
+ * Requires for a live run: the bench `.venv` with `datasets`/`requests` not
+ * needed — the test set is a single public CSV fetched over HTTP — plus a
+ * grader key (TANGLE_API_KEY). For offline/CI verification set
+ * SIMPLEQA_FIXTURES=1 to load the committed fixtures
+ * (bench/fixtures/simpleqa.json) — no network.
+ */
+
+/**
+ * Typed seam for the research worker. The benchmark adapter scores a plain
+ * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its
+ * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an
+ * optional `CITATIONS:` block) into that string before judging.
+ */
+interface ResearchTask {
+    id: string;
+    question: string;
+    /** Gold short answer — the grader's target. */
+    gold: string;
+    /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */
+    goldSources: string[];
+    /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */
+    topic: string;
+    /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */
+    answerType: string;
+}
+interface ResearchAnswer {
+    finalAnswer: string;
+    citations: string[];
+}
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the last non-empty line. Returns '' when nothing is parseable
+ * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.
+ */
+declare function parseFinalAnswer(artifact: string): string;
+/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */
+declare function parseCitations(artifact: string): string[];
+declare function createSimpleQaAdapter(): BenchmarkAdapter;
+
+export { type ResearchAnswer, type ResearchTask, createSimpleQaAdapter, parseCitations, parseFinalAnswer };

package/dist/benchmarks/simpleqa.js

@@ -0,0 +1,11 @@
+import {
+  createSimpleQaAdapter,
+  parseCitations,
+  parseFinalAnswer
+} from "../chunk-SEVJPLZC.js";
+export {
+  createSimpleQaAdapter,
+  parseCitations,
+  parseFinalAnswer
+};
+//# sourceMappingURL=simpleqa.js.map

package/dist/benchmarks/simpleqa.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/swe-bench.d.ts

@@ -0,0 +1,56 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { StagedRunCaptureSpec } from './_harness.js';
+import { BenchmarkAdapter, BenchScore } from './types.js';
+
+/**
+ * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =
+ * the official `swebench` harness: apply the patch in the instance's Docker
+ * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic
+ * — no LLM judge.
+ *
+ * Requires: the bench `.venv` with `swebench` installed + a running Docker
+ * daemon (per-instance images are pulled/built on first run).
+ *
+ * Process/Docker/report plumbing is shared via ./_harness; this file owns the
+ * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the
+ * predictions-file → run_evaluation argv → report-shape mapping.
+ */
+
+/**
+ * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
+ *
+ * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's
+ * actual edits, read from the cloned repo's STATE inside the box (standard
+ * SWE-bench practice). This event-stream parse only runs when that diff is empty
+ * — a model that edited the source correctly but never printed a fenced diff (the
+ * exact failure this replaces) still scores off its real changes, not its prose.
+ */
+declare const swePatchOutput: OutputAdapter<string>;
+type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance';
+interface SweBenchArtifactCaptureContext {
+    readonly taskId: string;
+    readonly runId: string;
+    /** One-based sequence unique within this adapter instance. */
+    readonly attemptSequence: number;
+}
+interface SweBenchAdapterOptions {
+    readonly timeoutMs?: number;
+    readonly cacheLevel?: SweBenchCacheLevel;
+    /**
+     * Return a unique destination for any attempt whose complete official
+     * evaluator directory and process logs should be retained.
+     */
+    readonly captureEvaluatorArtifacts?: (context: SweBenchArtifactCaptureContext) => StagedRunCaptureSpec | undefined;
+}
+/** Convert one official report into a score without turning evaluator failures into agent failures. */
+declare function scoreSweReport(taskId: string, value: unknown): BenchScore;
+declare function sweEvaluationArgv(args: {
+    readonly predictionsPath: string;
+    readonly runId: string;
+    readonly instanceId: string;
+    readonly cacheLevel: SweBenchCacheLevel;
+    readonly namespace?: 'swebench' | 'none';
+}): string[];
+declare function createSweBenchAdapter(options?: SweBenchAdapterOptions): BenchmarkAdapter;
+
+export { type SweBenchAdapterOptions, type SweBenchArtifactCaptureContext, type SweBenchCacheLevel, createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput };

package/dist/benchmarks/swe-bench.js

@@ -0,0 +1,14 @@
+import {
+  createSweBenchAdapter,
+  scoreSweReport,
+  sweEvaluationArgv,
+  swePatchOutput
+} from "../chunk-CKUVRZ2T.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createSweBenchAdapter,
+  scoreSweReport,
+  sweEvaluationArgv,
+  swePatchOutput
+};
+//# sourceMappingURL=swe-bench.js.map

package/dist/benchmarks/swe-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/t2-ragbench.d.ts

@@ -0,0 +1,14 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * T2-RAGBench adapter.
+ *
+ * T2-RAGBench stresses text+table retrieval and numerical reasoning over
+ * financial documents. The judge uses the shared deterministic answer scorer
+ * with numeric tolerance enabled by default.
+ */
+
+declare function createT2RagBenchAdapter(): BenchmarkAdapter;
+
+export { createT2RagBenchAdapter };

package/dist/benchmarks/t2-ragbench.js

@@ -0,0 +1,9 @@
+import {
+  createT2RagBenchAdapter
+} from "../chunk-Y6O2OCUO.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createT2RagBenchAdapter
+};
+//# sourceMappingURL=t2-ragbench.js.map

package/dist/benchmarks/t2-ragbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/tau-bench-shared.d.ts

@@ -0,0 +1,26 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * Shared tau-bench adapter spine.
+ *
+ * tau2 and tau3 live in the same upstream repository/package namespace today:
+ * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation
+ * for task loading and reward recomputation so the domain/version adapters only
+ * choose env names, default domain, and fixture file.
+ */
+
+interface TauBenchConfig {
+    name: string;
+    fixturePath: string;
+    fixturesEnv: string;
+    dirEnv: string;
+    domainEnv: string;
+    defaultDomain: string;
+    installHint: string;
+    taskIntro: string;
+}
+declare const tauResultsOutput: OutputAdapter<string>;
+declare function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter;
+
+export { type TauBenchConfig, createTauBenchAdapter, tauResultsOutput };

package/dist/benchmarks/tau-bench-shared.js

@@ -0,0 +1,10 @@
+import {
+  createTauBenchAdapter,
+  tauResultsOutput
+} from "../chunk-R36V2VP7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTauBenchAdapter,
+  tauResultsOutput
+};
+//# sourceMappingURL=tau-bench-shared.js.map

package/dist/benchmarks/tau-bench-shared.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/tau2-bench.d.ts

@@ -0,0 +1,7 @@
+import * as _tangle_network_agent_runtime_loops from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+declare const tau2ResultsOutput: _tangle_network_agent_runtime_loops.OutputAdapter<string>;
+declare function createTau2BenchAdapter(): BenchmarkAdapter;
+
+export { createTau2BenchAdapter, tau2ResultsOutput };

package/dist/benchmarks/tau2-bench.js

@@ -0,0 +1,11 @@
+import {
+  createTau2BenchAdapter,
+  tau2ResultsOutput
+} from "../chunk-HBSWHQNJ.js";
+import "../chunk-R36V2VP7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTau2BenchAdapter,
+  tau2ResultsOutput
+};
+//# sourceMappingURL=tau2-bench.js.map

package/dist/benchmarks/tau2-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/tau3-banking.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * tau3-banking adapter.
+ *
+ * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with
+ * package namespace `tau2` and a new `banking_knowledge` domain. This adapter
+ * reuses the shared tau reward recomputation path and only changes the default
+ * domain/env names.
+ */
+
+declare function createTau3BankingAdapter(): BenchmarkAdapter;
+
+export { createTau3BankingAdapter };

package/dist/benchmarks/tau3-banking.js

@@ -0,0 +1,9 @@
+import {
+  createTau3BankingAdapter
+} from "../chunk-UAIOHCUK.js";
+import "../chunk-R36V2VP7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTau3BankingAdapter
+};
+//# sourceMappingURL=tau3-banking.js.map

package/dist/benchmarks/tau3-banking.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/terminal-bench.d.ts

@@ -0,0 +1,24 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Terminal-Bench adapter. Each task = a Docker environment + an English
+ * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact
+ * is the shell script the agent ran to attempt the task; the judge REPLAYS that
+ * script in a fresh task container via the Terminal-Bench harness (`tb run` with
+ * our ScriptAgent), then the task's own verifier scores the resulting state.
+ * Fully deterministic — no LLM judge, no self-authored score.
+ *
+ * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`
+ * installed + a running Docker daemon (per-task images are built on first run).
+ * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the
+ * Terminal-Bench registry on first run.
+ *
+ * Process/Docker/report plumbing is shared via ./_harness; this file owns the
+ * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay
+ * argv, and the results.json shape.
+ */
+
+declare function createTerminalBenchAdapter(): BenchmarkAdapter;
+
+export { createTerminalBenchAdapter };

package/dist/benchmarks/terminal-bench.js

@@ -0,0 +1,8 @@
+import {
+  createTerminalBenchAdapter
+} from "../chunk-JRWWGMK7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTerminalBenchAdapter
+};
+//# sourceMappingURL=terminal-bench.js.map

package/dist/benchmarks/terminal-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/toollm.d.ts

@@ -0,0 +1,16 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * ToolLLM/ToolBench adapter.
+ *
+ * ToolBench task loading is useful for breadth, but the official ToolEval pass
+ * rate evaluator is LLM-based and stochastic. This adapter therefore scores
+ * only ToolBench's deterministic API-selection labels (`relevant APIs`). It
+ * never records a full ToolEval pass-rate score.
+ */
+
+declare const toollmOutput: OutputAdapter<string>;
+declare function createToolLlmAdapter(): BenchmarkAdapter;
+
+export { createToolLlmAdapter, toollmOutput };

package/dist/benchmarks/toollm.js

@@ -0,0 +1,10 @@
+import {
+  createToolLlmAdapter,
+  toollmOutput
+} from "../chunk-2PVVP7GN.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createToolLlmAdapter,
+  toollmOutput
+};
+//# sourceMappingURL=toollm.js.map

package/dist/benchmarks/toollm.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/trata-hedge.d.ts

@@ -0,0 +1,32 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial
+ * analysis tasks across 6 domains (private equity, managed care, industrials,
+ * vertical SaaS, REITs, insurance). Each task bundles real earnings-call
+ * transcripts, financial statements, press releases, and SEC filings under
+ * `environment/data/`; the agent must produce a grounded analysis citing those
+ * files.
+ *
+ * Harbor architecture: the original benchmark runs each task in a Docker
+ * container with file-read tools. This adapter skips Harbor and embeds the data
+ * files directly in the worker prompt so any router backend can score it. Large
+ * files (> 30 KB) are truncated to fit within model context limits. The judge
+ * reads the actual files from disk for citation verification.
+ *
+ * Judge: a 3-stage LLM cascade faithful to grade.py —
+ *   Task 1: hallucination check (cited-file context + agent answer)
+ *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)
+ *   Task 3: synthesis check (ground truth + agent answer)
+ * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,
+ * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;
+ * resolved = score 4 (sparse reward).
+ *
+ * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.
+ * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench
+ */
+
+declare function createTrataHedgeAdapter(): BenchmarkAdapter;
+
+export { createTrataHedgeAdapter };

package/dist/benchmarks/trata-hedge.js

@@ -0,0 +1,7 @@
+import {
+  createTrataHedgeAdapter
+} from "../chunk-VQRS7VUC.js";
+export {
+  createTrataHedgeAdapter
+};
+//# sourceMappingURL=trata-hedge.js.map

package/dist/benchmarks/trata-hedge.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/types.d.ts

@@ -0,0 +1,107 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * BenchmarkAdapter — the seam every external benchmark implements so the
+ * agent-runtime loop can be A/B'd (blind vs steering vs steering+memory) against
+ * a REAL, DETERMINISTIC judge. The worker produces an `artifact` (a patch, a
+ * command transcript, a final state); the benchmark's own harness scores it.
+ *
+ * The point: no self-authored judge, no invented score noise. The number comes
+ * from the benchmark's published evaluation harness.
+ */
+
+interface BenchTask {
+    /** Stable benchmark instance id. */
+    id: string;
+    /** The task statement handed to the worker agent. */
+    prompt: string;
+    split?: string;
+    /** Benchmark-specific fields the worker/judge need (repo, base_commit, gold, …). */
+    metadata?: Record<string, unknown>;
+}
+interface JudgeArtifactFileReceipt {
+    /** POSIX path relative to the capture directory. */
+    path: string;
+    /** Exact byte length of the retained file or symbolic-link target. */
+    byteLength: number;
+    /** SHA-256 over the retained file bytes or UTF-8 symbolic-link target. */
+    sha256: `sha256:${string}`;
+    kind: 'file' | 'symlink';
+}
+/** Durable evidence written before a staged evaluator's temporary directory is removed. */
+interface JudgeArtifactReceipt {
+    schema: 'agent-bench/judge-artifacts/v1';
+    /** Absolute directory containing `evaluator/`, `process/`, and `receipt.json`. */
+    directory: string;
+    /** Exact copy of the evaluator working directory. */
+    evaluatorDirectory: string;
+    manifestPath: string;
+    evaluatorSucceeded: boolean;
+    files: JudgeArtifactFileReceipt[];
+    fileCount: number;
+    byteLength: number;
+    /** SHA-256 over every sorted path, kind, byte length, and content hash. */
+    treeSha256: `sha256:${string}`;
+}
+interface BenchScore {
+    /** Did the deterministic judge pass (tests resolved / state correct)? */
+    resolved: boolean;
+    /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */
+    score: number;
+    detail?: string;
+    /** Present only when the caller explicitly requested durable judge evidence. */
+    judgeArtifacts?: JudgeArtifactReceipt;
+}
+interface LoadOptions {
+    limit?: number;
+    split?: string;
+    ids?: string[];
+}
+interface BenchmarkAdapter {
+    readonly name: string;
+    /** Throw with actionable guidance when the harness/judge isn't installed/runnable. */
+    preflight(): Promise<void>;
+    loadTasks(opts?: LoadOptions): Promise<BenchTask[]>;
+    /** DETERMINISTIC judge: score the worker's produced artifact for a task. */
+    judge(task: BenchTask, artifact: string): Promise<BenchScore>;
+    /** Gold/oracle artifact — lets us self-verify the judge before spending model tokens. */
+    goldArtifact(task: BenchTask): Promise<string | undefined>;
+    /** How to extract the judged artifact from a run's event stream. Optional —
+     *  defaults to the agent's final answer text (the research/QA case). SWE sets
+     *  it to a patch parser. This is `benchmark = adapter` owning its deliverable,
+     *  so the gate runner (`runGate` / `runBenchmark`) needs no
+     *  per-benchmark branching. */
+    output?: OutputAdapter<string>;
+    /** Post-shot deliverable extraction from the box FILESYSTEM, not the event stream.
+     *  When set, the shot runner execs `command` in the STILL-ALIVE box after the agent
+     *  turn drains and uses its stdout as the judged artifact — the durable way to capture
+     *  a git diff of the agent's in-box edits (standard SWE-bench practice: SWE-agent /
+     *  OpenHands read the diff from repo STATE), instead of hoping the model printed a
+     *  fenced diff in its reply. Empty stdout ⇒ the runner falls back to `output` (the
+     *  event-stream parse). `cwd` defaults to the box root. */
+    boxExtract?(task: BenchTask): {
+        command: string;
+        cwd?: string;
+    };
+    /** Optional workspace pre-stage run in the box BEFORE the agent shot (same
+     *  session as `boxExtract`). For repo-state benchmarks (SWE-bench) this clones
+     *  the instance repo at `base_commit` into a fixed path so the agent only edits
+     *  — the harness owns the checkout, not the (stochastic) model. A non-zero exit
+     *  fails the shot loud rather than letting the agent run against an empty box. */
+    boxSetup?(task: BenchTask): {
+        command: string;
+        cwd?: string;
+    };
+    /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the
+     *  worker (e.g. AppWorld's interactive ReAct episode runs inside the engine,
+     *  not as a chat completion) — the experiment uses this instead of the
+     *  BACKEND-selected client; the steer still flows through the per-round prompt.
+     *  Typed loosely to avoid a runtime import cycle; the harness casts it. */
+    leafClient?: (cfg: {
+        model: string;
+        routerBaseUrl: string;
+        routerKey: string;
+    }) => unknown;
+}
+
+export type { BenchScore, BenchTask, BenchmarkAdapter, JudgeArtifactFileReceipt, JudgeArtifactReceipt, LoadOptions };

package/dist/benchmarks/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/benchmarks/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/webarena-verified.d.ts

@@ -0,0 +1,16 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * WebArena-Verified adapter (ServiceNow/webarena-verified).
+ *
+ * Worker artifact = a WebArena-Verified run output directory, not final chat text.
+ * Judge = the official `webarena_verified eval-tasks` evaluator over that output
+ * directory. The adapter refuses to score a plain answer so we do not turn a DOM
+ * benchmark into a fake text benchmark.
+ */
+
+declare const webarenaOutputDirOutput: OutputAdapter<string>;
+declare function createWebArenaVerifiedAdapter(): BenchmarkAdapter;
+
+export { createWebArenaVerifiedAdapter, webarenaOutputDirOutput };

package/dist/benchmarks/webarena-verified.js

@@ -0,0 +1,10 @@
+import {
+  createWebArenaVerifiedAdapter,
+  webarenaOutputDirOutput
+} from "../chunk-Z7ML6L77.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createWebArenaVerifiedAdapter,
+  webarenaOutputDirOutput
+};
+//# sourceMappingURL=webarena-verified.js.map

package/dist/benchmarks/webarena-verified.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}