@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/benchmarks/hotpotqa.d.ts

@@ -0,0 +1,48 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').
+ * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.
+ *
+ * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no
+ * LLM. Both the predicted final answer and the gold are normalized (lowercase,
+ * strip articles a/an/the, strip punctuation, collapse whitespace), then scored
+ * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS
+ * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the
+ * suite a judge that needs no model tokens at all.
+ *
+ * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of
+ * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,
+ * never part of the score.
+ *
+ * Requires for a live run: the bench `.venv` with `datasets` installed + network
+ * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the
+ * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.
+ */
+
+/**
+ * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,
+ * drop articles (a/an/the), collapse whitespace. Token comparisons run on the
+ * output of this exactly as the published evaluator does.
+ */
+declare function normalizeAnswer(input: string): string;
+/** Exact match on the normalized strings. */
+declare function exactMatch(prediction: string, gold: string): boolean;
+/**
+ * Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of
+ * normalized tokens. Mirrors the published evaluator's special-case handling of
+ * yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both
+ * are empty, else 0.
+ */
+declare function tokenF1(prediction: string, gold: string): number;
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the trimmed last non-empty line. Returns '' when nothing is
+ * parseable (fail-closed — never guess), which judge() counts as resolved=false.
+ */
+declare function parseFinalAnswer(artifact: string): string;
+declare function createHotpotqaAdapter(): BenchmarkAdapter;
+
+export { createHotpotqaAdapter, exactMatch, normalizeAnswer, parseFinalAnswer, tokenF1 };

package/dist/benchmarks/hotpotqa.js

@@ -0,0 +1,15 @@
+import {
+  createHotpotqaAdapter,
+  exactMatch,
+  normalizeAnswer,
+  parseFinalAnswer,
+  tokenF1
+} from "../chunk-PA2ZKHJC.js";
+export {
+  createHotpotqaAdapter,
+  exactMatch,
+  normalizeAnswer,
+  parseFinalAnswer,
+  tokenF1
+};
+//# sourceMappingURL=hotpotqa.js.map

package/dist/benchmarks/hotpotqa.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/humaneval.d.ts

@@ -0,0 +1,53 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the
+ * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout
+ * through the `Supervisor` that self-corrects across rounds, vs blind
+ * random@k. This is the experiment `humaneval-gate.mts` names as "the next one" —
+ * the gate measures SELECTION over stateless single completions; this measures
+ * whether observe→steer (self-correction) beats blind compute at equal k.
+ *
+ * Worker artifact = the model's reply (a Python function, fenced or raw). The
+ * DETERMINISTIC judge runs the candidate against the task's own `test` in an
+ * isolated `--network=none` python container — exit 0 = pass. No gold
+ * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only
+ * to self-verify the judge before spending tokens.
+ *
+ * The primitives (loader / extractor / Docker checker) live here and are reused by
+ * `humaneval-gate.mts` — one home, no duplication.
+ */
+
+interface HumanEvalTask {
+    taskId: string;
+    prompt: string;
+    test: string;
+    entryPoint: string;
+    /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */
+    canonicalSolution?: string;
+}
+/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch
+ *  or a malformed line — a silently-short task set would poison the gate. `offset`
+ *  selects a deeper slice (the later tasks are harder) so the worker has a
+ *  correctable middle band rather than a saturated easy prefix. */
+declare function loadHumanEval(limit: number, offset?: number): Promise<HumanEvalTask[]>;
+declare function basePrompt(task: HumanEvalTask): string;
+/** Extract the function source from a model reply: prefer a fenced ```python (or
+ *  bare ```) block, else fall back to the raw text. The deployable program adds the
+ *  prompt header (imports + signature context), so a candidate that returns only a
+ *  body still runs; a candidate that re-defines the function shadows the header. */
+declare function extractCode(reply: string): string;
+interface CheckResult {
+    /** {0,1} pass-count for this candidate (1 = the check() suite passed). */
+    pass: number;
+    /** On failure: the interpreter stderr tail (traceback / failing assertion). The
+     *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */
+    detail?: string;
+}
+declare function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult>;
+/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle
+ *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */
+declare function createHumanEvalAdapter(): BenchmarkAdapter;
+
+export { type CheckResult, type HumanEvalTask, basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker };

package/dist/benchmarks/humaneval.js

@@ -0,0 +1,15 @@
+import {
+  basePrompt,
+  createHumanEvalAdapter,
+  extractCode,
+  loadHumanEval,
+  runChecker
+} from "../chunk-PPYSEKFM.js";
+export {
+  basePrompt,
+  createHumanEvalAdapter,
+  extractCode,
+  loadHumanEval,
+  runChecker
+};
+//# sourceMappingURL=humaneval.js.map

package/dist/benchmarks/humaneval.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/mind2web.d.ts

@@ -0,0 +1,41 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).
+ * Web-agent ACTION PREDICTION: each task is one step of a real web task — given
+ * the natural-language goal and the page's candidate elements, the worker picks
+ * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).
+ *
+ * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:
+ *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids
+ *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches
+ *   resolved (Step-SR) ⇔ element correct AND operation correct
+ *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right
+ *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)
+ * This is the low-noise reward a certifiable directive-lift needs: the number is a
+ * programmatic match against human-verified ground truth, not a judge's opinion.
+ *
+ * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the
+ * judge extracts deterministically; the prompt presents the candidate set as a
+ * choice over backend_node_id ordered by id (position uncorrelated with the answer).
+ *
+ * Each step carries the dataset's OWN page screenshot (written to a temp file by
+ * the loader); the worker drops it into a browser.<op> span so run-capsule's screen
+ * capsule turns the run into a film — the real page, not a re-rendered DOM.
+ *
+ * Requires for a live run: a python with `datasets` + `pillow` and network to
+ * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).
+ */
+
+interface ParsedAction {
+    elementId: string;
+    op: string;
+    value: string;
+}
+/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.
+ *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */
+declare function parseAction(artifact: string): ParsedAction | null;
+declare function createMind2WebAdapter(): BenchmarkAdapter;
+
+export { createMind2WebAdapter, parseAction };

package/dist/benchmarks/mind2web.js

@@ -0,0 +1,9 @@
+import {
+  createMind2WebAdapter,
+  parseAction
+} from "../chunk-TBKU5XQI.js";
+export {
+  createMind2WebAdapter,
+  parseAction
+};
+//# sourceMappingURL=mind2web.js.map

package/dist/benchmarks/mind2web.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/nomiracl.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * NoMIRACL adapter.
+ *
+ * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does
+ * not generate an answer here; it classifies whether the supplied passages
+ * contain enough evidence to answer the query. This directly measures false
+ * positive / false negative behavior for RAG abstention.
+ */
+
+declare function createNoMiraclAdapter(): BenchmarkAdapter;
+
+export { createNoMiraclAdapter };

package/dist/benchmarks/nomiracl.js

@@ -0,0 +1,9 @@
+import {
+  createNoMiraclAdapter
+} from "../chunk-TSWPNOYM.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createNoMiraclAdapter
+};
+//# sourceMappingURL=nomiracl.js.map

package/dist/benchmarks/nomiracl.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/open-rag-bench.d.ts

@@ -0,0 +1,14 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Open RAG Bench adapter.
+ *
+ * This targets Vectara-style Open RAG Bench exports over PDF-derived text,
+ * table, and image contexts. The deterministic judge scores final-answer
+ * agreement and surfaces modality/document metadata for diagnostics.
+ */
+
+declare function createOpenRagBenchAdapter(): BenchmarkAdapter;
+
+export { createOpenRagBenchAdapter };

package/dist/benchmarks/open-rag-bench.js

@@ -0,0 +1,9 @@
+import {
+  createOpenRagBenchAdapter
+} from "../chunk-KDIKRJGB.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createOpenRagBenchAdapter
+};
+//# sourceMappingURL=open-rag-bench.js.map

package/dist/benchmarks/open-rag-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/programbench.d.ts

@@ -0,0 +1,38 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom
+ * reverse-engineering: rebuild a black-box executable's behavior from scratch.
+ * The agent is given only the gold `./executable` (run-only) + stripped docs and
+ * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`
+ * with identical observable behavior. Judge = the official `programbench`
+ * harness: it extracts the submission, runs compile.sh in the per-task cleanroom
+ * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via
+ * `programbench blob sync`). Score = fraction of non-ignored tests passed
+ * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests
+ * pass. Fully deterministic — no LLM judge.
+ *
+ * OutputAdapter is stream-only, so the worker emits its codebase as fenced
+ * `path:`-prefixed file blocks (including compile.sh); the adapter materializes
+ * those into submission.tar.gz. Test execution is delegated to the real
+ * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT
+ * reimplemented here.
+ *
+ * Requires for a live run: the bench `.venv` with `programbench` installed +
+ * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for
+ * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load
+ * the committed instance ids (bench/fixtures/programbench.json); judging still
+ * needs the harness + Docker and fails loud without them — never a fabricated score.
+ */
+
+/**
+ * Parse the worker stream into the submission text the driver materializes: the
+ * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to
+ * the python driver, which tars it. Empty when the worker emitted no file block
+ * (fail-closed → the harness scores a missing compile.sh as 0).
+ */
+declare const programbenchSubmissionOutput: OutputAdapter<string>;
+declare function createProgrambenchAdapter(): BenchmarkAdapter;
+
+export { createProgrambenchAdapter, programbenchSubmissionOutput };

package/dist/benchmarks/programbench.js

@@ -0,0 +1,10 @@
+import {
+  createProgrambenchAdapter,
+  programbenchSubmissionOutput
+} from "../chunk-ZEWMTR5M.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createProgrambenchAdapter,
+  programbenchSubmissionOutput
+};
+//# sourceMappingURL=programbench.js.map

package/dist/benchmarks/programbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/rag-shared.d.ts

@@ -0,0 +1,42 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchScore, BenchTask, LoadOptions } from './types.js';
+
+declare const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+interface RagContext {
+    id: string;
+    text: string;
+    title?: string;
+    source?: string;
+    relevant?: boolean;
+}
+interface RagAnswerScore {
+    resolved: boolean;
+    score: number;
+    finalAnswer: string;
+    bestGold: string | null;
+    exact: boolean;
+    numeric: boolean;
+    f1: number;
+    threshold: number;
+}
+declare const ragAnswerOutput: OutputAdapter<string>;
+declare function parseFinalAnswer(artifact: string): string;
+declare function parseCitations(artifact: string): string[];
+declare function normalizeAnswer(input: string): string;
+declare function tokenF1(candidate: string, gold: string): number;
+declare function scoreAnswerArtifact(artifact: string, golds: readonly string[], options?: {
+    threshold?: number;
+    numericTolerance?: number;
+}): RagAnswerScore;
+declare function answerScoreToBenchScore(score: RagAnswerScore, detail: Record<string, unknown>): BenchScore;
+declare function readJsonRows(path: string): Promise<unknown[]>;
+declare function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[];
+declare function stringFrom(value: unknown): string | undefined;
+declare function stringArrayFrom(value: unknown): string[];
+declare function firstString(row: Record<string, unknown>, keys: readonly string[]): string;
+declare function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[];
+declare function contextsFrom(value: unknown): RagContext[];
+declare function contextBlock(contexts: readonly RagContext[]): string;
+declare function isObject(value: unknown): value is Record<string, unknown>;
+
+export { FINAL_ANSWER_SENTINEL, type RagAnswerScore, type RagContext, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringArrayFrom, stringFrom, tokenF1 };

package/dist/benchmarks/rag-shared.js

@@ -0,0 +1,39 @@
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringArrayFrom,
+  stringFrom,
+  tokenF1
+} from "../chunk-X3BTXCJ4.js";
+export {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringArrayFrom,
+  stringFrom,
+  tokenF1
+};
+//# sourceMappingURL=rag-shared.js.map

package/dist/benchmarks/rag-shared.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/ragbench.d.ts

@@ -0,0 +1,16 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * RAGBench-compatible adapter.
+ *
+ * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a
+ * compatible table. Rows must carry a query and at least one reference answer.
+ * Contexts, TRACe labels, and source metadata are preserved in task metadata
+ * for diagnostics; the deterministic judge scores the worker's final answer
+ * against the reference answer(s).
+ */
+
+declare function createRagBenchAdapter(): BenchmarkAdapter;
+
+export { createRagBenchAdapter };

package/dist/benchmarks/ragbench.js

@@ -0,0 +1,9 @@
+import {
+  createRagBenchAdapter
+} from "../chunk-R67DFVLO.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createRagBenchAdapter
+};
+//# sourceMappingURL=ragbench.js.map

package/dist/benchmarks/ragbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/simpleqa.d.ts

@@ -0,0 +1,64 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact
+ * = a single free-text final answer string (optionally preceded by a CITATIONS:
+ * block). Each item ships a short gold `answer` plus metadata (topic,
+ * answer_type, source urls).
+ *
+ * Judge is the official SimpleQA grader — an LLM classifier that maps
+ * (question, gold target, predicted answer) to exactly one of:
+ *   A = CORRECT       — fully contains the gold, no contradiction
+ *   B = INCORRECT     — contradicts / contains a different factual value
+ *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given
+ * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter
+ * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from
+ * INCORRECT so the scorecard can separate abstention from error.
+ *
+ * There is no deterministic tier: SimpleQA's rubric (containment + abstention)
+ * is the grader's job by design, so judge() always calls the pinned grader model
+ * (temperature 0) and fails loud on unparseable grader output. The final-answer
+ * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').
+ *
+ * Requires for a live run: the bench `.venv` with `datasets`/`requests` not
+ * needed — the test set is a single public CSV fetched over HTTP — plus a
+ * grader key (TANGLE_API_KEY). For offline/CI verification set
+ * SIMPLEQA_FIXTURES=1 to load the committed fixtures
+ * (bench/fixtures/simpleqa.json) — no network.
+ */
+
+/**
+ * Typed seam for the research worker. The benchmark adapter scores a plain
+ * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its
+ * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an
+ * optional `CITATIONS:` block) into that string before judging.
+ */
+interface ResearchTask {
+    id: string;
+    question: string;
+    /** Gold short answer — the grader's target. */
+    gold: string;
+    /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */
+    goldSources: string[];
+    /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */
+    topic: string;
+    /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */
+    answerType: string;
+}
+interface ResearchAnswer {
+    finalAnswer: string;
+    citations: string[];
+}
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the last non-empty line. Returns '' when nothing is parseable
+ * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.
+ */
+declare function parseFinalAnswer(artifact: string): string;
+/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */
+declare function parseCitations(artifact: string): string[];
+declare function createSimpleQaAdapter(): BenchmarkAdapter;
+
+export { type ResearchAnswer, type ResearchTask, createSimpleQaAdapter, parseCitations, parseFinalAnswer };

package/dist/benchmarks/simpleqa.js

@@ -0,0 +1,11 @@
+import {
+  createSimpleQaAdapter,
+  parseCitations,
+  parseFinalAnswer
+} from "../chunk-SEVJPLZC.js";
+export {
+  createSimpleQaAdapter,
+  parseCitations,
+  parseFinalAnswer
+};
+//# sourceMappingURL=simpleqa.js.map

package/dist/benchmarks/simpleqa.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/swe-bench.d.ts

@@ -0,0 +1,56 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { StagedRunCaptureSpec } from './_harness.js';
+import { BenchmarkAdapter, BenchScore } from './types.js';
+
+/**
+ * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =
+ * the official `swebench` harness: apply the patch in the instance's Docker
+ * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic
+ * — no LLM judge.
+ *
+ * Requires: the bench `.venv` with `swebench` installed + a running Docker
+ * daemon (per-instance images are pulled/built on first run).
+ *
+ * Process/Docker/report plumbing is shared via ./_harness; this file owns the
+ * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the
+ * predictions-file → run_evaluation argv → report-shape mapping.
+ */
+
+/**
+ * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
+ *
+ * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's
+ * actual edits, read from the cloned repo's STATE inside the box (standard
+ * SWE-bench practice). This event-stream parse only runs when that diff is empty
+ * — a model that edited the source correctly but never printed a fenced diff (the
+ * exact failure this replaces) still scores off its real changes, not its prose.
+ */
+declare const swePatchOutput: OutputAdapter<string>;
+type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance';
+interface SweBenchArtifactCaptureContext {
+    readonly taskId: string;
+    readonly runId: string;
+    /** One-based sequence unique within this adapter instance. */
+    readonly attemptSequence: number;
+}
+interface SweBenchAdapterOptions {
+    readonly timeoutMs?: number;
+    readonly cacheLevel?: SweBenchCacheLevel;
+    /**
+     * Return a unique destination for any attempt whose complete official
+     * evaluator directory and process logs should be retained.
+     */
+    readonly captureEvaluatorArtifacts?: (context: SweBenchArtifactCaptureContext) => StagedRunCaptureSpec | undefined;
+}
+/** Convert one official report into a score without turning evaluator failures into agent failures. */
+declare function scoreSweReport(taskId: string, value: unknown): BenchScore;
+declare function sweEvaluationArgv(args: {
+    readonly predictionsPath: string;
+    readonly runId: string;
+    readonly instanceId: string;
+    readonly cacheLevel: SweBenchCacheLevel;
+    readonly namespace?: 'swebench' | 'none';
+}): string[];
+declare function createSweBenchAdapter(options?: SweBenchAdapterOptions): BenchmarkAdapter;
+
+export { type SweBenchAdapterOptions, type SweBenchArtifactCaptureContext, type SweBenchCacheLevel, createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput };

package/dist/benchmarks/swe-bench.js

@@ -0,0 +1,14 @@
+import {
+  createSweBenchAdapter,
+  scoreSweReport,
+  sweEvaluationArgv,
+  swePatchOutput
+} from "../chunk-CKUVRZ2T.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createSweBenchAdapter,
+  scoreSweReport,
+  sweEvaluationArgv,
+  swePatchOutput
+};
+//# sourceMappingURL=swe-bench.js.map

package/dist/benchmarks/swe-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/t2-ragbench.d.ts

@@ -0,0 +1,14 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * T2-RAGBench adapter.
+ *
+ * T2-RAGBench stresses text+table retrieval and numerical reasoning over
+ * financial documents. The judge uses the shared deterministic answer scorer
+ * with numeric tolerance enabled by default.
+ */
+
+declare function createT2RagBenchAdapter(): BenchmarkAdapter;
+
+export { createT2RagBenchAdapter };

package/dist/benchmarks/t2-ragbench.js

@@ -0,0 +1,9 @@
+import {
+  createT2RagBenchAdapter
+} from "../chunk-Y6O2OCUO.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createT2RagBenchAdapter
+};
+//# sourceMappingURL=t2-ragbench.js.map

package/dist/benchmarks/t2-ragbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/tau-bench-shared.d.ts

@@ -0,0 +1,26 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * Shared tau-bench adapter spine.
+ *
+ * tau2 and tau3 live in the same upstream repository/package namespace today:
+ * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation
+ * for task loading and reward recomputation so the domain/version adapters only
+ * choose env names, default domain, and fixture file.
+ */
+
+interface TauBenchConfig {
+    name: string;
+    fixturePath: string;
+    fixturesEnv: string;
+    dirEnv: string;
+    domainEnv: string;
+    defaultDomain: string;
+    installHint: string;
+    taskIntro: string;
+}
+declare const tauResultsOutput: OutputAdapter<string>;
+declare function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter;
+
+export { type TauBenchConfig, createTauBenchAdapter, tauResultsOutput };

package/dist/benchmarks/tau-bench-shared.js

@@ -0,0 +1,10 @@
+import {
+  createTauBenchAdapter,
+  tauResultsOutput
+} from "../chunk-R36V2VP7.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createTauBenchAdapter,
+  tauResultsOutput
+};
+//# sourceMappingURL=tau-bench-shared.js.map