@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/benchmarks/dabstep.d.ts

@@ -0,0 +1,18 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic
+ * payment files. Worker artifact = final answer text. Judge = the official
+ * DABStep `grade.py` normalization/matching function. No LLM judge.
+ *
+ * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that
+ * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter
+ * exposes `metadata.resourceRoot` so runners can mount the benchmark files into
+ * AgentProfile.resources.files; it does not paste the dataset into prompt text.
+ */
+
+declare const dabstepAnswerOutput: OutputAdapter<string>;
+declare function createDabstepAdapter(): BenchmarkAdapter;
+
+export { createDabstepAdapter, dabstepAnswerOutput };

package/dist/benchmarks/dabstep.js

@@ -0,0 +1,10 @@
+import {
+  createDabstepAdapter,
+  dabstepAnswerOutput
+} from "../chunk-J3KDJNX2.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createDabstepAdapter,
+  dabstepAnswerOutput
+};
+//# sourceMappingURL=dabstep.js.map

package/dist/benchmarks/dabstep.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/enterpriseops-gym.d.ts

@@ -0,0 +1,38 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —
+ * stateful agentic planning + tool use in enterprise settings. Each record is an
+ * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,
+ * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`
+ * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).
+ * Worker artifact = the ordered tool-call transcript the agent would issue against
+ * those servers, emitted as a single fenced ```json block of
+ * `{ "calls": [ { "tool": ..., "arguments": {...}, "gym_name"?: ... } ] }`.
+ *
+ * Judge = the benchmark's OWN deterministic state-checker. The driver replays the
+ * transcript against a freshly-seeded gym server (mutating its database), then runs
+ * each task's `database_state` verifier — an SQL SELECT executed via the gym
+ * server's /api/sql-runner endpoint, compared to `expected_value` under
+ * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =
+ * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;
+ * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully
+ * deterministic — no LLM judge.
+ *
+ * loadTasks enumerates the real suite from the HF rows server (config = tool-set
+ * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed
+ * sample (bench/fixtures/enterpriseops-gym.json) loads offline.
+ *
+ * Requires for a LIVE judge run: a Docker daemon with the domain gym images
+ * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up
+ * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge
+ * fail loud with the exact pull/run/unzip step when a server is unreachable — never
+ * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.
+ */
+
+/** Worker transcript = the last fenced ```json block, else the raw text. */
+declare const enterpriseOpsTranscriptOutput: OutputAdapter<string>;
+declare function createEnterpriseOpsGymAdapter(): BenchmarkAdapter;
+
+export { createEnterpriseOpsGymAdapter, enterpriseOpsTranscriptOutput };

package/dist/benchmarks/enterpriseops-gym.js

@@ -0,0 +1,10 @@
+import {
+  createEnterpriseOpsGymAdapter,
+  enterpriseOpsTranscriptOutput
+} from "../chunk-X5YKXC6V.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createEnterpriseOpsGymAdapter,
+  enterpriseOpsTranscriptOutput
+};
+//# sourceMappingURL=enterpriseops-gym.js.map

package/dist/benchmarks/enterpriseops-gym.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/finresearchbench.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * FinResearchBench-compatible adapter.
+ *
+ * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial
+ * research reports, but there is no stable public scorer package wired here.
+ * Live mode therefore requires a local data export whose rows carry the official
+ * judge prompt/template/logic tree. The adapter refuses to invent a judge.
+ */
+
+declare function createFinResearchBenchAdapter(): BenchmarkAdapter;
+
+export { createFinResearchBenchAdapter };

package/dist/benchmarks/finresearchbench.js

@@ -0,0 +1,8 @@
+import {
+  createFinResearchBenchAdapter
+} from "../chunk-YCGY7UIZ.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createFinResearchBenchAdapter
+};
+//# sourceMappingURL=finresearchbench.js.map

package/dist/benchmarks/finresearchbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/finsearchcomp.d.ts

@@ -0,0 +1,49 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +
+ * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).
+ *
+ * Each record is SELF-CONTAINED and carries its OWN judge: a per-record
+ * `judge_system_prompt` + a `judge_prompt_template` with {prompt},
+ * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),
+ * and {response} (the student answer) placeholders. We faithfully REPLICATE the
+ * benchmark's judge — fill the template, run it under the record's system prompt
+ * via the pinned router (temperature 0), and parse the JSON the judge emits. The
+ * judge returns an `answer_score` field; 1/positive = resolved, else not.
+ * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).
+ *
+ * There is NO deterministic Tier-1 path here: the gold answer is prose with
+ * region-specific units, embedded tolerances ("允许1%的误差"), and multi-point
+ * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,
+ * so the judge IS the score — fail loud on unparseable judge output (never
+ * default to resolved).
+ *
+ * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)
+ * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)
+ * needs a live market snapshot / akshare ground truth to fill the judge's
+ * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.
+ * See {@link T1Seam}.
+ *
+ * Requires for a live run: network access to the GitHub-hosted dataset JSON and
+ * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back
+ * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit
+ * console.warn — never a silent fallback.
+ */
+
+/**
+ * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a
+ * live market snapshot the judge consumes via a {ground_truth} template slot;
+ * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a
+ * resolver that fetches the akshare/market value for the record at judge time
+ * and filling {ground_truth}. Until then T1 is excluded from loadTasks.
+ */
+interface T1Seam {
+    promptId: string;
+    /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */
+    resolveGroundTruth(promptId: string): Promise<string>;
+}
+declare function createFinsearchcompAdapter(): BenchmarkAdapter;
+
+export { type T1Seam, createFinsearchcompAdapter };

package/dist/benchmarks/finsearchcomp.js

@@ -0,0 +1,7 @@
+import {
+  createFinsearchcompAdapter
+} from "../chunk-UPAMRDX4.js";
+export {
+  createFinsearchcompAdapter
+};
+//# sourceMappingURL=finsearchcomp.js.map

package/dist/benchmarks/finsearchcomp.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/frames.d.ts

@@ -0,0 +1,59 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text
+ * final answer string (optionally preceded by a citations block). FRAMES ships,
+ * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs
+ * needed to answer) — the gold-citation column is what lets the loop's critic
+ * check citation coverage deterministically.
+ *
+ * Judge is two-tier, deterministic-first:
+ *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).
+ *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,
+ *            temperature 0), fired ONLY when Tier 1 misses, to absorb the
+ *            paraphrase/alias equivalence FRAMES intends to allow.
+ *
+ * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge
+ * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced
+ * in `detail` for the critic, never a hard pass criterion.
+ *
+ * Requires for a live run: the bench `.venv` with `datasets` installed and a
+ * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to
+ * load the committed fixtures (bench/fixtures/frames.json) — no HF download.
+ */
+
+/**
+ * Typed seam for the future dynamic-topology research worker. The benchmark
+ * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the
+ * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes
+ * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.
+ */
+interface ResearchTask {
+    id: string;
+    question: string;
+    /** Gold short answer — judge resolves strictly against this. */
+    gold: string;
+    /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */
+    goldSources: string[];
+    /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */
+    reasoningTypes: string;
+}
+interface ResearchAnswer {
+    finalAnswer: string;
+    citations: string[];
+}
+/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */
+declare function normalizeAnswer(input: string): string;
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the last non-empty line. Returns '' when nothing is parseable
+ * (fail-closed — never guess), which judge() counts as resolved=false.
+ */
+declare function parseFinalAnswer(artifact: string): string;
+/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */
+declare function parseCitations(artifact: string): string[];
+declare function createFramesAdapter(): BenchmarkAdapter;
+
+export { type ResearchAnswer, type ResearchTask, createFramesAdapter, normalizeAnswer, parseCitations, parseFinalAnswer };

package/dist/benchmarks/frames.js

@@ -0,0 +1,13 @@
+import {
+  createFramesAdapter,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer
+} from "../chunk-INNOYXCP.js";
+export {
+  createFramesAdapter,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer
+};
+//# sourceMappingURL=frames.js.map

package/dist/benchmarks/frames.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/hotpotqa.d.ts

@@ -0,0 +1,48 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').
+ * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.
+ *
+ * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no
+ * LLM. Both the predicted final answer and the gold are normalized (lowercase,
+ * strip articles a/an/the, strip punctuation, collapse whitespace), then scored
+ * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS
+ * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the
+ * suite a judge that needs no model tokens at all.
+ *
+ * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of
+ * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,
+ * never part of the score.
+ *
+ * Requires for a live run: the bench `.venv` with `datasets` installed + network
+ * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the
+ * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.
+ */
+
+/**
+ * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,
+ * drop articles (a/an/the), collapse whitespace. Token comparisons run on the
+ * output of this exactly as the published evaluator does.
+ */
+declare function normalizeAnswer(input: string): string;
+/** Exact match on the normalized strings. */
+declare function exactMatch(prediction: string, gold: string): boolean;
+/**
+ * Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of
+ * normalized tokens. Mirrors the published evaluator's special-case handling of
+ * yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both
+ * are empty, else 0.
+ */
+declare function tokenF1(prediction: string, gold: string): number;
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the trimmed last non-empty line. Returns '' when nothing is
+ * parseable (fail-closed — never guess), which judge() counts as resolved=false.
+ */
+declare function parseFinalAnswer(artifact: string): string;
+declare function createHotpotqaAdapter(): BenchmarkAdapter;
+
+export { createHotpotqaAdapter, exactMatch, normalizeAnswer, parseFinalAnswer, tokenF1 };

package/dist/benchmarks/hotpotqa.js

@@ -0,0 +1,15 @@
+import {
+  createHotpotqaAdapter,
+  exactMatch,
+  normalizeAnswer,
+  parseFinalAnswer,
+  tokenF1
+} from "../chunk-PA2ZKHJC.js";
+export {
+  createHotpotqaAdapter,
+  exactMatch,
+  normalizeAnswer,
+  parseFinalAnswer,
+  tokenF1
+};
+//# sourceMappingURL=hotpotqa.js.map

package/dist/benchmarks/hotpotqa.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/humaneval.d.ts

@@ -0,0 +1,62 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the
+ * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout
+ * through the `Supervisor` that self-corrects across rounds, vs blind
+ * random@k. This is the experiment `humaneval-gate.mts` names as "the next one" —
+ * the gate measures SELECTION over stateless single completions; this measures
+ * whether observe→steer (self-correction) beats blind compute at equal k.
+ *
+ * Worker artifact = the model's reply (a Python function, fenced or raw). The
+ * DETERMINISTIC judge runs the candidate against the task's own `test` in an
+ * isolated `--network=none` python container — exit 0 = pass. No gold
+ * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only
+ * to self-verify the judge before spending tokens.
+ *
+ * The primitives (loader / extractor / Docker checker) live here and are reused by
+ * `humaneval-gate.mts` — one home, no duplication.
+ */
+
+interface HumanEvalTask {
+    taskId: string;
+    prompt: string;
+    test: string;
+    entryPoint: string;
+    /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */
+    canonicalSolution?: string;
+}
+/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch
+ *  or a malformed line — a silently-short task set would poison the gate. `offset`
+ *  selects a deeper slice (the later tasks are harder) so the worker has a
+ *  correctable middle band rather than a saturated easy prefix. */
+declare function loadHumanEval(limit: number, offset?: number): Promise<HumanEvalTask[]>;
+declare function basePrompt(task: HumanEvalTask): string;
+/** Extract the function source from a model reply: prefer a fenced ```python (or
+ *  bare ```) block, else fall back to the raw text. The deployable program adds the
+ *  prompt header (imports + signature context), so a candidate that returns only a
+ *  body still runs; a candidate that re-defines the function shadows the header. */
+declare function extractCode(reply: string): string;
+interface CheckResult {
+    /** {0,1} pass-count for this candidate (1 = the check() suite passed). */
+    pass: number;
+    /** On failure: the interpreter stderr tail (traceback / failing assertion). The
+     *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */
+    detail?: string;
+}
+interface PythonProgramResult {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+}
+/** Run arbitrary Python in the same isolated container used by the HumanEval
+ * checker. This is also the execution-tool primitive for experiments that let
+ * a model test its own snippets; model-written code must never run on the host. */
+declare function runPythonProgram(program: string, timeoutMs?: number): Promise<PythonProgramResult>;
+declare function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult>;
+/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle
+ *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */
+declare function createHumanEvalAdapter(): BenchmarkAdapter;
+
+export { type CheckResult, type HumanEvalTask, type PythonProgramResult, basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker, runPythonProgram };

package/dist/benchmarks/humaneval.js

@@ -0,0 +1,17 @@
+import {
+  basePrompt,
+  createHumanEvalAdapter,
+  extractCode,
+  loadHumanEval,
+  runChecker,
+  runPythonProgram
+} from "../chunk-5H5XV76F.js";
+export {
+  basePrompt,
+  createHumanEvalAdapter,
+  extractCode,
+  loadHumanEval,
+  runChecker,
+  runPythonProgram
+};
+//# sourceMappingURL=humaneval.js.map

package/dist/benchmarks/humaneval.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/mind2web.d.ts

@@ -0,0 +1,41 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).
+ * Web-agent ACTION PREDICTION: each task is one step of a real web task — given
+ * the natural-language goal and the page's candidate elements, the worker picks
+ * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).
+ *
+ * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:
+ *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids
+ *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches
+ *   resolved (Step-SR) ⇔ element correct AND operation correct
+ *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right
+ *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)
+ * This is the low-noise reward a certifiable directive-lift needs: the number is a
+ * programmatic match against human-verified ground truth, not a judge's opinion.
+ *
+ * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the
+ * judge extracts deterministically; the prompt presents the candidate set as a
+ * choice over backend_node_id ordered by id (position uncorrelated with the answer).
+ *
+ * Each step carries the dataset's OWN page screenshot (written to a temp file by
+ * the loader); the worker drops it into a browser.<op> span so run-capsule's screen
+ * capsule turns the run into a film — the real page, not a re-rendered DOM.
+ *
+ * Requires for a live run: a python with `datasets` + `pillow` and network to
+ * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).
+ */
+
+interface ParsedAction {
+    elementId: string;
+    op: string;
+    value: string;
+}
+/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.
+ *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */
+declare function parseAction(artifact: string): ParsedAction | null;
+declare function createMind2WebAdapter(): BenchmarkAdapter;
+
+export { createMind2WebAdapter, parseAction };

package/dist/benchmarks/mind2web.js

@@ -0,0 +1,9 @@
+import {
+  createMind2WebAdapter,
+  parseAction
+} from "../chunk-TBKU5XQI.js";
+export {
+  createMind2WebAdapter,
+  parseAction
+};
+//# sourceMappingURL=mind2web.js.map

package/dist/benchmarks/mind2web.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/nomiracl.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * NoMIRACL adapter.
+ *
+ * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does
+ * not generate an answer here; it classifies whether the supplied passages
+ * contain enough evidence to answer the query. This directly measures false
+ * positive / false negative behavior for RAG abstention.
+ */
+
+declare function createNoMiraclAdapter(): BenchmarkAdapter;
+
+export { createNoMiraclAdapter };

package/dist/benchmarks/nomiracl.js

@@ -0,0 +1,9 @@
+import {
+  createNoMiraclAdapter
+} from "../chunk-TSWPNOYM.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createNoMiraclAdapter
+};
+//# sourceMappingURL=nomiracl.js.map

package/dist/benchmarks/nomiracl.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/open-rag-bench.d.ts

@@ -0,0 +1,14 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Open RAG Bench adapter.
+ *
+ * This targets Vectara-style Open RAG Bench exports over PDF-derived text,
+ * table, and image contexts. The deterministic judge scores final-answer
+ * agreement and surfaces modality/document metadata for diagnostics.
+ */
+
+declare function createOpenRagBenchAdapter(): BenchmarkAdapter;
+
+export { createOpenRagBenchAdapter };

package/dist/benchmarks/open-rag-bench.js

@@ -0,0 +1,9 @@
+import {
+  createOpenRagBenchAdapter
+} from "../chunk-KDIKRJGB.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createOpenRagBenchAdapter
+};
+//# sourceMappingURL=open-rag-bench.js.map

package/dist/benchmarks/open-rag-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/programbench.d.ts

@@ -0,0 +1,38 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom
+ * reverse-engineering: rebuild a black-box executable's behavior from scratch.
+ * The agent is given only the gold `./executable` (run-only) + stripped docs and
+ * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`
+ * with identical observable behavior. Judge = the official `programbench`
+ * harness: it extracts the submission, runs compile.sh in the per-task cleanroom
+ * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via
+ * `programbench blob sync`). Score = fraction of non-ignored tests passed
+ * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests
+ * pass. Fully deterministic — no LLM judge.
+ *
+ * OutputAdapter is stream-only, so the worker emits its codebase as fenced
+ * `path:`-prefixed file blocks (including compile.sh); the adapter materializes
+ * those into submission.tar.gz. Test execution is delegated to the real
+ * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT
+ * reimplemented here.
+ *
+ * Requires for a live run: the bench `.venv` with `programbench` installed +
+ * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for
+ * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load
+ * the committed instance ids (bench/fixtures/programbench.json); judging still
+ * needs the harness + Docker and fails loud without them — never a fabricated score.
+ */
+
+/**
+ * Parse the worker stream into the submission text the driver materializes: the
+ * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to
+ * the python driver, which tars it. Empty when the worker emitted no file block
+ * (fail-closed → the harness scores a missing compile.sh as 0).
+ */
+declare const programbenchSubmissionOutput: OutputAdapter<string>;
+declare function createProgrambenchAdapter(): BenchmarkAdapter;
+
+export { createProgrambenchAdapter, programbenchSubmissionOutput };

package/dist/benchmarks/programbench.js

@@ -0,0 +1,10 @@
+import {
+  createProgrambenchAdapter,
+  programbenchSubmissionOutput
+} from "../chunk-ZEWMTR5M.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createProgrambenchAdapter,
+  programbenchSubmissionOutput
+};
+//# sourceMappingURL=programbench.js.map

package/dist/benchmarks/programbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/rag-shared.d.ts

@@ -0,0 +1,42 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchScore, BenchTask, LoadOptions } from './types.js';
+
+declare const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+interface RagContext {
+    id: string;
+    text: string;
+    title?: string;
+    source?: string;
+    relevant?: boolean;
+}
+interface RagAnswerScore {
+    resolved: boolean;
+    score: number;
+    finalAnswer: string;
+    bestGold: string | null;
+    exact: boolean;
+    numeric: boolean;
+    f1: number;
+    threshold: number;
+}
+declare const ragAnswerOutput: OutputAdapter<string>;
+declare function parseFinalAnswer(artifact: string): string;
+declare function parseCitations(artifact: string): string[];
+declare function normalizeAnswer(input: string): string;
+declare function tokenF1(candidate: string, gold: string): number;
+declare function scoreAnswerArtifact(artifact: string, golds: readonly string[], options?: {
+    threshold?: number;
+    numericTolerance?: number;
+}): RagAnswerScore;
+declare function answerScoreToBenchScore(score: RagAnswerScore, detail: Record<string, unknown>): BenchScore;
+declare function readJsonRows(path: string): Promise<unknown[]>;
+declare function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[];
+declare function stringFrom(value: unknown): string | undefined;
+declare function stringArrayFrom(value: unknown): string[];
+declare function firstString(row: Record<string, unknown>, keys: readonly string[]): string;
+declare function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[];
+declare function contextsFrom(value: unknown): RagContext[];
+declare function contextBlock(contexts: readonly RagContext[]): string;
+declare function isObject(value: unknown): value is Record<string, unknown>;
+
+export { FINAL_ANSWER_SENTINEL, type RagAnswerScore, type RagContext, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringArrayFrom, stringFrom, tokenF1 };