@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/benchmarks/aec-bench.js

@@ -0,0 +1,8 @@
+import {
+  createAecBenchAdapter
+} from "../chunk-ODT47UAY.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createAecBenchAdapter
+};
+//# sourceMappingURL=aec-bench.js.map

package/dist/benchmarks/aec-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/agentbench.d.ts

@@ -0,0 +1,16 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * AgentBench deterministic subset adapter.
+ *
+ * This targets AgentBench DBBench rows only: question + table + published label.
+ * It does not wrap AgentBench's controller protocol or the non-deterministic game
+ * environments. Worker artifact = final answer text. Judge = exact match against
+ * the official DBBench label list after light whitespace/case normalization.
+ */
+
+declare const agentbenchAnswerOutput: OutputAdapter<string>;
+declare function createAgentBenchAdapter(): BenchmarkAdapter;
+
+export { agentbenchAnswerOutput, createAgentBenchAdapter };

package/dist/benchmarks/agentbench.js

@@ -0,0 +1,10 @@
+import {
+  agentbenchAnswerOutput,
+  createAgentBenchAdapter
+} from "../chunk-7WSD27QQ.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  agentbenchAnswerOutput,
+  createAgentBenchAdapter
+};
+//# sourceMappingURL=agentbench.js.map

package/dist/benchmarks/agentbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/appworld.d.ts

@@ -0,0 +1,37 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's
+ * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`
+ * surface AppWorld exposes inside `world.execute(...)`), ending in
+ * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic
+ * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,
+ * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`
+ * (per-requirement total) and the `passes`/`failures` lists. Score =
+ * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.
+ *
+ * loadTasks enumerates the real task suite via `load_task_ids(split)`
+ * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.
+ * The OutputAdapter is stream-only, so the worker emits its solution as a fenced
+ * ```python block which the driver executes.
+ *
+ * Requires for a live run: the bench `.venv` with `appworld` installed + the
+ * unpacked engine + downloaded data (`appworld install` ; `appworld download
+ * data`). preflight + loadTasks + judge all fail loud with the exact step when the
+ * engine/data is absent — never a fabricated score.
+ */
+
+/** Worker solution code = the last fenced ```python block, else the raw text. */
+declare const appworldSolutionOutput: OutputAdapter<string>;
+declare function createAppWorldAdapter(): BenchmarkAdapter;
+/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
+declare function appworldToolLoopClient(cfg: {
+    model: string;
+    routerBaseUrl: string;
+    routerKey: string;
+    maxTurns?: number;
+}): unknown;
+declare function createAppWorldReactAdapter(): BenchmarkAdapter;
+
+export { appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };

package/dist/benchmarks/appworld.js

@@ -0,0 +1,14 @@
+import {
+  appworldSolutionOutput,
+  appworldToolLoopClient,
+  createAppWorldAdapter,
+  createAppWorldReactAdapter
+} from "../chunk-SYDW647C.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  appworldSolutionOutput,
+  appworldToolLoopClient,
+  createAppWorldAdapter,
+  createAppWorldReactAdapter
+};
+//# sourceMappingURL=appworld.js.map

package/dist/benchmarks/appworld.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/bfcl.d.ts

@@ -0,0 +1,18 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * Berkeley Function Calling Leaderboard adapter.
+ *
+ * Scope: deterministic function-call ground-truth categories from the official
+ * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic
+ * web-search/memory categories and BFCL's own model-response harness remain
+ * upstream responsibilities. The adapter loads official JSONL rows plus their
+ * `possible_answer` file and scores structured function-call artifacts against
+ * allowed function/argument values.
+ */
+
+declare const bfclOutput: OutputAdapter<string>;
+declare function createBfclAdapter(): BenchmarkAdapter;
+
+export { bfclOutput, createBfclAdapter };

package/dist/benchmarks/bfcl.js

@@ -0,0 +1,10 @@
+import {
+  bfclOutput,
+  createBfclAdapter
+} from "../chunk-IFAV6KEM.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  bfclOutput,
+  createBfclAdapter
+};
+//# sourceMappingURL=bfcl.js.map

package/dist/benchmarks/bfcl.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cad-design.d.ts

@@ -0,0 +1,45 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the
+ * REAL OpenSCAD engine: compile + export STL, then measure the produced solid
+ * geometry against the task's spec. Fully deterministic — the authoritative CAD
+ * kernel is the gate, not an LLM and not the agent's own claim.
+ *
+ * This is verifiable-reward CAD: an agent is far better at *writing* exact
+ * parametric code than at clicking a GUI, and the kernel either produces the
+ * specified geometry or it doesn't. Every judged artifact also renders a PNG,
+ * so a refine loop's attempts become a watchable "model getting better" reel
+ * (run-capsule consumes the renders).
+ *
+ * Spec checks are geometric and ungameable:
+ *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)
+ *   - volumes       : disconnected-solid count within [min,max]
+ *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)
+ *   - detail        : triangle count ≥ floor (not a degenerate single cube)
+ *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat
+ *                     box can't fake this; a real gabled/hipped roof tapers
+ *   - hollow        : interior cavity present (walls, not a solid block) — the
+ *                     bbox volume materially exceeds the printed solid volume
+ *
+ * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).
+ */
+
+/** Spec assertions a CAD task can require. All are deterministic + geometric. */
+interface CadSpec {
+    volumes?: [number, number];
+    bbox?: {
+        x?: [number, number];
+        y?: [number, number];
+        z?: [number, number];
+    };
+    minTriangles?: number;
+    /** Top-band XY footprint must be < this fraction of the base footprint. */
+    pitchedRoof?: number;
+    /** Printed solid volume must be < this fraction of the bbox volume (hollow). */
+    hollowBelow?: number;
+}
+declare function createCadDesignAdapter(): BenchmarkAdapter;
+
+export { type CadSpec, createCadDesignAdapter };

package/dist/benchmarks/cad-design.js

@@ -0,0 +1,7 @@
+import {
+  createCadDesignAdapter
+} from "../chunk-53UPUNBZ.js";
+export {
+  createCadDesignAdapter
+};
+//# sourceMappingURL=cad-design.js.map

package/dist/benchmarks/cad-design.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cadbench.d.ts

@@ -0,0 +1,19 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).
+ * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria
+ * eval: render the produced model to standardized views, then a vision judge
+ * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the
+ * rendered images + the script text. score = fraction of criteria satisfied.
+ *
+ * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,
+ * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL
+ * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from
+ * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).
+ */
+
+declare function createCadBenchAdapter(): BenchmarkAdapter;
+
+export { createCadBenchAdapter };

package/dist/benchmarks/cadbench.js

@@ -0,0 +1,8 @@
+import {
+  createCadBenchAdapter
+} from "../chunk-PUIRNYI7.js";
+import "../chunk-JTHWEDEW.js";
+export {
+  createCadBenchAdapter
+};
+//# sourceMappingURL=cadbench.js.map

package/dist/benchmarks/cadbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cadgenbench.d.ts

@@ -0,0 +1,22 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part
+ * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN
+ * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to
+ * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.
+ * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.
+ *
+ * The official task set (private GT, server-side graded) isn't released yet, so
+ * tasks here are seeded from the repo's dimension-named geometry fixtures (real
+ * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap
+ * loadTasks to read the published fixtures' description.yaml + ground_truth.step.
+ *
+ * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +
+ * xvfb (the scorer's alignment renders need a display).
+ */
+
+declare function createCadGenBenchAdapter(): BenchmarkAdapter;
+
+export { createCadGenBenchAdapter };

package/dist/benchmarks/cadgenbench.js

@@ -0,0 +1,8 @@
+import {
+  createCadGenBenchAdapter
+} from "../chunk-C7T7WEK2.js";
+import "../chunk-JTHWEDEW.js";
+export {
+  createCadGenBenchAdapter
+};
+//# sourceMappingURL=cadgenbench.js.map

package/dist/benchmarks/cadgenbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/commit0.d.ts

@@ -0,0 +1,31 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.
+ * Each record is a Python repo stubbed at `base_commit` (public function bodies
+ * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker
+ * artifact = a unified diff that fills in `src_dir`. Judge = the official
+ * `commit0` test harness: it stages the starter repo, applies the worker's diff,
+ * builds the library's deps and runs `pytest`, writing a per-repo pytest-json
+ * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the
+ * macro-averaged unit-test pass-rate the leaderboard reports). Fully
+ * deterministic — no LLM judge.
+ *
+ * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its
+ * implementation as a fenced ```diff against the stubbed repo — same deliverable
+ * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the
+ * real `commit0` harness on a local Docker backend (NOT reimplemented here).
+ *
+ * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed
+ * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +
+ * a Docker daemon (`--backend local`). For offline/CI dataset listing set
+ * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)
+ * — judging still needs the harness + Docker and fails loud, never a fabricated score.
+ */
+
+/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */
+declare const commit0DiffOutput: OutputAdapter<string>;
+declare function createCommit0Adapter(): BenchmarkAdapter;
+
+export { commit0DiffOutput, createCommit0Adapter };

package/dist/benchmarks/commit0.js

@@ -0,0 +1,10 @@
+import {
+  commit0DiffOutput,
+  createCommit0Adapter
+} from "../chunk-2XU6OGEN.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  commit0DiffOutput,
+  createCommit0Adapter
+};
+//# sourceMappingURL=commit0.js.map

package/dist/benchmarks/commit0.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/crag.d.ts

@@ -0,0 +1,14 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CRAG adapter (Comprehensive RAG Benchmark).
+ *
+ * Live mode expects an official or compatible CRAG JSON/JSONL export. The
+ * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final
+ * answers deterministically against the provided gold answer list.
+ */
+
+declare function createCragAdapter(): BenchmarkAdapter;
+
+export { createCragAdapter };

package/dist/benchmarks/crag.js

@@ -0,0 +1,9 @@
+import {
+  createCragAdapter
+} from "../chunk-HHXFIHXC.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createCragAdapter
+};
+//# sourceMappingURL=crag.js.map

package/dist/benchmarks/crag.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/dabstep.d.ts

@@ -0,0 +1,18 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic
+ * payment files. Worker artifact = final answer text. Judge = the official
+ * DABStep `grade.py` normalization/matching function. No LLM judge.
+ *
+ * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that
+ * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter
+ * exposes `metadata.resourceRoot` so runners can mount the benchmark files into
+ * AgentProfile.resources.files; it does not paste the dataset into prompt text.
+ */
+
+declare const dabstepAnswerOutput: OutputAdapter<string>;
+declare function createDabstepAdapter(): BenchmarkAdapter;
+
+export { createDabstepAdapter, dabstepAnswerOutput };

package/dist/benchmarks/dabstep.js

@@ -0,0 +1,10 @@
+import {
+  createDabstepAdapter,
+  dabstepAnswerOutput
+} from "../chunk-J3KDJNX2.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createDabstepAdapter,
+  dabstepAnswerOutput
+};
+//# sourceMappingURL=dabstep.js.map

package/dist/benchmarks/dabstep.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/enterpriseops-gym.d.ts

@@ -0,0 +1,38 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —
+ * stateful agentic planning + tool use in enterprise settings. Each record is an
+ * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,
+ * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`
+ * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).
+ * Worker artifact = the ordered tool-call transcript the agent would issue against
+ * those servers, emitted as a single fenced ```json block of
+ * `{ "calls": [ { "tool": ..., "arguments": {...}, "gym_name"?: ... } ] }`.
+ *
+ * Judge = the benchmark's OWN deterministic state-checker. The driver replays the
+ * transcript against a freshly-seeded gym server (mutating its database), then runs
+ * each task's `database_state` verifier — an SQL SELECT executed via the gym
+ * server's /api/sql-runner endpoint, compared to `expected_value` under
+ * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =
+ * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;
+ * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully
+ * deterministic — no LLM judge.
+ *
+ * loadTasks enumerates the real suite from the HF rows server (config = tool-set
+ * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed
+ * sample (bench/fixtures/enterpriseops-gym.json) loads offline.
+ *
+ * Requires for a LIVE judge run: a Docker daemon with the domain gym images
+ * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up
+ * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge
+ * fail loud with the exact pull/run/unzip step when a server is unreachable — never
+ * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.
+ */
+
+/** Worker transcript = the last fenced ```json block, else the raw text. */
+declare const enterpriseOpsTranscriptOutput: OutputAdapter<string>;
+declare function createEnterpriseOpsGymAdapter(): BenchmarkAdapter;
+
+export { createEnterpriseOpsGymAdapter, enterpriseOpsTranscriptOutput };

package/dist/benchmarks/enterpriseops-gym.js

@@ -0,0 +1,10 @@
+import {
+  createEnterpriseOpsGymAdapter,
+  enterpriseOpsTranscriptOutput
+} from "../chunk-X5YKXC6V.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createEnterpriseOpsGymAdapter,
+  enterpriseOpsTranscriptOutput
+};
+//# sourceMappingURL=enterpriseops-gym.js.map

package/dist/benchmarks/enterpriseops-gym.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/finresearchbench.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * FinResearchBench-compatible adapter.
+ *
+ * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial
+ * research reports, but there is no stable public scorer package wired here.
+ * Live mode therefore requires a local data export whose rows carry the official
+ * judge prompt/template/logic tree. The adapter refuses to invent a judge.
+ */
+
+declare function createFinResearchBenchAdapter(): BenchmarkAdapter;
+
+export { createFinResearchBenchAdapter };

package/dist/benchmarks/finresearchbench.js

@@ -0,0 +1,8 @@
+import {
+  createFinResearchBenchAdapter
+} from "../chunk-YCGY7UIZ.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createFinResearchBenchAdapter
+};
+//# sourceMappingURL=finresearchbench.js.map

package/dist/benchmarks/finresearchbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/finsearchcomp.d.ts

@@ -0,0 +1,49 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +
+ * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).
+ *
+ * Each record is SELF-CONTAINED and carries its OWN judge: a per-record
+ * `judge_system_prompt` + a `judge_prompt_template` with {prompt},
+ * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),
+ * and {response} (the student answer) placeholders. We faithfully REPLICATE the
+ * benchmark's judge — fill the template, run it under the record's system prompt
+ * via the pinned router (temperature 0), and parse the JSON the judge emits. The
+ * judge returns an `answer_score` field; 1/positive = resolved, else not.
+ * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).
+ *
+ * There is NO deterministic Tier-1 path here: the gold answer is prose with
+ * region-specific units, embedded tolerances ("允许1%的误差"), and multi-point
+ * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,
+ * so the judge IS the score — fail loud on unparseable judge output (never
+ * default to resolved).
+ *
+ * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)
+ * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)
+ * needs a live market snapshot / akshare ground truth to fill the judge's
+ * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.
+ * See {@link T1Seam}.
+ *
+ * Requires for a live run: network access to the GitHub-hosted dataset JSON and
+ * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back
+ * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit
+ * console.warn — never a silent fallback.
+ */
+
+/**
+ * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a
+ * live market snapshot the judge consumes via a {ground_truth} template slot;
+ * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a
+ * resolver that fetches the akshare/market value for the record at judge time
+ * and filling {ground_truth}. Until then T1 is excluded from loadTasks.
+ */
+interface T1Seam {
+    promptId: string;
+    /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */
+    resolveGroundTruth(promptId: string): Promise<string>;
+}
+declare function createFinsearchcompAdapter(): BenchmarkAdapter;
+
+export { type T1Seam, createFinsearchcompAdapter };

package/dist/benchmarks/finsearchcomp.js

@@ -0,0 +1,7 @@
+import {
+  createFinsearchcompAdapter
+} from "../chunk-UPAMRDX4.js";
+export {
+  createFinsearchcompAdapter
+};
+//# sourceMappingURL=finsearchcomp.js.map

package/dist/benchmarks/finsearchcomp.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/frames.d.ts

@@ -0,0 +1,59 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text
+ * final answer string (optionally preceded by a citations block). FRAMES ships,
+ * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs
+ * needed to answer) — the gold-citation column is what lets the loop's critic
+ * check citation coverage deterministically.
+ *
+ * Judge is two-tier, deterministic-first:
+ *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).
+ *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,
+ *            temperature 0), fired ONLY when Tier 1 misses, to absorb the
+ *            paraphrase/alias equivalence FRAMES intends to allow.
+ *
+ * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge
+ * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced
+ * in `detail` for the critic, never a hard pass criterion.
+ *
+ * Requires for a live run: the bench `.venv` with `datasets` installed and a
+ * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to
+ * load the committed fixtures (bench/fixtures/frames.json) — no HF download.
+ */
+
+/**
+ * Typed seam for the future dynamic-topology research worker. The benchmark
+ * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the
+ * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes
+ * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.
+ */
+interface ResearchTask {
+    id: string;
+    question: string;
+    /** Gold short answer — judge resolves strictly against this. */
+    gold: string;
+    /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */
+    goldSources: string[];
+    /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */
+    reasoningTypes: string;
+}
+interface ResearchAnswer {
+    finalAnswer: string;
+    citations: string[];
+}
+/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */
+declare function normalizeAnswer(input: string): string;
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the last non-empty line. Returns '' when nothing is parseable
+ * (fail-closed — never guess), which judge() counts as resolved=false.
+ */
+declare function parseFinalAnswer(artifact: string): string;
+/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */
+declare function parseCitations(artifact: string): string[];
+declare function createFramesAdapter(): BenchmarkAdapter;
+
+export { type ResearchAnswer, type ResearchTask, createFramesAdapter, normalizeAnswer, parseCitations, parseFinalAnswer };

package/dist/benchmarks/frames.js

@@ -0,0 +1,13 @@
+import {
+  createFramesAdapter,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer
+} from "../chunk-INNOYXCP.js";
+export {
+  createFramesAdapter,
+  normalizeAnswer,
+  parseCitations,
+  parseFinalAnswer
+};
+//# sourceMappingURL=frames.js.map

package/dist/benchmarks/frames.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}