@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/index.d.ts

@@ -0,0 +1,355 @@
+export { ADAPTERS, resolveAdapter } from './adapters.js';
+export { createCragAdapter } from './benchmarks/crag.js';
+export { createNoMiraclAdapter } from './benchmarks/nomiracl.js';
+export { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench.js';
+export { createRagBenchAdapter } from './benchmarks/ragbench.js';
+export { FINAL_ANSWER_SENTINEL, RagAnswerScore, RagContext, answerScoreToBenchScore, contextBlock, contextsFrom, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, scoreAnswerArtifact, tokenF1 } from './benchmarks/rag-shared.js';
+export { createT2RagBenchAdapter } from './benchmarks/t2-ragbench.js';
+export { SweBenchAdapterOptions, SweBenchArtifactCaptureContext, SweBenchCacheLevel, createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput } from './benchmarks/swe-bench.js';
+export { StagedJudgeError, StagedRunCaptureSpec, StagedRunSpec, runStagedJudge } from './benchmarks/_harness.js';
+import { BenchmarkAdapter, BenchTask } from './benchmarks/types.js';
+export { BenchScore, JudgeArtifactFileReceipt, JudgeArtifactReceipt, LoadOptions } from './benchmarks/types.js';
+import { SandboxClient, AgentProfile } from '@tangle-network/agent-runtime/loops';
+import { PreparedAgentCandidateExecution, AgentCandidateExecutionClaimStore, AgentCandidateOutputArtifactPort, AgentCandidateBenchmarkGraderPort, AgentCandidateExecutorRequest, AgentCandidateExecutorStopRequest, AgentCandidateExecutorPort, AgentCandidateRunFinalization } from '@tangle-network/agent-runtime';
+import { TraceStore } from '@tangle-network/agent-eval';
+
+/**
+ * The ONE harness → worker-client mapping for the bench layer. Every bench entrypoint
+ * (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runLoop` drives from a single
+ * selector instead of hand-rolling the branch:
+ *
+ *   - `router`               → OFF-BOX: a router chat-completion as the leaf executor, presented
+ *                              as a SandboxClient (no sandbox dependency). For deployable-checker
+ *                              domains whose worker is a completion, or where box egress is blocked.
+ *   - `router` + searchProvider → OFF-BOX `router-tools`: the same off-box loop with a live
+ *                              `web_search` tool (the capability axis research benches need).
+ *   - `bridge`               → OFF-BOX: a local cli-bridge fronting a harness CLI
+ *                              (opencode/kimi-code/…) as the leaf executor. Same resumable
+ *                              `bridgeExecutor` the supervisor uses; harness+model ride the
+ *                              bridge `model` id (`${harness}/${model}`).
+ *   - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
+ *                              TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
+ *                              this only decides off-box-vs-in-box transport for `runLoop`.
+ *
+ * Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
+ * the mapping can't drift between callers.
+ */
+
+interface ResolveBenchClientOptions {
+    /** The selector (the `BACKEND` env value): `router` = off-box; anything else = in-box Sandbox. */
+    backend: string;
+    routerBaseUrl: string;
+    routerKey: string;
+    model: string;
+    /** When set on the `router` backend, the off-box worker becomes a `router-tools` agentic loop
+     *  with a live `web_search` tool backed by this provider (`you`/`exa`/…). */
+    searchProvider?: string;
+    sandboxBaseUrl?: string;
+    /** In-box sandbox timeout (ms). Also the per-turn deadline for the `bridge` backend. */
+    timeoutMs?: number;
+    /** `bridge` backend: cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. */
+    bridgeUrl?: string;
+    /** `bridge` backend: bearer the bridge requires. Falls back to `routerKey` when unset. */
+    bridgeBearer?: string;
+}
+declare function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClient;
+
+/**
+ * `runBenchmarks` — the bench unifier.
+ *
+ * `gate.ts` runs ONE adapter as a diverse-vs-blind research gate through the recursive atom.
+ * This is the other half it references: run a SUBSET of the registry's benchmarks over a MATRIX of
+ * agent cells (harness × model × persona), each scored by the benchmark's OWN deterministic judge,
+ * and return a ranked leaderboard. It is the "which harness/model/persona combination wins on which
+ * benchmark" question, answered over an arbitrary subset in one call.
+ *
+ * It owns no new mechanism. Each cell is one `openSandboxRun` shot (the same per-run primitive the
+ * SWE worker uses) driven by `resolveBenchClient` (off-box router completion OR in-box Sandbox; the
+ * harness rides `sandboxOverrides.backend.type`). The deliverable is the adapter's OWN parser
+ * (`adapter.output`), defaulting to the final answer text — so `runBenchmarks` needs no
+ * per-benchmark branching. Concurrency is the shared `runPool`. The number comes from
+ * `adapter.judge`, never a self-authored judge.
+ *
+ * Subset = the `benchmarks` and `cells` arrays plus `n`/`ids`/`split`. A benchmark whose
+ * `preflight()` fails (missing Docker/venv/dataset) is recorded as unavailable and skipped — the
+ * sweep never aborts because one bench's harness is absent.
+ *
+ *   const report = await runBenchmarks({
+ *     benchmarks: ['humaneval', 'swe-bench'],
+ *     cells: [
+ *       { label: 'opencode/glm-4.6', model: 'glm-4.6', harness: 'opencode' },
+ *       { label: 'codex/gpt-5',      model: 'gpt-5',    harness: 'codex'    },
+ *     ],
+ *     routerBaseUrl, routerKey, n: 20,
+ *   })
+ */
+
+/** One agent under test: a profile (prompt/tools/persona) plus the harness + model that run it. */
+interface BenchCell {
+    /** Leaderboard row label, e.g. `opencode/glm-4.6` or `tax-agent@v1`. */
+    readonly label: string;
+    /** Model id for this cell. */
+    readonly model: string;
+    /** Coding harness for the in-box path (`opencode`/`codex`/`claude-code`/`kimi-code`). Defaults to
+     *  `profile.metadata.backendType`, then `opencode`. Ignored on the `router` transport. */
+    readonly harness?: string;
+    /** Transport: `router` (off-box completion, default), `sandbox`, or a BackendType for in-box. */
+    readonly backend?: string;
+    /** Web-search provider for the `router` transport (turns the leaf into a `router-tools` loop). */
+    readonly searchProvider?: string;
+    /** The agent under test. Defaults to a minimal `{ name, metadata.backendType }` profile. */
+    readonly profile?: AgentProfile;
+}
+/** Runs one (adapter, task, cell) shot and returns the deliverable text. The default uses
+ *  `openSandboxRun`; tests inject a deterministic stub so the matrix runs offline. */
+type BenchShot = (input: {
+    readonly adapter: BenchmarkAdapter;
+    readonly task: BenchTask;
+    readonly cell: BenchCell;
+    /** Prompt to hand to the worker. Defaults to `task.prompt`; looped runs pass revised prompts. */
+    readonly prompt?: string;
+    /** 1-based attempt index for looped runs. */
+    readonly attempt?: number;
+    readonly routerBaseUrl: string;
+    readonly routerKey: string;
+    readonly bridgeUrl?: string;
+    readonly bridgeBearer?: string;
+    readonly sandboxBaseUrl?: string;
+    readonly timeoutMs?: number;
+    readonly resolveClient?: typeof resolveBenchClient;
+}) => Promise<{
+    artifact: string;
+    ok: boolean;
+    detail?: string;
+}>;
+interface RunBenchmarksOptions {
+    /** Registry keys (`resolveAdapter`) — the benchmark subset to run. */
+    readonly benchmarks: readonly string[];
+    /** The agent cells to rank. */
+    readonly cells: readonly BenchCell[];
+    readonly routerBaseUrl: string;
+    readonly routerKey: string;
+    readonly bridgeUrl?: string;
+    readonly bridgeBearer?: string;
+    readonly sandboxBaseUrl?: string;
+    /** Tasks per benchmark (the n). */
+    readonly n?: number;
+    readonly ids?: string[];
+    readonly split?: string;
+    /** Replicates per (benchmark × cell × task). Default 1. */
+    readonly reps?: number;
+    /** Bounded concurrency across all shots. Default 4. */
+    readonly concurrency?: number;
+    /** Per-shot wall-clock (ms). */
+    readonly timeoutMs?: number;
+    /** Test seam: resolve the runtime transport. Defaults to `resolveBenchClient`. */
+    readonly resolveClient?: typeof resolveBenchClient;
+    /** Max attempts per (benchmark × cell × task). Default 1. Attempts after the first receive
+     *  non-answer checker feedback and the previous artifacts; the loop stops early on pass. */
+    readonly loopAttempts?: number;
+    /** Self-verify each benchmark's judge against its gold artifact on the first task before spending
+     *  model tokens; a benchmark whose judge rejects its own gold is recorded unavailable. Default true. */
+    readonly verifyJudge?: boolean;
+    /** Test seam: a deterministic shot runner. Defaults to the `openSandboxRun` leaf. */
+    readonly runShot?: BenchShot;
+    /** Test seam: resolve a benchmark key to an adapter. Defaults to the registry `resolveAdapter`. */
+    readonly resolveAdapter?: (key: string) => BenchmarkAdapter;
+    readonly onResult?: (r: BenchCellTaskResult) => void;
+}
+interface BenchCellTaskResult {
+    readonly benchmark: string;
+    readonly cell: string;
+    readonly taskId: string;
+    readonly rep: number;
+    readonly resolved: boolean;
+    readonly score: number;
+    /** false = the shot threw or produced no artifact (infra/empty), excluded from the resolve
+     *  denominator so a harness outage can't masquerade as a 0% capability result. */
+    readonly ok: boolean;
+    readonly detail?: string;
+    readonly wallMs: number;
+}
+interface BenchLeaderboardRow {
+    readonly benchmark: string;
+    readonly cell: string;
+    readonly n: number;
+    readonly resolved: number;
+    readonly errored: number;
+    /** resolved / (n - errored). */
+    readonly resolveRate: number;
+    /** Mean graded score over non-errored shots (partial credit where the judge supports it). */
+    readonly meanScore: number;
+}
+interface RunBenchmarksReport {
+    /** One row per (benchmark × cell), sorted by benchmark then descending resolveRate. */
+    readonly rows: readonly BenchLeaderboardRow[];
+    readonly perTask: readonly BenchCellTaskResult[];
+    readonly benchmarks: readonly string[];
+    readonly cells: readonly string[];
+    /** Benchmarks skipped because `preflight`/judge-self-check failed, with the reason. */
+    readonly unavailable: ReadonlyArray<{
+        readonly benchmark: string;
+        readonly reason: string;
+    }>;
+}
+declare function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenchmarksReport>;
+/** Render the leaderboard as a fixed-width table for a CLI/log. */
+declare function printBenchmarksReport(report: RunBenchmarksReport): string;
+
+interface StagePreparedPierCandidateOptions {
+    readonly prepared: PreparedAgentCandidateExecution;
+    /** Evaluator-owned directory persisted with the Pier trial. */
+    readonly directory: string;
+    /** Exact Pier package version pinned by the experiment contract. */
+    readonly pierVersion: string;
+}
+interface StagedPierCandidateExecution {
+    readonly executionId: string;
+    readonly directory: string;
+    readonly taskDirectory: string;
+    readonly candidateDirectory?: string;
+    readonly profileDirectory: string;
+    readonly planPath: string;
+    readonly receiptPath: string;
+    readonly agentArgs: readonly string[];
+    /** Executor-only model and trace bindings; never present on the prepared object or disk. */
+    readonly evaluatorEnv: Readonly<Record<string, string>>;
+    /** One prepared execution is exactly one Pier trial attempt. */
+    readonly attemptArgs: readonly ['--n-attempts', '1', '--max-retries', '0'];
+}
+interface PierCandidateTerminationAcknowledgement {
+    /** The Pier process has exited and has been reaped. */
+    readonly processExited: true;
+    /** Every task container created for this one trial has been removed. */
+    readonly containersRemoved: true;
+}
+interface PierCandidateTrialHandle {
+    /** Non-secret durable identity shared with a fresh evaluator process. */
+    readonly identity: PierCandidateTrialIdentity;
+    /** Resolves only after the Pier process exits and its task container is gone. */
+    readonly result: Promise<PierCandidateTrialResult>;
+    /** Idempotently kill/reap Pier and remove its task container, then acknowledge their death. */
+    readonly terminateAndWait: () => Promise<PierCandidateTerminationAcknowledgement>;
+}
+type PierCandidateTrialIdentity = Readonly<AgentCandidateExecutorStopRequest>;
+/**
+ * Evaluator-owned lifecycle whose stop path works without the process-local
+ * handle returned by `start`.
+ */
+interface PierCandidateTrialController {
+    start(staged: StagedPierCandidateExecution, context: {
+        readonly request: AgentCandidateExecutorRequest;
+        readonly traceStore: TraceStore;
+        readonly signal: AbortSignal;
+        readonly deadlineAtMs: number;
+    }): PierCandidateTrialHandle;
+    terminateAndWait(identity: PierCandidateTrialIdentity): Promise<PierCandidateTerminationAcknowledgement>;
+    /** Read immutable official bytes after termination; undefined proves no result was emitted. */
+    captureResult(identity: PierCandidateTrialIdentity): Promise<PierCandidateTrialResult | undefined>;
+}
+/** Evaluator-owned bytes captured from one completed official Pier trial. */
+interface PierCandidateTrialResult {
+    /** Parsed value of the exact `result.json` bytes. */
+    readonly value: unknown;
+    /** Exact official `result.json` bytes used as grader evidence. */
+    readonly resultBytes: Uint8Array;
+    /** Exact `/logs/artifacts/model.patch` bytes emitted before official tests. */
+    readonly taskPatch: Uint8Array;
+}
+interface PierCandidateOfficialResult {
+    readonly value: unknown;
+    readonly bytes: Uint8Array;
+}
+type RuntimeGraderInput = Parameters<AgentCandidateBenchmarkGraderPort['run']>[0];
+type RuntimeGraderResult = Awaited<ReturnType<AgentCandidateBenchmarkGraderPort['run']>>;
+/** Executes the exact admitted grader bytes against the official Pier result. */
+interface PierCandidateGraderPort {
+    readonly name: string;
+    readonly version: string;
+    readonly artifact: AgentCandidateBenchmarkGraderPort['artifact'];
+    run(input: RuntimeGraderInput & {
+        readonly officialResult: PierCandidateOfficialResult;
+    }): Promise<RuntimeGraderResult>;
+}
+interface ExecutePreparedPierCandidateOptions extends StagePreparedPierCandidateOptions {
+    readonly traceStore: TraceStore;
+    /** Durable one-shot store shared by every process capable of running this benchmark. */
+    readonly claimStore: AgentCandidateExecutionClaimStore;
+    readonly outputArtifacts: AgentCandidateOutputArtifactPort;
+    readonly grader: PierCandidateGraderPort;
+    /**
+     * Starts exactly one Pier trial synchronously and persists its non-secret
+     * process/container identity before returning.
+     */
+    readonly controller: PierCandidateTrialController;
+}
+/** Recovery-only runtime executor for an expired attempt owned by another process. */
+declare function createPierCandidateRecoveryExecutor(controller: PierCandidateTrialController): AgentCandidateExecutorPort;
+/** Execute and finalize through the runtime's only gradable candidate path. */
+declare function executePreparedPierCandidate(options: ExecutePreparedPierCandidateOptions): Promise<AgentCandidateRunFinalization>;
+
+interface PierCandidateProcessSpec {
+    /** Returns launch data only. The controller, not this callback, starts the process. */
+    readonly command: string;
+    readonly args: readonly string[];
+    readonly cwd: string;
+    /** Exact Pier child environment. The controller adds its Docker connection variables. */
+    readonly env: Readonly<Record<string, string | undefined>>;
+    readonly jobsDirectory: string;
+    /** Must be unique: the controller atomically reserves this Pier job directory. */
+    readonly jobName: string;
+    readonly dockerCommand?: string;
+    /** Called only by the originating process after the supervisor reports clean exit. */
+    readonly readResult: () => Promise<PierCandidateTrialResult>;
+}
+interface PierDockerConnection {
+    /** Stable, non-secret name that every recovery worker maps to the same Docker endpoint. */
+    readonly id: string;
+    /** Exact variables needed by both Pier and Docker cleanup; values are never persisted. */
+    readonly env: Readonly<Record<string, string | undefined>>;
+}
+interface FilePierCandidateTrialControllerOptions {
+    /** Shared evaluator-owned control root, available to crash-recovery workers. */
+    readonly directory: string;
+    readonly launch?: (staged: StagedPierCandidateExecution, context: {
+        readonly request: AgentCandidateExecutorRequest;
+        readonly traceStore: TraceStore;
+        readonly signal: AbortSignal;
+        readonly deadlineAtMs: number;
+    }) => PierCandidateProcessSpec;
+    /** Restart-safe reader for immutable official bytes under the persisted Pier job identity. */
+    readonly readResult?: (input: {
+        readonly jobsDirectory: string;
+        readonly jobName: string;
+    }) => Promise<PierCandidateTrialResult | undefined>;
+    /** Omit only for the default local Docker socket with no environment variables. */
+    readonly dockerConnection?: PierDockerConnection;
+    readonly supervisorPath?: string;
+    readonly pollIntervalMs?: number;
+}
+declare class FilePierCandidateTrialController implements PierCandidateTrialController {
+    private readonly directory;
+    private readonly launch?;
+    private readonly readResult?;
+    private readonly dockerConnection;
+    private readonly supervisorPath;
+    private readonly pollIntervalMs;
+    constructor(options: FilePierCandidateTrialControllerOptions);
+    start(staged: StagedPierCandidateExecution, context: {
+        readonly request: AgentCandidateExecutorRequest;
+        readonly traceStore: TraceStore;
+        readonly signal: AbortSignal;
+        readonly deadlineAtMs: number;
+    }): PierCandidateTrialHandle;
+    terminateAndWait(requested: PierCandidateTrialIdentity): Promise<PierCandidateTerminationAcknowledgement>;
+    captureResult(requested: PierCandidateTrialIdentity): Promise<PierCandidateTrialResult | undefined>;
+    private controlDirectory;
+    private assertDockerConnection;
+    private validateSpec;
+    private waitForResult;
+    private forceRecovery;
+}
+
+declare function createPierResultGrader(descriptor: Pick<PierCandidateGraderPort, 'name' | 'version' | 'artifact'>): PierCandidateGraderPort;
+
+export { type BenchCell, type BenchCellTaskResult, type BenchLeaderboardRow, type BenchShot, BenchTask, BenchmarkAdapter, type ExecutePreparedPierCandidateOptions, FilePierCandidateTrialController, type FilePierCandidateTrialControllerOptions, type PierCandidateGraderPort, type PierCandidateOfficialResult, type PierCandidateProcessSpec, type PierCandidateTerminationAcknowledgement, type PierCandidateTrialController, type PierCandidateTrialHandle, type PierCandidateTrialIdentity, type PierCandidateTrialResult, type PierDockerConnection, type RunBenchmarksOptions, type RunBenchmarksReport, type StagedPierCandidateExecution, createPierCandidateRecoveryExecutor, createPierResultGrader, executePreparedPierCandidate, printBenchmarksReport, runBenchmarks };