@tangle-network/agent-eval 0.23.1 → 0.25.0

package/dist/wire/index.d.ts

@@ -1,8 +1,14 @@
+import { n as FeedbackTrajectoryStore } from '../feedback-trajectory-DfFdrraJ.js';
+import { T as TraceStore } from '../store-Db2Bv8Cf.js';
 import { z } from 'zod';
 import { OpenAPIObject } from 'openapi3-ts/oas31';
 import * as hono_types from 'hono/types';
 import { ServerType } from '@hono/node-server';
 import { Hono } from 'hono';
+import '../control-runtime-BuJHoLg0.js';
+import '../emitter-DP_cSSiw.js';
+import '../dataset-CiK_3LDr.js';
+import '../errors-BZ9sTdz7.js';
 
 declare const RubricDimensionSchema: z.ZodObject<{
     id: z.ZodString;
@@ -105,6 +111,287 @@ declare const HealthResponseSchema: z.ZodObject<{
     status: z.ZodLiteral<"ok">;
     uptimeSec: z.ZodNumber;
 }, z.core.$strip>;
+/**
+ * Minimal `TraceEvent` shape that the production runtime emits.
+ * Matches `trace/schema.ts` `TraceEvent` but is duplicated here as a
+ * wire schema so non-TypeScript clients can validate without depending
+ * on internal types.
+ */
+declare const TraceEventSchema: z.ZodObject<{
+    eventId: z.ZodString;
+    runId: z.ZodString;
+    spanId: z.ZodOptional<z.ZodString>;
+    kind: z.ZodEnum<{
+        policy_violation: "policy_violation";
+        custom: "custom";
+        error: "error";
+        log: "log";
+        budget_decrement: "budget_decrement";
+        budget_breach: "budget_breach";
+        state_mutation: "state_mutation";
+        redaction_applied: "redaction_applied";
+    }>;
+    timestamp: z.ZodNumber;
+    payload: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+}, z.core.$strip>;
+declare const TracesIngestRequestSchema: z.ZodObject<{
+    events: z.ZodArray<z.ZodObject<{
+        eventId: z.ZodString;
+        runId: z.ZodString;
+        spanId: z.ZodOptional<z.ZodString>;
+        kind: z.ZodEnum<{
+            policy_violation: "policy_violation";
+            custom: "custom";
+            error: "error";
+            log: "log";
+            budget_decrement: "budget_decrement";
+            budget_breach: "budget_breach";
+            state_mutation: "state_mutation";
+            redaction_applied: "redaction_applied";
+        }>;
+        timestamp: z.ZodNumber;
+        payload: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const TracesIngestResponseSchema: z.ZodObject<{
+    accepted: z.ZodNumber;
+    rejected: z.ZodNumber;
+    errors: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        eventId: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+declare const FeedbackLabelSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    source: z.ZodEnum<{
+        judge: "judge";
+        system: "system";
+        user: "user";
+        policy: "policy";
+        environment: "environment";
+        metric: "metric";
+    }>;
+    kind: z.ZodEnum<{
+        approve: "approve";
+        reject: "reject";
+        select: "select";
+        edit: "edit";
+        rank: "rank";
+        rate: "rate";
+        comment: "comment";
+        metric_outcome: "metric_outcome";
+        policy_block: "policy_block";
+        revision_request: "revision_request";
+    }>;
+    value: z.ZodUnknown;
+    reason: z.ZodOptional<z.ZodString>;
+    severity: z.ZodOptional<z.ZodEnum<{
+        error: "error";
+        info: "info";
+        warning: "warning";
+        critical: "critical";
+    }>>;
+    createdAt: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+declare const FeedbackAttemptSchema: z.ZodObject<{
+    id: z.ZodString;
+    stepIndex: z.ZodNumber;
+    artifactType: z.ZodEnum<{
+        action: "action";
+        decision: "decision";
+        text: "text";
+        code: "code";
+        plan: "plan";
+        research: "research";
+        ui: "ui";
+        data: "data";
+        other: "other";
+    }>;
+    artifact: z.ZodUnknown;
+    options: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
+    proposedAction: z.ZodOptional<z.ZodObject<{
+        type: z.ZodString;
+        risk: z.ZodOptional<z.ZodEnum<{
+            medium: "medium";
+            low: "low";
+            high: "high";
+        }>>;
+        costUsd: z.ZodOptional<z.ZodNumber>;
+        externalSideEffect: z.ZodOptional<z.ZodBoolean>;
+        requiresApproval: z.ZodOptional<z.ZodBoolean>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>;
+    feedback: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodOptional<z.ZodString>;
+        source: z.ZodEnum<{
+            judge: "judge";
+            system: "system";
+            user: "user";
+            policy: "policy";
+            environment: "environment";
+            metric: "metric";
+        }>;
+        kind: z.ZodEnum<{
+            approve: "approve";
+            reject: "reject";
+            select: "select";
+            edit: "edit";
+            rank: "rank";
+            rate: "rate";
+            comment: "comment";
+            metric_outcome: "metric_outcome";
+            policy_block: "policy_block";
+            revision_request: "revision_request";
+        }>;
+        value: z.ZodUnknown;
+        reason: z.ZodOptional<z.ZodString>;
+        severity: z.ZodOptional<z.ZodEnum<{
+            error: "error";
+            info: "info";
+            warning: "warning";
+            critical: "critical";
+        }>>;
+        createdAt: z.ZodString;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>>;
+    createdAt: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+declare const FeedbackTrajectorySchema: z.ZodObject<{
+    id: z.ZodString;
+    projectId: z.ZodOptional<z.ZodString>;
+    scenarioId: z.ZodOptional<z.ZodString>;
+    task: z.ZodObject<{
+        intent: z.ZodString;
+        context: z.ZodOptional<z.ZodUnknown>;
+    }, z.core.$strip>;
+    attempts: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        stepIndex: z.ZodNumber;
+        artifactType: z.ZodEnum<{
+            action: "action";
+            decision: "decision";
+            text: "text";
+            code: "code";
+            plan: "plan";
+            research: "research";
+            ui: "ui";
+            data: "data";
+            other: "other";
+        }>;
+        artifact: z.ZodUnknown;
+        options: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
+        proposedAction: z.ZodOptional<z.ZodObject<{
+            type: z.ZodString;
+            risk: z.ZodOptional<z.ZodEnum<{
+                medium: "medium";
+                low: "low";
+                high: "high";
+            }>>;
+            costUsd: z.ZodOptional<z.ZodNumber>;
+            externalSideEffect: z.ZodOptional<z.ZodBoolean>;
+            requiresApproval: z.ZodOptional<z.ZodBoolean>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>;
+        feedback: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodOptional<z.ZodString>;
+            source: z.ZodEnum<{
+                judge: "judge";
+                system: "system";
+                user: "user";
+                policy: "policy";
+                environment: "environment";
+                metric: "metric";
+            }>;
+            kind: z.ZodEnum<{
+                approve: "approve";
+                reject: "reject";
+                select: "select";
+                edit: "edit";
+                rank: "rank";
+                rate: "rate";
+                comment: "comment";
+                metric_outcome: "metric_outcome";
+                policy_block: "policy_block";
+                revision_request: "revision_request";
+            }>;
+            value: z.ZodUnknown;
+            reason: z.ZodOptional<z.ZodString>;
+            severity: z.ZodOptional<z.ZodEnum<{
+                error: "error";
+                info: "info";
+                warning: "warning";
+                critical: "critical";
+            }>>;
+            createdAt: z.ZodString;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>>;
+        createdAt: z.ZodString;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>>;
+    labels: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        id: z.ZodOptional<z.ZodString>;
+        source: z.ZodEnum<{
+            judge: "judge";
+            system: "system";
+            user: "user";
+            policy: "policy";
+            environment: "environment";
+            metric: "metric";
+        }>;
+        kind: z.ZodEnum<{
+            approve: "approve";
+            reject: "reject";
+            select: "select";
+            edit: "edit";
+            rank: "rank";
+            rate: "rate";
+            comment: "comment";
+            metric_outcome: "metric_outcome";
+            policy_block: "policy_block";
+            revision_request: "revision_request";
+        }>;
+        value: z.ZodUnknown;
+        reason: z.ZodOptional<z.ZodString>;
+        severity: z.ZodOptional<z.ZodEnum<{
+            error: "error";
+            info: "info";
+            warning: "warning";
+            critical: "critical";
+        }>>;
+        createdAt: z.ZodString;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>>;
+    outcome: z.ZodOptional<z.ZodObject<{
+        success: z.ZodOptional<z.ZodBoolean>;
+        score: z.ZodOptional<z.ZodNumber>;
+        metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+        costUsd: z.ZodOptional<z.ZodNumber>;
+        detail: z.ZodOptional<z.ZodString>;
+        observedAt: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>;
+    split: z.ZodOptional<z.ZodEnum<{
+        train: "train";
+        dev: "dev";
+        test: "test";
+        holdout: "holdout";
+    }>>;
+    tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    createdAt: z.ZodString;
+    updatedAt: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+declare const FeedbackIngestResponseSchema: z.ZodObject<{
+    id: z.ZodString;
+    persisted: z.ZodBoolean;
+}, z.core.$strip>;
+type TraceEvent = z.infer<typeof TraceEventSchema>;
+type TracesIngestRequest = z.infer<typeof TracesIngestRequestSchema>;
+type TracesIngestResponse = z.infer<typeof TracesIngestResponseSchema>;
+type FeedbackTrajectory = z.infer<typeof FeedbackTrajectorySchema>;
+type FeedbackIngestResponse = z.infer<typeof FeedbackIngestResponseSchema>;
 declare const ErrorResponseSchema: z.ZodObject<{
     error: z.ZodObject<{
         code: z.ZodString;
@@ -132,6 +419,18 @@ declare const WIRE_VERSION = "1.0.0";
  */
 declare function hashRubric(rubric: Rubric): string;
 
+/**
+ * Pure handler functions — the "business logic" behind every wire-protocol
+ * method. The HTTP server (`server.ts`) and the stdio RPC (`rpc.ts`) both
+ * call these. Tests call these directly without spinning a server.
+ *
+ * Each handler:
+ *   - Takes a parsed request (already Zod-validated by the transport).
+ *   - Returns a result that matches the response schema.
+ *   - Throws `WireError` for caller-fixable errors (404, 400, 422).
+ *   - Lets unexpected errors bubble — the transport maps them to 500.
+ */
+
 /** Caller-fixable error. The transport renders this to 4xx + ErrorResponse. */
 declare class WireError extends Error {
     readonly code: string;
@@ -142,6 +441,57 @@ declare class WireError extends Error {
 declare function handleJudge(req: JudgeRequest): Promise<JudgeResult>;
 declare function handleListRubrics(): ListRubricsResponse;
 declare function handleVersion(): VersionResponse;
+/**
+ * Pluggable stores the wire layer routes ingestion writes into. Both
+ * are optional — when omitted, the corresponding endpoint returns 503.
+ *
+ * Production deployments wire a `FileSystemTraceStore` and
+ * `FileSystemFeedbackTrajectoryStore` here. Tests substitute in-memory
+ * stores.
+ */
+interface IngestionStores {
+    traceStore?: TraceStore;
+    feedbackStore?: FeedbackTrajectoryStore;
+}
+/**
+ * `POST /v1/traces/ingest` — accept a batch of `TraceEvent`s from the
+ * production runtime. Best-effort: each event is appended independently;
+ * one bad event does not poison the batch.
+ *
+ * Idempotency: the underlying store is append-only; consumers retrying
+ * the same payload will get duplicate events. Consumers should
+ * de-duplicate by `eventId` downstream — production traces frequently
+ * land via at-least-once buses (Kafka, SQS) where dedup is unavoidable.
+ */
+declare function handleTracesIngest(req: TracesIngestRequest, stores: IngestionStores): Promise<TracesIngestResponse>;
+/**
+ * `POST /v1/feedback` — accept a single `FeedbackTrajectory` from the
+ * production runtime. Idempotent on `id`: re-posting the same trajectory
+ * replaces the prior record.
+ */
+declare function handleFeedbackIngest(req: FeedbackTrajectory, stores: IngestionStores): Promise<FeedbackIngestResponse>;
+
+declare function buildOpenApi(packageVersion: string): OpenAPIObject;
+
+interface RpcRequest {
+    method: 'judge' | 'listRubrics' | 'version';
+    params?: unknown;
+}
+interface RpcSuccess {
+    result: unknown;
+}
+interface RpcError {
+    error: {
+        code: string;
+        message: string;
+        details?: unknown;
+    };
+}
+declare function dispatchRpc(req: RpcRequest): Promise<RpcSuccess | RpcError>;
+/** Read one JSON request from stdin, write one JSON response to stdout. */
+declare function runRpcOnce(method?: string): Promise<number>;
+/** Read JSONL requests from stdin, write JSONL responses to stdout. */
+declare function runRpcBatch(method?: string): Promise<number>;
 
 /**
  * Built-in rubrics shipped with agent-eval.
@@ -177,10 +527,24 @@ declare function listBuiltinRubrics(): {
     rubricVersion: string;
 }[];
 
-declare function buildOpenApi(packageVersion: string): OpenAPIObject;
-
-declare function createApp(): Hono<hono_types.BlankEnv, hono_types.BlankSchema, "/">;
-interface ServeOptions {
+interface CreateAppOptions {
+    /** Stores wired to the ingestion endpoints. */
+    stores?: IngestionStores;
+    /**
+     * Bearer-token auth. When provided, every endpoint EXCEPT `/healthz`
+     * and `/v1/version` requires `Authorization: Bearer <token>`. The
+     * token may be a static string OR a function for time-bounded /
+     * rotating tokens.
+     *
+     * Recommended for any server that accepts ingestion writes from the
+     * public internet. Read-only deployments may omit it.
+     */
+    auth?: {
+        bearer: string | ((token: string) => boolean | Promise<boolean>);
+    };
+}
+declare function createApp(opts?: CreateAppOptions): Hono<hono_types.BlankEnv, hono_types.BlankSchema, "/">;
+interface ServeOptions extends CreateAppOptions {
     /** Default 5005. */
     port?: number;
     /** Default '127.0.0.1'. Set to '0.0.0.0' to listen on all interfaces. */
@@ -188,24 +552,4 @@ interface ServeOptions {
 }
 declare function startServer(opts?: ServeOptions): ServerType;
 
-interface RpcRequest {
-    method: 'judge' | 'listRubrics' | 'version';
-    params?: unknown;
-}
-interface RpcSuccess {
-    result: unknown;
-}
-interface RpcError {
-    error: {
-        code: string;
-        message: string;
-        details?: unknown;
-    };
-}
-declare function dispatchRpc(req: RpcRequest): Promise<RpcSuccess | RpcError>;
-/** Read one JSON request from stdin, write one JSON response to stdout. */
-declare function runRpcOnce(method?: string): Promise<number>;
-/** Read JSONL requests from stdin, write JSONL responses to stdout. */
-declare function runRpcBatch(method?: string): Promise<number>;
-
-export { BUILTIN_RUBRICS, type ErrorResponse, ErrorResponseSchema, type FailureMode, FailureModeSchema, HealthResponseSchema, type JudgeRequest, JudgeRequestSchema, type JudgeResult, JudgeResultSchema, type ListRubricsResponse, ListRubricsResponseSchema, type Rubric, type RubricDimension, RubricDimensionSchema, type RubricInfo, RubricInfoSchema, RubricSchema, type ServeOptions, type VersionResponse, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleJudge, handleListRubrics, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer };
+export { BUILTIN_RUBRICS, type ErrorResponse, ErrorResponseSchema, type FailureMode, FailureModeSchema, FeedbackAttemptSchema, type FeedbackIngestResponse, FeedbackIngestResponseSchema, FeedbackLabelSchema, type FeedbackTrajectory, FeedbackTrajectorySchema, HealthResponseSchema, type IngestionStores, type JudgeRequest, JudgeRequestSchema, type JudgeResult, JudgeResultSchema, type ListRubricsResponse, ListRubricsResponseSchema, type Rubric, type RubricDimension, RubricDimensionSchema, type RubricInfo, RubricInfoSchema, RubricSchema, type ServeOptions, type TraceEvent, TraceEventSchema, type TracesIngestRequest, TracesIngestRequestSchema, type TracesIngestResponse, TracesIngestResponseSchema, type VersionResponse, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer };

package/dist/wire/index.js

@@ -2,6 +2,10 @@ import {
   BUILTIN_RUBRICS,
   ErrorResponseSchema,
   FailureModeSchema,
+  FeedbackAttemptSchema,
+  FeedbackIngestResponseSchema,
+  FeedbackLabelSchema,
+  FeedbackTrajectorySchema,
   HealthResponseSchema,
   JudgeRequestSchema,
   JudgeResultSchema,
@@ -9,6 +13,9 @@ import {
   RubricDimensionSchema,
   RubricInfoSchema,
   RubricSchema,
+  TraceEventSchema,
+  TracesIngestRequestSchema,
+  TracesIngestResponseSchema,
   VersionResponseSchema,
   WIRE_VERSION,
   WireError,
@@ -16,22 +23,29 @@ import {
   createApp,
   dispatchRpc,
   getBuiltinRubric,
+  handleFeedbackIngest,
   handleJudge,
   handleListRubrics,
+  handleTracesIngest,
   handleVersion,
   hashRubric,
   listBuiltinRubrics,
   runRpcBatch,
   runRpcOnce,
   startServer
-} from "../chunk-6KQG5HAH.js";
-import "../chunk-KAO3Q65R.js";
-import "../chunk-SQQLHODJ.js";
+} from "../chunk-5LBB5B3Z.js";
+import "../chunk-4S4BM3QQ.js";
+import "../chunk-PC4UYEBM.js";
+import "../chunk-NG236HPC.js";
 import "../chunk-PZ5AY32C.js";
 export {
   BUILTIN_RUBRICS,
   ErrorResponseSchema,
   FailureModeSchema,
+  FeedbackAttemptSchema,
+  FeedbackIngestResponseSchema,
+  FeedbackLabelSchema,
+  FeedbackTrajectorySchema,
   HealthResponseSchema,
   JudgeRequestSchema,
   JudgeResultSchema,
@@ -39,6 +53,9 @@ export {
   RubricDimensionSchema,
   RubricInfoSchema,
   RubricSchema,
+  TraceEventSchema,
+  TracesIngestRequestSchema,
+  TracesIngestResponseSchema,
   VersionResponseSchema,
   WIRE_VERSION,
   WireError,
@@ -46,8 +63,10 @@ export {
   createApp,
   dispatchRpc,
   getBuiltinRubric,
+  handleFeedbackIngest,
   handleJudge,
   handleListRubrics,
+  handleTracesIngest,
   handleVersion,
   hashRubric,
   listBuiltinRubrics,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.23.1",
-  "description": "Trace-first evaluation infrastructure for agent systems: traces, harnesses, verifier pipelines, judges, datasets, gates, optimization, and reporting.",
+  "version": "0.25.0",
+  "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
     "type": "git",
@@ -64,6 +64,36 @@
       "import": "./dist/benchmarks/index.js",
       "default": "./dist/benchmarks/index.js"
     },
+    "./pipelines": {
+      "types": "./dist/pipelines/index.d.ts",
+      "import": "./dist/pipelines/index.js",
+      "default": "./dist/pipelines/index.js"
+    },
+    "./meta-eval": {
+      "types": "./dist/meta-eval/index.d.ts",
+      "import": "./dist/meta-eval/index.js",
+      "default": "./dist/meta-eval/index.js"
+    },
+    "./prm": {
+      "types": "./dist/prm/index.d.ts",
+      "import": "./dist/prm/index.js",
+      "default": "./dist/prm/index.js"
+    },
+    "./builder-eval": {
+      "types": "./dist/builder-eval/index.d.ts",
+      "import": "./dist/builder-eval/index.js",
+      "default": "./dist/builder-eval/index.js"
+    },
+    "./governance": {
+      "types": "./dist/governance/index.d.ts",
+      "import": "./dist/governance/index.js",
+      "default": "./dist/governance/index.js"
+    },
+    "./knowledge": {
+      "types": "./dist/knowledge/index.d.ts",
+      "import": "./dist/knowledge/index.js",
+      "default": "./dist/knowledge/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -79,15 +109,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "prepare": "pnpm build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
-  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -97,20 +118,25 @@
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@biomejs/biome": "^2.4.15",
     "@types/node": "^25.6.0",
     "openapi3-ts": "^4.5.0",
     "tsup": "^8.0.0",
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
-  "pnpm": {
-    "overrides": {
-      "postcss@<8.5.10": "^8.5.10"
-    }
-  },
   "engines": {
     "node": ">=20"
   },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
-}
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src",
+    "format": "biome format --write src",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
+  }
+}

package/dist/chunk-42I2QC2L.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/index.ts","../src/benchmarks/types.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/routing/dataset.ts"],"sourcesContent":["/**\n * Reference benchmark wrappers — entry point.\n *\n * Core surface (exported here):\n *   - The `BenchmarkAdapter` contract.\n *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.\n *   - `routing` — synthetic 16-task router benchmark. The only novel\n *     benchmark we built; ships in the package.\n *\n * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):\n *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset\n *                       not bundled).\n *   - `swebench-lite` — 30-instance SWE-Bench subset via an external\n *                       grader command.\n *\n * The example wrappers are reference implementations of `BenchmarkAdapter`.\n * Read them, copy them, adapt them. They're intentionally not in the main\n * entry — every team will configure them differently.\n */\n\nexport type {\n  BenchmarkAdapter,\n  BenchmarkDatasetItem,\n  BenchmarkEvaluation,\n} from './types'\nexport { deterministicSplit, BENCHMARK_SPLIT_SEED } from './types'\n\nexport * as routing from './routing/index'\n","/**\n * Shared types for the reference benchmark wrappers under\n * `src/benchmarks/`. Each wrapper exports the three functions in\n * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.\n */\n\nimport type { RunSplitTag } from '../run-record'\n\nexport interface BenchmarkDatasetItem<TPayload = unknown> {\n  /** Stable dataset-local item id (used for split assignment + paper\n   *  references). Unique within a benchmark. */\n  id: string\n  /** Free-form payload. Each benchmark defines its own shape. */\n  payload: TPayload\n}\n\nexport interface BenchmarkEvaluation {\n  /** [0, 1] score for the response on this item. Exact-match\n   *  benchmarks use 0/1; partial-credit benchmarks may return\n   *  fractional values. */\n  score: number\n  /** Optional bag of raw scoring signals — e.g. parsed numeric\n   *  answer, regex match, judge sub-scores. */\n  raw: Record<string, unknown>\n}\n\n/** Common signature implemented by every adapter under `src/benchmarks/*`. */\n// `TPayload` is the per-item payload type; `_TItem` is preserved for\n// downstream type-narrowing extensions (a richer `BenchmarkDatasetItem`\n// subclass that adds e.g. provenance metadata) but is intentionally\n// unused here. `noUnusedLocals` requires the leading underscore.\nexport interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {\n  /** Load the dataset for the given split. May hit the network on\n   *  first call but should be cache-friendly. Adapters that don't\n   *  ship the dataset itself MUST throw a clearly-marked error\n   *  pointing the caller at the loader script. */\n  loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>\n  /** Score a single response. Pure with respect to the inputs. */\n  evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>\n  /** Deterministic split assignment via item id hashing. The\n   *  fraction of items in each split is implementation-defined but\n   *  MUST be stable across processes and platforms. */\n  assignSplit(itemId: string): RunSplitTag\n}\n\n// ── Deterministic split assignment ───────────────────────────────────\n\n/**\n * 32-bit FNV-1a hash. Stable, allocation-free, deterministic across\n * runtimes. We use it to assign items to splits rather than depending\n * on a polyfilled crypto.subtle path.\n */\nfunction fnv1a32(input: string): number {\n  let h = 0x811c9dc5\n  for (let i = 0; i < input.length; i++) {\n    h ^= input.charCodeAt(i) & 0xff\n    h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0\n  }\n  return h >>> 0\n}\n\n/** Split-assignment seed shared across all benchmarks. Bumping this\n *  value reshuffles every split — do NOT do that lightly. */\nexport const BENCHMARK_SPLIT_SEED = 'agent-eval-v1'\n\n/**\n * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a\n * stable 32-bit hash of `${seed}::${id}`. Default proportions:\n *\n *   search: 60%   (optimization-readable)\n *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)\n *   holdout:20%   (paper-grade held-out, gated reads)\n */\nexport function deterministicSplit(\n  itemId: string,\n  seed: string = BENCHMARK_SPLIT_SEED,\n): RunSplitTag {\n  const h = fnv1a32(`${seed}::${itemId}`)\n  const pos = h / 0x100000000\n  if (pos < 0.6) return 'search'\n  if (pos < 0.8) return 'dev'\n  return 'holdout'\n}\n","/**\n * Routing benchmark — synthetic, dependency-free, ships in the\n * package. 16 cross-category items in `dataset.ts`. See\n * `routing/README.md` for the format.\n *\n * `evaluate` does case-insensitive exact match against the canonical\n * route plus declared synonyms. The first valid route token in the\n * response wins; everything else is ignored. Wrong answers also\n * report whether they hit a hard negative — useful when triaging\n * \"always picks the popular route\" failure modes.\n */\n\nimport type {\n  BenchmarkAdapter,\n  BenchmarkDatasetItem,\n  BenchmarkEvaluation,\n} from '../types'\nimport { deterministicSplit } from '../types'\nimport type { RunSplitTag } from '../../run-record'\nimport { ROUTING_DATASET, type RoutingItem } from './dataset'\n\nexport type { RoutingItem }\nexport type RoutingPayload = RoutingItem\nexport type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>\n\nclass RoutingAdapter\n  implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload>\n{\n  async loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]> {\n    return ROUTING_DATASET\n      .map((item) => ({ id: item.id, payload: item }))\n      .filter((it) => assignSplitImpl(it.id) === split)\n  }\n\n  async evaluate(\n    item: RoutingDatasetItem,\n    response: string,\n  ): Promise<BenchmarkEvaluation> {\n    const tokens = extractRouteTokens(response)\n    const correct = new Set<string>([item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()))\n    const hardNeg = new Set<string>(item.payload.hardNegatives.map((s) => s.toLowerCase()))\n    const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null\n    const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null\n    const score = firstMatch ? 1 : 0\n    return {\n      score,\n      raw: {\n        firstToken: tokens[0] ?? null,\n        matchedRoute: firstMatch,\n        hitHardNegative: Boolean(firstHardNeg),\n        hardNegativeRoute: firstHardNeg,\n        category: item.payload.category,\n      },\n    }\n  }\n\n  assignSplit(itemId: string): RunSplitTag {\n    return assignSplitImpl(itemId)\n  }\n}\n\nfunction assignSplitImpl(itemId: string): RunSplitTag {\n  return deterministicSplit(`routing::${itemId}`)\n}\n\n/**\n * Pull route-shaped tokens out of a model response. Routes look like\n * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics\n * are not routes, but `category.action` patterns are robust to most\n * model wrappers (JSON output, prose explanations, code fences).\n */\nexport function extractRouteTokens(response: string): string[] {\n  const matches = response.match(/[a-z][a-z0-9_]*\\.[a-z][a-z0-9_]*/gi)\n  return matches ?? []\n}\n\nconst adapter = new RoutingAdapter()\n\nexport const loadDataset = adapter.loadDataset.bind(adapter)\nexport const evaluate = adapter.evaluate.bind(adapter)\nexport const assignSplit = adapter.assignSplit.bind(adapter)\nexport { RoutingAdapter, ROUTING_DATASET }\n","/**\n * Synthetic routing dataset. 16 tasks across 4 categories. Used as a\n * deterministic, dependency-free benchmark for any router that maps a\n * natural-language request to one of a fixed set of route labels.\n *\n * Format (see `routing/README.md` for prose):\n *\n *   {\n *     id:           stable per-task ID (matches across processes).\n *     category:     one of the four route labels.\n *     prompt:       the user-facing request the router must classify.\n *     route:        the ground-truth route the router should pick.\n *     synonyms:     other strings that count as a correct answer.\n *     hardNegatives:close-but-wrong route labels — used to detect the\n *                   \"always picks the popular route\" failure mode.\n *   }\n *\n * The four categories are intentionally cross-domain (file ops,\n * math, search, conversation) so a router that collapses to one\n * category is easy to spot.\n */\n\nexport interface RoutingItem {\n  id: string\n  category: 'file' | 'math' | 'search' | 'chat'\n  prompt: string\n  /** Canonical correct route label. */\n  route: string\n  /** Alternate route labels that also count as correct. */\n  synonyms: string[]\n  /** Wrong-but-tempting route labels (for analysis, not grading). */\n  hardNegatives: string[]\n}\n\nexport const ROUTING_DATASET: RoutingItem[] = [\n  {\n    id: 'file_001',\n    category: 'file',\n    prompt: 'Save the meeting notes to /tmp/notes-2025-04.md as markdown.',\n    route: 'fs.write',\n    synonyms: ['filesystem.write', 'write_file'],\n    hardNegatives: ['fs.read', 'chat.reply'],\n  },\n  {\n    id: 'file_002',\n    category: 'file',\n    prompt: 'Read the contents of /etc/hosts and summarize the entries.',\n    route: 'fs.read',\n    synonyms: ['filesystem.read', 'read_file'],\n    hardNegatives: ['fs.write', 'search.web'],\n  },\n  {\n    id: 'file_003',\n    category: 'file',\n    prompt: 'List every Python file under src/ recursively.',\n    route: 'fs.list',\n    synonyms: ['filesystem.list', 'list_files'],\n    hardNegatives: ['fs.read', 'search.code'],\n  },\n  {\n    id: 'file_004',\n    category: 'file',\n    prompt: 'Delete the cached build at .turbo/cache.',\n    route: 'fs.delete',\n    synonyms: ['filesystem.delete', 'remove_file'],\n    hardNegatives: ['fs.write', 'fs.list'],\n  },\n  {\n    id: 'math_001',\n    category: 'math',\n    prompt: 'What is the integral of 3x^2 + 2x from 0 to 5?',\n    route: 'math.integral',\n    synonyms: ['calculator.integral', 'math.solve'],\n    hardNegatives: ['math.derivative', 'chat.reply'],\n  },\n  {\n    id: 'math_002',\n    category: 'math',\n    prompt: 'Compute the derivative of sin(x) * cos(x).',\n    route: 'math.derivative',\n    synonyms: ['calculator.derivative', 'math.solve'],\n    hardNegatives: ['math.integral', 'math.algebra'],\n  },\n  {\n    id: 'math_003',\n    category: 'math',\n    prompt: 'Solve 2x + 7 = 19 for x.',\n    route: 'math.algebra',\n    synonyms: ['calculator.algebra', 'math.solve'],\n    hardNegatives: ['math.derivative', 'math.integral'],\n  },\n  {\n    id: 'math_004',\n    category: 'math',\n    prompt: 'What is the prime factorization of 360?',\n    route: 'math.numbertheory',\n    synonyms: ['calculator.factor', 'math.solve'],\n    hardNegatives: ['math.algebra', 'search.web'],\n  },\n  {\n    id: 'search_001',\n    category: 'search',\n    prompt: 'Find recent papers on agent prompt optimization with held-out promotion gates.',\n    route: 'search.web',\n    synonyms: ['web.search', 'search.papers'],\n    hardNegatives: ['search.code', 'chat.reply'],\n  },\n  {\n    id: 'search_002',\n    category: 'search',\n    prompt: 'Search the codebase for every call site of `runProposeReview`.',\n    route: 'search.code',\n    synonyms: ['code.search', 'grep'],\n    hardNegatives: ['search.web', 'fs.read'],\n  },\n  {\n    id: 'search_003',\n    category: 'search',\n    prompt: 'What is the latest release of the Tangle network on GitHub?',\n    route: 'search.web',\n    synonyms: ['web.search', 'github.releases'],\n    hardNegatives: ['search.code', 'chat.reply'],\n  },\n  {\n    id: 'search_004',\n    category: 'search',\n    prompt: 'Find all TODO comments in the agent-eval src tree.',\n    route: 'search.code',\n    synonyms: ['code.search', 'grep'],\n    hardNegatives: ['search.web', 'fs.list'],\n  },\n  {\n    id: 'chat_001',\n    category: 'chat',\n    prompt: 'Hi there, how are you doing today?',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply'],\n    hardNegatives: ['search.web', 'fs.read'],\n  },\n  {\n    id: 'chat_002',\n    category: 'chat',\n    prompt: 'Please explain the difference between an LLM and a foundation model.',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply', 'qa.answer'],\n    hardNegatives: ['search.web', 'math.algebra'],\n  },\n  {\n    id: 'chat_003',\n    category: 'chat',\n    prompt: 'Tell me a short joke about distributed systems.',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply'],\n    hardNegatives: ['search.web', 'fs.read'],\n  },\n  {\n    id: 'chat_004',\n    category: 'chat',\n    prompt: 'Acknowledge my last message with a thumbs up.',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply', 'react'],\n    hardNegatives: ['fs.write', 'search.web'],\n  },\n]\n"],"mappings":";;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACoDA,SAAS,QAAQ,OAAuB;AACtC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,SAAK,MAAM,WAAW,CAAC,IAAI;AAC3B,QAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,SAAU;AAAA,EACxE;AACA,SAAO,MAAM;AACf;AAIO,IAAM,uBAAuB;AAU7B,SAAS,mBACd,QACA,OAAe,sBACF;AACb,QAAM,IAAI,QAAQ,GAAG,IAAI,KAAK,MAAM,EAAE;AACtC,QAAM,MAAM,IAAI;AAChB,MAAI,MAAM,IAAK,QAAO;AACtB,MAAI,MAAM,IAAK,QAAO;AACtB,SAAO;AACT;;;AClFA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACkCO,IAAM,kBAAiC;AAAA,EAC5C;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB,YAAY;AAAA,IAC3C,eAAe,CAAC,WAAW,YAAY;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,mBAAmB,WAAW;AAAA,IACzC,eAAe,CAAC,YAAY,YAAY;AAAA,EAC1C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,mBAAmB,YAAY;AAAA,IAC1C,eAAe,CAAC,WAAW,aAAa;AAAA,EAC1C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,qBAAqB,aAAa;AAAA,IAC7C,eAAe,CAAC,YAAY,SAAS;AAAA,EACvC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,uBAAuB,YAAY;AAAA,IAC9C,eAAe,CAAC,mBAAmB,YAAY;AAAA,EACjD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,yBAAyB,YAAY;AAAA,IAChD,eAAe,CAAC,iBAAiB,cAAc;AAAA,EACjD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,YAAY;AAAA,IAC7C,eAAe,CAAC,mBAAmB,eAAe;AAAA,EACpD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,qBAAqB,YAAY;AAAA,IAC5C,eAAe,CAAC,gBAAgB,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,cAAc,eAAe;AAAA,IACxC,eAAe,CAAC,eAAe,YAAY;AAAA,EAC7C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,eAAe,MAAM;AAAA,IAChC,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,cAAc,iBAAiB;AAAA,IAC1C,eAAe,CAAC,eAAe,YAAY;AAAA,EAC7C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,eAAe,MAAM;AAAA,IAChC,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB;AAAA,IAC/B,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,WAAW;AAAA,IAC5C,eAAe,CAAC,cAAc,cAAc;AAAA,EAC9C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB;AAAA,IAC/B,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,OAAO;AAAA,IACxC,eAAe,CAAC,YAAY,YAAY;AAAA,EAC1C;AACF;;;AD1IA,IAAM,iBAAN,MAEA;AAAA,EACE,MAAM,YAAY,OAAmD;AACnE,WAAO,gBACJ,IAAI,CAAC,UAAU,EAAE,IAAI,KAAK,IAAI,SAAS,KAAK,EAAE,EAC9C,OAAO,CAAC,OAAO,gBAAgB,GAAG,EAAE,MAAM,KAAK;AAAA,EACpD;AAAA,EAEA,MAAM,SACJ,MACA,UAC8B;AAC9B,UAAM,SAAS,mBAAmB,QAAQ;AAC1C,UAAM,UAAU,IAAI,IAAY,CAAC,KAAK,QAAQ,OAAO,GAAG,KAAK,QAAQ,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AAC1G,UAAM,UAAU,IAAI,IAAY,KAAK,QAAQ,cAAc,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACtF,UAAM,aAAa,OAAO,KAAK,CAAC,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;AACvE,UAAM,eAAe,OAAO,KAAK,CAAC,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;AACzE,UAAM,QAAQ,aAAa,IAAI;AAC/B,WAAO;AAAA,MACL;AAAA,MACA,KAAK;AAAA,QACH,YAAY,OAAO,CAAC,KAAK;AAAA,QACzB,cAAc;AAAA,QACd,iBAAiB,QAAQ,YAAY;AAAA,QACrC,mBAAmB;AAAA,QACnB,UAAU,KAAK,QAAQ;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,YAAY,QAA6B;AACvC,WAAO,gBAAgB,MAAM;AAAA,EAC/B;AACF;AAEA,SAAS,gBAAgB,QAA6B;AACpD,SAAO,mBAAmB,YAAY,MAAM,EAAE;AAChD;AAQO,SAAS,mBAAmB,UAA4B;AAC7D,QAAM,UAAU,SAAS,MAAM,oCAAoC;AACnE,SAAO,WAAW,CAAC;AACrB;AAEA,IAAM,UAAU,IAAI,eAAe;AAE5B,IAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;AACpD,IAAM,WAAW,QAAQ,SAAS,KAAK,OAAO;AAC9C,IAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;","names":[]}