@tangle-network/agent-eval 0.12.0 → 0.14.0

package/dist/wire/index.d.ts

@@ -0,0 +1,211 @@
+import { z } from 'zod';
+import { OpenAPIObject } from 'openapi3-ts/oas31';
+import * as hono_types from 'hono/types';
+import { ServerType } from '@hono/node-server';
+import { Hono } from 'hono';
+
+declare const RubricDimensionSchema: z.ZodObject<{
+    id: z.ZodString;
+    description: z.ZodString;
+    weight: z.ZodDefault<z.ZodNumber>;
+    min: z.ZodDefault<z.ZodNumber>;
+    max: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+declare const FailureModeSchema: z.ZodObject<{
+    id: z.ZodString;
+    description: z.ZodString;
+}, z.core.$strip>;
+declare const RubricSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodString;
+    systemPrompt: z.ZodString;
+    dimensions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        description: z.ZodString;
+        weight: z.ZodDefault<z.ZodNumber>;
+        min: z.ZodDefault<z.ZodNumber>;
+        max: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>>;
+    failureModes: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        description: z.ZodString;
+    }, z.core.$strip>>>;
+    wins: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        description: z.ZodString;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+declare const JudgeRequestSchema: z.ZodObject<{
+    rubricName: z.ZodOptional<z.ZodString>;
+    rubric: z.ZodOptional<z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        systemPrompt: z.ZodString;
+        dimensions: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            description: z.ZodString;
+            weight: z.ZodDefault<z.ZodNumber>;
+            min: z.ZodDefault<z.ZodNumber>;
+            max: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strip>>;
+        failureModes: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            description: z.ZodString;
+        }, z.core.$strip>>>;
+        wins: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            description: z.ZodString;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>;
+    content: z.ZodString;
+    context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    model: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+declare const JudgeResultSchema: z.ZodObject<{
+    composite: z.ZodNumber;
+    dimensions: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    wins: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    rationale: z.ZodString;
+    rubricVersion: z.ZodString;
+    model: z.ZodString;
+    durationMs: z.ZodNumber;
+}, z.core.$strip>;
+declare const RubricInfoSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodString;
+    dimensions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        description: z.ZodString;
+        weight: z.ZodNumber;
+    }, z.core.$strip>>;
+    failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    rubricVersion: z.ZodString;
+}, z.core.$strip>;
+declare const ListRubricsResponseSchema: z.ZodObject<{
+    rubrics: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        dimensions: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            description: z.ZodString;
+            weight: z.ZodNumber;
+        }, z.core.$strip>>;
+        failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        rubricVersion: z.ZodString;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const VersionResponseSchema: z.ZodObject<{
+    package: z.ZodString;
+    version: z.ZodString;
+    wireVersion: z.ZodString;
+    apiSurface: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+declare const HealthResponseSchema: z.ZodObject<{
+    status: z.ZodLiteral<"ok">;
+    uptimeSec: z.ZodNumber;
+}, z.core.$strip>;
+declare const ErrorResponseSchema: z.ZodObject<{
+    error: z.ZodObject<{
+        code: z.ZodString;
+        message: z.ZodString;
+        details: z.ZodOptional<z.ZodUnknown>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+type RubricDimension = z.infer<typeof RubricDimensionSchema>;
+type FailureMode = z.infer<typeof FailureModeSchema>;
+type Rubric = z.infer<typeof RubricSchema>;
+type JudgeRequest = z.infer<typeof JudgeRequestSchema>;
+type JudgeResult = z.infer<typeof JudgeResultSchema>;
+type RubricInfo = z.infer<typeof RubricInfoSchema>;
+type ListRubricsResponse = z.infer<typeof ListRubricsResponseSchema>;
+type VersionResponse = z.infer<typeof VersionResponseSchema>;
+type ErrorResponse = z.infer<typeof ErrorResponseSchema>;
+/**
+ * Bump on any breaking change to a request/response schema.
+ * Non-breaking (additive) changes don't require a bump.
+ */
+declare const WIRE_VERSION = "1.0.0";
+/**
+ * Stable hash of a rubric. Used to make scores comparable across runs:
+ * if the rubricVersion matches, the rubric was identical.
+ */
+declare function hashRubric(rubric: Rubric): string;
+
+/** Caller-fixable error. The transport renders this to 4xx + ErrorResponse. */
+declare class WireError extends Error {
+    readonly code: string;
+    readonly status: number;
+    readonly details?: unknown | undefined;
+    constructor(code: string, message: string, status?: number, details?: unknown | undefined);
+}
+declare function handleJudge(req: JudgeRequest): Promise<JudgeResult>;
+declare function handleListRubrics(): ListRubricsResponse;
+declare function handleVersion(): VersionResponse;
+
+/**
+ * Built-in rubrics shipped with agent-eval.
+ *
+ * A rubric is a set of scoring axes plus a system prompt that tells the
+ * judging LLM how to grade against those axes. Built-in rubrics are
+ * curated for use cases that recur across Tangle projects — call them
+ * by name from any client.
+ *
+ * Adding a rubric:
+ *   1. Define the Rubric object below with a clear `description` and
+ *      named `dimensions`.
+ *   2. Register it in `BUILTIN_RUBRICS` at the bottom.
+ *   3. Add a test in `tests/wire/rubrics.test.ts`.
+ *
+ * Custom rubrics: callers pass `rubric` inline to /v1/judge instead of
+ * `rubricName` — see schemas.ts.
+ */
+
+declare const BUILTIN_RUBRICS: Record<string, Rubric>;
+/** Get a built-in rubric by name, or undefined. */
+declare function getBuiltinRubric(name: string): Rubric | undefined;
+/** List built-in rubrics with their stable versions. */
+declare function listBuiltinRubrics(): {
+    name: string;
+    description: string;
+    dimensions: {
+        id: string;
+        description: string;
+        weight: number;
+    }[];
+    failureModes: string[];
+    rubricVersion: string;
+}[];
+
+declare function buildOpenApi(packageVersion: string): OpenAPIObject;
+
+declare function createApp(): Hono<hono_types.BlankEnv, hono_types.BlankSchema, "/">;
+interface ServeOptions {
+    /** Default 5005. */
+    port?: number;
+    /** Default '127.0.0.1'. Set to '0.0.0.0' to listen on all interfaces. */
+    host?: string;
+}
+declare function startServer(opts?: ServeOptions): ServerType;
+
+interface RpcRequest {
+    method: 'judge' | 'listRubrics' | 'version';
+    params?: unknown;
+}
+interface RpcSuccess {
+    result: unknown;
+}
+interface RpcError {
+    error: {
+        code: string;
+        message: string;
+        details?: unknown;
+    };
+}
+declare function dispatchRpc(req: RpcRequest): Promise<RpcSuccess | RpcError>;
+/** Read one JSON request from stdin, write one JSON response to stdout. */
+declare function runRpcOnce(method?: string): Promise<number>;
+/** Read JSONL requests from stdin, write JSONL responses to stdout. */
+declare function runRpcBatch(method?: string): Promise<number>;
+
+export { BUILTIN_RUBRICS, type ErrorResponse, ErrorResponseSchema, type FailureMode, FailureModeSchema, HealthResponseSchema, type JudgeRequest, JudgeRequestSchema, type JudgeResult, JudgeResultSchema, type ListRubricsResponse, ListRubricsResponseSchema, type Rubric, type RubricDimension, RubricDimensionSchema, type RubricInfo, RubricInfoSchema, RubricSchema, type ServeOptions, type VersionResponse, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleJudge, handleListRubrics, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer };

package/dist/wire/index.js

@@ -0,0 +1,56 @@
+import {
+  BUILTIN_RUBRICS,
+  ErrorResponseSchema,
+  FailureModeSchema,
+  HealthResponseSchema,
+  JudgeRequestSchema,
+  JudgeResultSchema,
+  ListRubricsResponseSchema,
+  RubricDimensionSchema,
+  RubricInfoSchema,
+  RubricSchema,
+  VersionResponseSchema,
+  WIRE_VERSION,
+  WireError,
+  buildOpenApi,
+  createApp,
+  dispatchRpc,
+  getBuiltinRubric,
+  handleJudge,
+  handleListRubrics,
+  handleVersion,
+  hashRubric,
+  listBuiltinRubrics,
+  runRpcBatch,
+  runRpcOnce,
+  startServer
+} from "../chunk-OZPRSK4A.js";
+import "../chunk-ITN4YOZY.js";
+export {
+  BUILTIN_RUBRICS,
+  ErrorResponseSchema,
+  FailureModeSchema,
+  HealthResponseSchema,
+  JudgeRequestSchema,
+  JudgeResultSchema,
+  ListRubricsResponseSchema,
+  RubricDimensionSchema,
+  RubricInfoSchema,
+  RubricSchema,
+  VersionResponseSchema,
+  WIRE_VERSION,
+  WireError,
+  buildOpenApi,
+  createApp,
+  dispatchRpc,
+  getBuiltinRubric,
+  handleJudge,
+  handleListRubrics,
+  handleVersion,
+  hashRubric,
+  listBuiltinRubrics,
+  runRpcBatch,
+  runRpcOnce,
+  startServer
+};
+//# sourceMappingURL=index.js.map

package/dist/wire/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "Trace-first evaluation framework for Tangle agents. Core (spans, pipelines, sandbox harness, OTLP export), trust (dataset, red-team, calibration, behavior DSL), builder-of-builders (three-layer eval, resumable sessions, meta-runtime correlation), and frontier (meta-eval correlation study, Process Reward Modeling, bisector).",
   "type": "module",
   "main": "./dist/index.js",
@@ -20,8 +20,16 @@
       "types": "./dist/telemetry/file.d.ts",
       "import": "./dist/telemetry/file.js",
       "default": "./dist/telemetry/file.js"
+    },
+    "./wire": {
+      "types": "./dist/wire/index.d.ts",
+      "import": "./dist/wire/index.js",
+      "default": "./dist/wire/index.js"
     }
   },
+  "bin": {
+    "agent-eval": "./dist/cli.js"
+  },
   "files": [
     "dist"
   ],
@@ -29,11 +37,16 @@
     "access": "public"
   },
   "dependencies": {
+    "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
-    "@tangle-network/tcloud": "^0.2.0"
+    "@hono/node-server": "^2.0.0",
+    "@tangle-network/tcloud": "^0.2.0",
+    "hono": "^4.12.15",
+    "zod": "^4.3.6"
   },
   "devDependencies": {
     "@types/node": "^25.6.0",
+    "openapi3-ts": "^4.5.0",
     "tsup": "^8.0.0",
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
@@ -47,6 +60,7 @@
     "dev": "tsup --watch",
     "test": "vitest run",
     "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
   }
 }