@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/server-m5D9cvnG.js

@@ -0,0 +1,1040 @@
+import { i as CostLedger } from "./cost-ledger-DIgQUFZZ.js";
+import { a as assertLlmRoute, c as callLlmJson, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client--GR4JbZE.js";
+import { z } from "zod";
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { OpenAPIRegistry, OpenApiGeneratorV31, extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi";
+import { fileURLToPath } from "node:url";
+import { serve } from "@hono/node-server";
+import { Hono } from "hono";
+import { cors } from "hono/cors";
+//#region src/wire/schemas.ts
+/**
+* Wire-protocol schemas.
+*
+* These Zod schemas are the contract between the agent-eval runtime and
+* any non-TypeScript client (Python, Rust, Go, …). They get rendered to
+* OpenAPI by `wire/openapi.ts` and code-generators consume that spec to
+* produce typed clients in other languages.
+*
+* Rule: if it's not in this file, it isn't on the wire. Keep names and
+* shapes self-explanatory — every field has a `.describe()` so the
+* generated docs are useful without reading the source.
+*/
+extendZodWithOpenApi(z);
+const RubricDimensionSchema = z.object({
+	id: z.string().min(1).describe("Short stable id like \"buyer_quality\" — used as the key in scoring output."),
+	description: z.string().min(1).describe("One-line plain-English meaning. Read by humans reviewing low scores."),
+	weight: z.number().min(0).default(1).describe("Relative weight in the composite score. Default 1; 0 disables."),
+	min: z.number().default(0).describe("Lower bound of valid score for this dimension."),
+	max: z.number().default(1).describe("Upper bound of valid score for this dimension.")
+}).openapi("RubricDimension");
+const FailureModeSchema = z.object({
+	id: z.string().min(1).describe("Short stable id like \"ai-cadence\" — used in detection lists."),
+	description: z.string().min(1).describe("Plain-English description of the failure pattern.")
+}).openapi("FailureMode");
+const RubricSchema = z.object({
+	name: z.string().min(1).describe("Stable name like \"anti-slop\" — used by clients to invoke this rubric."),
+	description: z.string().min(1).describe("What this rubric measures. Shown in /v1/rubrics listing."),
+	systemPrompt: z.string().min(1).describe("Instructs the judging LLM. Should explain the persona (e.g. \"senior engineer reviewing voice\"), what to score on, and what to return."),
+	dimensions: z.array(RubricDimensionSchema).min(1).describe("Scoring axes. The composite score is a weighted sum of these."),
+	failureModes: z.array(FailureModeSchema).default([]).describe("Patterns to detect; each detected mode appears in the result.failureModes list."),
+	wins: z.array(FailureModeSchema).default([]).describe("Positive patterns; each detected one appears in the result.wins list.")
+}).openapi("Rubric");
+const JudgeRequestSchema = z.object({
+	rubricName: z.string().optional().describe("Use a built-in rubric by name. Mutually exclusive with `rubric`."),
+	rubric: RubricSchema.optional().describe("Inline rubric definition. Mutually exclusive with `rubricName`."),
+	content: z.string().min(1).describe("The text being judged — a tweet, a blog post, a code snippet, anything stringly."),
+	context: z.record(z.string(), z.unknown()).optional().describe("Free-form metadata for the rubric to use — analytics, source URL, author, etc. Surfaced to the LLM."),
+	model: z.string().optional().describe("Override the judge model configured by the server.")
+}).refine((v) => Boolean(v.rubricName) !== Boolean(v.rubric), { message: "Provide exactly one of `rubricName` or `rubric`." }).openapi("JudgeRequest");
+const JudgeResultSchema = z.object({
+	composite: z.number().min(0).max(1).describe("Weighted combination of dimension scores in 0..1. The single number to gate on."),
+	dimensions: z.record(z.string(), z.number()).describe("Per-dimension score, keyed by RubricDimension.id."),
+	failureModes: z.array(z.string()).default([]).describe("Failure-mode ids detected in the content (subset of rubric.failureModes ids)."),
+	wins: z.array(z.string()).default([]).describe("Win ids detected in the content (subset of rubric.wins ids)."),
+	rationale: z.string().describe("Plain-English explanation of the score. Surfaced to the human reviewer."),
+	rubricVersion: z.string().describe("Stable hash of the rubric used. Scores are only comparable across runs when this matches."),
+	model: z.string().describe("Model that produced the judgement, for reproducibility."),
+	durationMs: z.number().int().nonnegative().describe("End-to-end wall time for this call.")
+}).openapi("JudgeResult");
+const RubricInfoSchema = z.object({
+	name: z.string().describe("Pass this to /v1/judge as `rubricName`."),
+	description: z.string().describe("What this rubric measures."),
+	dimensions: z.array(z.object({
+		id: z.string(),
+		description: z.string(),
+		weight: z.number()
+	})).describe("The scoring axes this rubric uses, with weights."),
+	failureModes: z.array(z.string()).default([]).describe("Failure-mode ids this rubric detects."),
+	rubricVersion: z.string().describe("Stable hash — match this to compare scores across runs.")
+}).openapi("RubricInfo");
+const ListRubricsResponseSchema = z.object({ rubrics: z.array(RubricInfoSchema) }).openapi("ListRubricsResponse");
+const VersionResponseSchema = z.object({
+	package: z.string().describe("Package name (always \"@tangle-network/agent-eval\")."),
+	version: z.string().describe("Semver of the running server. Match your client to this."),
+	wireVersion: z.string().describe("Wire-protocol semver. Bumps separately from package version when the schema changes."),
+	apiSurface: z.array(z.string()).describe("List of supported method names.")
+}).openapi("VersionResponse");
+const HealthResponseSchema = z.object({
+	status: z.literal("ok"),
+	uptimeSec: z.number()
+}).openapi("HealthResponse");
+/**
+* Minimal `TraceEvent` shape that the production runtime emits.
+* Matches `trace/schema.ts` `TraceEvent` but is duplicated here as a
+* wire schema so non-TypeScript clients can validate without depending
+* on internal types.
+*/
+const TraceEventSchema = z.object({
+	eventId: z.string().min(1).describe("Stable id for the event. Use ULID or UUID."),
+	runId: z.string().min(1).describe("Run this event belongs to."),
+	spanId: z.string().optional().describe("Span that emitted the event, if any."),
+	kind: z.enum([
+		"log",
+		"error",
+		"budget_decrement",
+		"budget_breach",
+		"state_mutation",
+		"policy_violation",
+		"redaction_applied",
+		"custom"
+	]).describe("Coarse event category — matches the TraceSchema v1 EventKind enum."),
+	timestamp: z.number().int().nonnegative().describe("Unix millis. Must be monotonically non-decreasing within a span."),
+	payload: z.record(z.string(), z.unknown()).describe("Free-form payload — the runtime owns the shape.")
+}).openapi("TraceEvent");
+const TracesIngestRequestSchema = z.object({ events: z.array(TraceEventSchema).min(1).max(1e4).describe("Batch of events. Max 10k per call — bigger streams should be chunked.") }).openapi("TracesIngestRequest");
+const TracesIngestResponseSchema = z.object({
+	accepted: z.number().int().nonnegative().describe("Number of events persisted."),
+	rejected: z.number().int().nonnegative().describe("Number of events the store refused — see `errors[]` for reasons."),
+	errors: z.array(z.object({
+		eventId: z.string().describe("Event id this error applies to."),
+		message: z.string().describe("Why the event was rejected.")
+	})).default([])
+}).openapi("TracesIngestResponse");
+const FeedbackLabelSchema = z.object({
+	id: z.string().optional(),
+	source: z.enum([
+		"user",
+		"judge",
+		"environment",
+		"metric",
+		"policy",
+		"system"
+	]),
+	kind: z.enum([
+		"approve",
+		"reject",
+		"select",
+		"edit",
+		"rank",
+		"rate",
+		"comment",
+		"metric_outcome",
+		"policy_block",
+		"revision_request"
+	]),
+	value: z.unknown(),
+	reason: z.string().optional(),
+	severity: z.enum([
+		"info",
+		"warning",
+		"error",
+		"critical"
+	]).optional(),
+	createdAt: z.string().describe("ISO-8601 UTC."),
+	metadata: z.record(z.string(), z.unknown()).optional()
+}).openapi("FeedbackLabel");
+const FeedbackAttemptSchema = z.object({
+	id: z.string().min(1),
+	stepIndex: z.number().int().nonnegative(),
+	artifactType: z.enum([
+		"text",
+		"code",
+		"plan",
+		"research",
+		"action",
+		"ui",
+		"decision",
+		"data",
+		"other"
+	]),
+	artifact: z.unknown(),
+	options: z.array(z.unknown()).optional(),
+	proposedAction: z.object({
+		type: z.string(),
+		risk: z.enum([
+			"low",
+			"medium",
+			"high"
+		]).optional(),
+		costUsd: z.number().optional(),
+		externalSideEffect: z.boolean().optional(),
+		requiresApproval: z.boolean().optional(),
+		metadata: z.record(z.string(), z.unknown()).optional()
+	}).optional(),
+	feedback: z.array(FeedbackLabelSchema).optional(),
+	createdAt: z.string(),
+	metadata: z.record(z.string(), z.unknown()).optional()
+}).openapi("FeedbackAttempt");
+const FeedbackTrajectorySchema = z.object({
+	id: z.string().min(1).describe("Stable id; idempotency key for the trajectory."),
+	projectId: z.string().optional(),
+	scenarioId: z.string().optional(),
+	task: z.object({
+		intent: z.string().min(1),
+		context: z.unknown().optional()
+	}),
+	attempts: z.array(FeedbackAttemptSchema).default([]),
+	labels: z.array(FeedbackLabelSchema).default([]),
+	outcome: z.object({
+		success: z.boolean().optional(),
+		score: z.number().optional(),
+		metrics: z.record(z.string(), z.number()).optional(),
+		costUsd: z.number().optional(),
+		detail: z.string().optional(),
+		observedAt: z.string().optional(),
+		metadata: z.record(z.string(), z.unknown()).optional()
+	}).optional(),
+	split: z.enum([
+		"train",
+		"dev",
+		"test",
+		"holdout"
+	]).optional(),
+	tags: z.record(z.string(), z.string()).optional(),
+	createdAt: z.string().describe("ISO-8601 UTC."),
+	updatedAt: z.string().optional(),
+	metadata: z.record(z.string(), z.unknown()).optional()
+}).openapi("FeedbackTrajectory");
+const FeedbackIngestResponseSchema = z.object({
+	id: z.string().describe("Trajectory id that was persisted."),
+	persisted: z.boolean().describe("True when the trajectory was saved (idempotent on id).")
+}).openapi("FeedbackIngestResponse");
+const ErrorResponseSchema = z.object({ error: z.object({
+	code: z.string().describe("Machine-readable code: \"validation_error\", \"rubric_not_found\", \"judge_error\"."),
+	message: z.string().describe("Human-readable message."),
+	details: z.unknown().optional().describe("Optional structured detail.")
+}).describe("Errors are always wrapped in this shape across all endpoints.") }).openapi("ErrorResponse");
+/**
+* Bump on any breaking change to a request/response schema.
+* Non-breaking (additive) changes don't require a bump.
+*/
+const WIRE_VERSION = "1.0.0";
+/**
+* Stable hash of a rubric. Used to make scores comparable across runs:
+* if the rubricVersion matches, the rubric was identical.
+*/
+function hashRubric(rubric) {
+	const stable = stableStringify(rubric);
+	let h = 5381;
+	for (let i = 0; i < stable.length; i++) h = h * 33 ^ stable.charCodeAt(i);
+	return `${rubric.name}@${(h >>> 0).toString(16).padStart(8, "0")}`;
+}
+function stableStringify(value) {
+	if (Array.isArray(value)) return `[${value.map((item) => stableStringify(item)).join(",")}]`;
+	if (value && typeof value === "object") return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`).join(",")}}`;
+	return JSON.stringify(value);
+}
+const BUILTIN_RUBRICS = { "anti-slop": {
+	name: "anti-slop",
+	description: "Voice and signal quality for content aimed at senior engineers. Catches AI cadence, marketing tone, and engagement-bait shapes.",
+	systemPrompt: `You are evaluating a piece of content written for senior engineers and technical founders.
+
+You score three things:
+- buyer_quality (0..1): would a senior engineer in the target ICP find this worth their attention? High = specific, earned, technically interesting. Low = generic, hyped, off-target.
+- voice (0..1): does it read like a person who built the thing, or like AI/marketing copy?
+- signal (0..1): does it contain a non-obvious detail, constraint, or claim a reader couldn't get from the public docs?
+
+Detect failure modes (return ids matching):
+- ai-cadence: rule-of-three openings, em-dash flourish, "Let me explain", "Here's the thing", AI rhythm
+- marketing-tone: "We're excited to announce", "thrilled", "delighted", "game-changer", buzzword stack
+- vague-claim: technical claim without a specific component, file, or measurement
+- no-hook: opening doesn't earn attention from the target reader
+- engagement-bait: "agree?", "thoughts?", listicles, controversy-fishing, hook-detail-pitch
+- off-icp: content shape would attract motivational/grift/hype audiences instead of buyers
+- stale-claim: repeats a positioning line we've used many times this month
+
+Detect wins (return ids matching):
+- specific-component: names a real file, component, or measurement
+- earned-detail: shares a non-obvious detail not derivable from public docs
+- constraint-articulated: names a real tradeoff and the side chosen
+- honest-failure: describes a real failure mode and what was done about it
+
+Return ONLY JSON matching the response schema. Be conservative — most content has 0-1 wins and 1-2 failure modes, not many of each.`,
+	dimensions: [
+		{
+			id: "buyer_quality",
+			description: "Would the target buyer find this worth attention?",
+			weight: .5,
+			min: 0,
+			max: 1
+		},
+		{
+			id: "voice",
+			description: "Does it sound like a builder, not AI or marketing?",
+			weight: .3,
+			min: 0,
+			max: 1
+		},
+		{
+			id: "signal",
+			description: "Non-obvious detail, constraint, or claim?",
+			weight: .2,
+			min: 0,
+			max: 1
+		}
+	],
+	failureModes: [
+		{
+			id: "ai-cadence",
+			description: "AI-rhythm openings and transitions"
+		},
+		{
+			id: "marketing-tone",
+			description: "Buzzwords, hype, corporate-PR voice"
+		},
+		{
+			id: "vague-claim",
+			description: "Technical claim without specifics"
+		},
+		{
+			id: "no-hook",
+			description: "Opening fails to earn attention"
+		},
+		{
+			id: "engagement-bait",
+			description: "Listicle/controversy/agree-pattern"
+		},
+		{
+			id: "off-icp",
+			description: "Voice attracts the wrong audience"
+		},
+		{
+			id: "stale-claim",
+			description: "Reuses an over-used positioning line"
+		}
+	],
+	wins: [
+		{
+			id: "specific-component",
+			description: "Names a real file/component/number"
+		},
+		{
+			id: "earned-detail",
+			description: "Detail not in public docs"
+		},
+		{
+			id: "constraint-articulated",
+			description: "Names a real tradeoff"
+		},
+		{
+			id: "honest-failure",
+			description: "Describes a real failure honestly"
+		}
+	]
+} };
+/** Get a built-in rubric by name, or undefined. */
+function getBuiltinRubric(name) {
+	return BUILTIN_RUBRICS[name];
+}
+/** List built-in rubrics with their stable versions. */
+function listBuiltinRubrics() {
+	return Object.values(BUILTIN_RUBRICS).map((r) => ({
+		name: r.name,
+		description: r.description,
+		dimensions: r.dimensions.map((d) => ({
+			id: d.id,
+			description: d.description,
+			weight: d.weight
+		})),
+		failureModes: r.failureModes.map((f) => f.id),
+		rubricVersion: hashRubric(r)
+	}));
+}
+//#endregion
+//#region src/wire/handlers.ts
+/**
+* Pure handler functions — the "business logic" behind every wire-protocol
+* method. The HTTP server (`server.ts`) and the stdio RPC (`rpc.ts`) both
+* call these. Tests call these directly without spinning a server.
+*
+* Each handler:
+*   - Takes a parsed request (already Zod-validated by the transport).
+*   - Returns a result that matches the response schema.
+*   - Throws `WireError` for caller-fixable errors (404, 400, 422).
+*   - Lets unexpected errors bubble — the transport maps them to 500.
+*/
+/** Caller-fixable error. The transport renders this to 4xx + ErrorResponse. */
+var WireError = class extends Error {
+	code;
+	status;
+	details;
+	constructor(code, message, status = 400, details) {
+		super(message);
+		this.code = code;
+		this.status = status;
+		this.details = details;
+		this.name = "WireError";
+	}
+};
+/** The JSON schema we ask the judging LLM to fill in. */
+function judgeOutputSchema(rubric) {
+	return {
+		name: "JudgeOutput",
+		schema: {
+			type: "object",
+			additionalProperties: false,
+			properties: {
+				dimensions: {
+					type: "object",
+					additionalProperties: false,
+					properties: Object.fromEntries(rubric.dimensions.map((d) => [d.id, {
+						type: "number",
+						minimum: d.min,
+						maximum: d.max
+					}])),
+					required: rubric.dimensions.map((d) => d.id)
+				},
+				failureModes: {
+					type: "array",
+					items: {
+						type: "string",
+						enum: rubric.failureModes.map((f) => f.id)
+					}
+				},
+				wins: {
+					type: "array",
+					items: {
+						type: "string",
+						enum: rubric.wins.map((w) => w.id)
+					}
+				},
+				rationale: { type: "string" }
+			},
+			required: ["dimensions", "rationale"]
+		}
+	};
+}
+function validateJudgeOutput(value, rubric) {
+	if (!value || typeof value !== "object") throw new WireError("judge_error", "Judge returned malformed output.", 500, value);
+	const raw = value;
+	const rawDimensions = raw.dimensions;
+	if (!rawDimensions || typeof rawDimensions !== "object" || Array.isArray(rawDimensions)) throw new WireError("judge_error", "Judge returned malformed dimensions.", 500, value);
+	const dimensions = {};
+	const dimensionRecord = rawDimensions;
+	for (const dim of rubric.dimensions) {
+		const score = dimensionRecord[dim.id];
+		if (typeof score !== "number" || !Number.isFinite(score) || score < dim.min || score > dim.max) throw new WireError("judge_error", `Judge returned invalid score for dimension "${dim.id}".`, 500, value);
+		dimensions[dim.id] = score;
+	}
+	const allowedFailures = new Set(rubric.failureModes.map((mode) => mode.id));
+	const allowedWins = new Set(rubric.wins.map((win) => win.id));
+	const failureModes = validateIdArray(raw.failureModes, allowedFailures, "failureModes", value);
+	const wins = validateIdArray(raw.wins, allowedWins, "wins", value);
+	if (typeof raw.rationale !== "string" || raw.rationale.trim().length === 0) throw new WireError("judge_error", "Judge returned missing rationale.", 500, value);
+	return {
+		dimensions,
+		failureModes,
+		wins,
+		rationale: raw.rationale
+	};
+}
+function validateIdArray(raw, allowed, field, original) {
+	if (raw === void 0) return [];
+	if (!Array.isArray(raw)) throw new WireError("judge_error", `Judge returned non-array ${field}.`, 500, original);
+	const out = [];
+	for (const item of raw) {
+		if (typeof item !== "string" || !allowed.has(item)) throw new WireError("judge_error", `Judge returned unknown ${field} id "${String(item)}".`, 500, original);
+		out.push(item);
+	}
+	return out;
+}
+function compositeScore(dimensions, rubric) {
+	let weighted = 0;
+	let totalWeight = 0;
+	for (const dim of rubric.dimensions) {
+		const raw = dimensions[dim.id] ?? 0;
+		const range = dim.max - dim.min || 1;
+		const normalized = Math.max(0, Math.min(1, (raw - dim.min) / range));
+		weighted += normalized * dim.weight;
+		totalWeight += dim.weight;
+	}
+	return totalWeight > 0 ? weighted / totalWeight : 0;
+}
+function buildJudgePrompt(content, context) {
+	const ctx = context && Object.keys(context).length ? JSON.stringify(context) : "";
+	return [
+		`CONTENT TO JUDGE:`,
+		content,
+		"",
+		ctx ? `CONTEXT (metadata, analytics, etc.):` : "",
+		ctx ? ctx : ""
+	].filter(Boolean).join("\n");
+}
+const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6";
+async function handleJudge(req, options = {}) {
+	let rubric;
+	if (req.rubricName) {
+		const found = getBuiltinRubric(req.rubricName);
+		if (!found) throw new WireError("rubric_not_found", `No built-in rubric named "${req.rubricName}".`, 404);
+		rubric = found;
+	} else if (req.rubric) rubric = req.rubric;
+	else throw new WireError("validation_error", "Provide either `rubricName` or `rubric`.", 422);
+	if (options.routeRequirements) try {
+		assertLlmRoute(options.llm ?? {}, options.routeRequirements);
+	} catch (error) {
+		if (!(error instanceof LlmRouteAssertionError)) throw error;
+		const noEndpoint = error.reason === "no_explicit_base_url";
+		throw new WireError(noEndpoint ? "llm_not_configured" : "llm_route_rejected", noEndpoint ? "No model endpoint is configured. Pass llm.baseUrl or configure the CLI provider environment variables." : error.message, 503, { reason: error.reason });
+	}
+	const startedAt = Date.now();
+	const model = req.model ?? options.defaultModel ?? DEFAULT_JUDGE_MODEL;
+	const request = {
+		model,
+		messages: [{
+			role: "system",
+			content: rubric.systemPrompt
+		}, {
+			role: "user",
+			content: buildJudgePrompt(req.content, req.context)
+		}],
+		jsonSchema: judgeOutputSchema(rubric),
+		temperature: 0,
+		maxTokens: 4e3,
+		timeoutMs: 6e4
+	};
+	const paid = await (options.costLedger ?? new CostLedger()).runPaidCall({
+		channel: "judge",
+		phase: options.costPhase ?? "wire.judge",
+		actor: `wire.${req.rubricName ?? "inline"}`,
+		model,
+		maximumCharge: maximumChargeForLlmRequest(request, options.llm),
+		signal: options.signal,
+		execute: (signal, callId) => callLlmJson(request, {
+			...options.llm,
+			signal,
+			idempotencyKey: callId
+		}),
+		receipt: ({ result }) => costReceiptFromLlm(result),
+		receiptFromError: costReceiptFromLlmError
+	});
+	if (!paid.succeeded) throw paid.error;
+	const { value, result } = paid.value;
+	const output = validateJudgeOutput(value, rubric);
+	const composite = compositeScore(output.dimensions, rubric);
+	const durationMs = Date.now() - startedAt;
+	return {
+		composite,
+		dimensions: output.dimensions,
+		failureModes: output.failureModes ?? [],
+		wins: output.wins ?? [],
+		rationale: output.rationale,
+		rubricVersion: hashRubric(rubric),
+		model: result.model,
+		durationMs
+	};
+}
+function handleListRubrics() {
+	return { rubrics: listBuiltinRubrics() };
+}
+let CACHED_VERSION;
+function readPackageVersion() {
+	if (CACHED_VERSION) return CACHED_VERSION;
+	const here = dirname(fileURLToPath(import.meta.url));
+	const candidates = [resolve(here, "..", "..", "package.json"), resolve(here, "..", "package.json")];
+	for (const path of candidates) try {
+		const pkg = JSON.parse(readFileSync(path, "utf-8"));
+		if (pkg.version) {
+			CACHED_VERSION = pkg.version;
+			return pkg.version;
+		}
+	} catch {}
+	return "0.0.0-unknown";
+}
+function handleVersion() {
+	return {
+		package: "@tangle-network/agent-eval",
+		version: readPackageVersion(),
+		wireVersion: WIRE_VERSION,
+		apiSurface: [
+			"judge",
+			"listRubrics",
+			"version",
+			"feedback.ingest",
+			"traces.ingest"
+		]
+	};
+}
+/**
+* `POST /v1/traces/ingest` — accept a batch of `TraceEvent`s from the
+* production runtime. Best-effort: each event is appended independently;
+* one bad event does not poison the batch.
+*
+* Idempotency: the underlying store is append-only; consumers retrying
+* the same payload will get duplicate events. Consumers should
+* de-duplicate by `eventId` downstream — production traces frequently
+* land via at-least-once buses (Kafka, SQS) where dedup is unavoidable.
+*/
+async function handleTracesIngest(req, stores) {
+	if (!stores.traceStore) throw new WireError("service_unavailable", "No trace store configured on this server. Pass `traceStore` to `createApp`.", 503);
+	const errors = [];
+	let accepted = 0;
+	for (const event of req.events) try {
+		await stores.traceStore.appendEvent(event);
+		accepted++;
+	} catch (err) {
+		errors.push({
+			eventId: event.eventId,
+			message: err instanceof Error ? err.message : String(err)
+		});
+	}
+	return {
+		accepted,
+		rejected: errors.length,
+		errors
+	};
+}
+/**
+* `POST /v1/feedback` — accept a single `FeedbackTrajectory` from the
+* production runtime. Idempotent on `id`: re-posting the same trajectory
+* replaces the prior record.
+*/
+async function handleFeedbackIngest(req, stores) {
+	if (!stores.feedbackStore) throw new WireError("service_unavailable", "No feedback store configured on this server. Pass `feedbackStore` to `createApp`.", 503);
+	await stores.feedbackStore.save(req);
+	return {
+		id: req.id,
+		persisted: true
+	};
+}
+//#endregion
+//#region src/wire/openapi.ts
+/**
+* Build an OpenAPI spec from the wire schemas.
+*
+* The spec is the contract that other-language clients (Python, Rust,
+* Go, …) generate from. There is no hand-written client — clients are
+* derived artifacts of this file plus `schemas.ts`.
+*
+* Run `pnpm openapi` (defined in package.json) to write the spec to
+* `dist/openapi.json`. CI uses that file to regenerate the Python
+* client and gate the dual-publish workflow.
+*/
+function buildOpenApi(packageVersion) {
+	const registry = new OpenAPIRegistry();
+	registry.register("JudgeRequest", JudgeRequestSchema);
+	registry.register("JudgeResult", JudgeResultSchema);
+	registry.register("ListRubricsResponse", ListRubricsResponseSchema);
+	registry.register("VersionResponse", VersionResponseSchema);
+	registry.register("HealthResponse", HealthResponseSchema);
+	registry.register("ErrorResponse", ErrorResponseSchema);
+	registry.register("TracesIngestRequest", TracesIngestRequestSchema);
+	registry.register("TracesIngestResponse", TracesIngestResponseSchema);
+	registry.register("FeedbackTrajectory", FeedbackTrajectorySchema);
+	registry.register("FeedbackIngestResponse", FeedbackIngestResponseSchema);
+	registry.registerPath({
+		method: "post",
+		path: "/v1/judge",
+		summary: "Score a piece of content against a rubric",
+		description: "Runs the judging LLM with the named (or inline) rubric and returns dimension scores, detected failure modes, wins, and a composite score in 0..1.",
+		request: { body: { content: { "application/json": { schema: JudgeRequestSchema } } } },
+		responses: {
+			200: {
+				description: "Successful judgement",
+				content: { "application/json": { schema: JudgeResultSchema } }
+			},
+			400: {
+				description: "Validation error",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			},
+			404: {
+				description: "Rubric not found",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			},
+			500: {
+				description: "Judge error",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			}
+		}
+	});
+	registry.registerPath({
+		method: "get",
+		path: "/v1/rubrics",
+		summary: "List built-in rubrics",
+		description: "Returns every rubric registered server-side, with their dimensions and stable rubricVersion hash.",
+		responses: { 200: {
+			description: "Listing",
+			content: { "application/json": { schema: ListRubricsResponseSchema } }
+		} }
+	});
+	registry.registerPath({
+		method: "get",
+		path: "/v1/version",
+		summary: "Server and wire-protocol version",
+		description: "Match your client version to `version`; check `wireVersion` for compatibility.",
+		responses: { 200: {
+			description: "Version info",
+			content: { "application/json": { schema: VersionResponseSchema } }
+		} }
+	});
+	registry.registerPath({
+		method: "get",
+		path: "/healthz",
+		summary: "Liveness check",
+		responses: { 200: {
+			description: "OK",
+			content: { "application/json": { schema: HealthResponseSchema } }
+		} }
+	});
+	registry.registerPath({
+		method: "post",
+		path: "/v1/traces/ingest",
+		summary: "Ingest a batch of production TraceEvents",
+		description: "Append a batch of TraceEvents to the configured TraceStore. Accepts application/json ({events:[...]}) or application/x-ndjson (one event per line). Returns counts of accepted + rejected events.",
+		request: { body: { content: {
+			"application/json": { schema: TracesIngestRequestSchema },
+			"application/x-ndjson": { schema: TracesIngestRequestSchema }
+		} } },
+		responses: {
+			200: {
+				description: "Ingestion summary",
+				content: { "application/json": { schema: TracesIngestResponseSchema } }
+			},
+			400: {
+				description: "Validation error",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			},
+			401: {
+				description: "Unauthorized (when bearer auth is configured)",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			},
+			503: {
+				description: "No trace store configured",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			}
+		}
+	});
+	registry.registerPath({
+		method: "post",
+		path: "/v1/feedback",
+		summary: "Ingest a FeedbackTrajectory from production",
+		description: "Persist a single FeedbackTrajectory. Idempotent on trajectory.id — re-posting replaces the prior record. Used by production runtimes to forward user 👍/👎/edits into the eval substrate.",
+		request: { body: { content: { "application/json": { schema: FeedbackTrajectorySchema } } } },
+		responses: {
+			200: {
+				description: "Persisted",
+				content: { "application/json": { schema: FeedbackIngestResponseSchema } }
+			},
+			400: {
+				description: "Validation error",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			},
+			401: {
+				description: "Unauthorized (when bearer auth is configured)",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			},
+			503: {
+				description: "No feedback store configured",
+				content: { "application/json": { schema: ErrorResponseSchema } }
+			}
+		}
+	});
+	const doc = new OpenApiGeneratorV31(registry.definitions).generateDocument({
+		openapi: "3.1.0",
+		info: {
+			title: "@tangle-network/agent-eval — wire protocol",
+			version: packageVersion,
+			description: `HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.
+
+Wire-protocol version: ${WIRE_VERSION}. Bumps on breaking changes to request/response schemas.`,
+			contact: {
+				name: "Tangle Network",
+				url: "https://github.com/tangle-network/agent-eval"
+			},
+			license: { name: "MIT" }
+		},
+		servers: [{
+			url: "http://localhost:5005",
+			description: "Local agent-eval serve"
+		}]
+	});
+	const rubricRef = { $ref: "#/components/schemas/Rubric" };
+	const commonJudgeFields = {
+		content: {
+			type: "string",
+			minLength: 1
+		},
+		context: {
+			type: "object",
+			additionalProperties: true
+		},
+		model: { type: "string" }
+	};
+	doc.components ??= {};
+	doc.components.schemas ??= {};
+	doc.components.schemas.JudgeRequest = {
+		oneOf: [{
+			type: "object",
+			additionalProperties: false,
+			required: ["rubricName", "content"],
+			properties: {
+				rubricName: {
+					type: "string",
+					minLength: 1
+				},
+				...commonJudgeFields
+			}
+		}, {
+			type: "object",
+			additionalProperties: false,
+			required: ["rubric", "content"],
+			properties: {
+				rubric: rubricRef,
+				...commonJudgeFields
+			}
+		}],
+		description: "Judge request. Provide exactly one of rubricName or rubric."
+	};
+	return doc;
+}
+//#endregion
+//#region src/wire/rpc.ts
+async function dispatchRpc(req, options = {}) {
+	try {
+		switch (req.method) {
+			case "judge": {
+				const parsed = JudgeRequestSchema.safeParse(req.params);
+				if (!parsed.success) return { error: {
+					code: "validation_error",
+					message: "params did not match JudgeRequest schema.",
+					details: parsed.error.issues
+				} };
+				return { result: await handleJudge(parsed.data, {
+					llm: options.llm,
+					defaultModel: options.judgeModel,
+					routeRequirements: options.llmRouteRequirements
+				}) };
+			}
+			case "listRubrics": return { result: handleListRubrics() };
+			case "version": return { result: handleVersion() };
+			default: return { error: {
+				code: "unknown_method",
+				message: `No such method: ${req.method}`
+			} };
+		}
+	} catch (err) {
+		if (err instanceof WireError) return { error: {
+			code: err.code,
+			message: err.message,
+			details: err.details
+		} };
+		return { error: {
+			code: "internal_error",
+			message: err instanceof Error ? err.message : String(err)
+		} };
+	}
+}
+async function readAll(stream) {
+	const chunks = [];
+	for await (const chunk of stream) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+	return Buffer.concat(chunks).toString("utf-8");
+}
+/** Read one JSON request from stdin, write one JSON response to stdout. */
+async function runRpcOnce(method, options = {}) {
+	const raw = await readAll(process.stdin);
+	let req;
+	try {
+		const body = JSON.parse(raw);
+		req = method ? {
+			method,
+			params: body
+		} : body;
+	} catch (err) {
+		process.stdout.write(`${JSON.stringify({ error: {
+			code: "parse_error",
+			message: `stdin was not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+		} })}\n`);
+		return 1;
+	}
+	const out = await dispatchRpc(req, options);
+	process.stdout.write(`${JSON.stringify(out)}\n`);
+	return "error" in out ? 1 : 0;
+}
+/** Read JSONL requests from stdin, write JSONL responses to stdout. */
+async function runRpcBatch(method, options = {}) {
+	const lines = (await readAll(process.stdin)).split("\n").filter((l) => l.trim().length > 0);
+	let exitCode = 0;
+	for (const line of lines) {
+		let req;
+		try {
+			const body = JSON.parse(line);
+			req = method ? {
+				method,
+				params: body
+			} : body;
+		} catch (err) {
+			process.stdout.write(`${JSON.stringify({ error: {
+				code: "parse_error",
+				message: `line was not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+			} })}\n`);
+			exitCode = 1;
+			continue;
+		}
+		const out = await dispatchRpc(req, options);
+		process.stdout.write(`${JSON.stringify(out)}\n`);
+		if ("error" in out) exitCode = 1;
+	}
+	return exitCode;
+}
+//#endregion
+//#region src/wire/server.ts
+/**
+* HTTP transport for the wire protocol.
+*
+* Hono + @hono/node-server. Every endpoint:
+*   1. Validates the request against its Zod schema.
+*   2. Calls the matching handler in `handlers.ts`.
+*   3. Renders 4xx for `WireError` with structured body, 500 for unexpected.
+*
+* The server holds optional `IngestionStores` (passed to `createApp`)
+* to receive production traces and user feedback. With no stores wired,
+* the ingestion endpoints return 503 — read endpoints (`/v1/judge`,
+* `/v1/rubrics`, `/v1/version`) remain fully functional.
+*
+* Run via `agent-eval serve --port 5005`.
+*/
+const STARTED_AT = Date.now();
+const AUTH_EXEMPT_PATHS = /* @__PURE__ */ new Set([
+	"/healthz",
+	"/v1/version",
+	"/openapi.json"
+]);
+function createApp(opts = {}) {
+	const app = new Hono();
+	app.use("*", cors());
+	if (opts.auth) {
+		const verify = opts.auth.bearer;
+		app.use("*", async (c, next) => {
+			const path = new URL(c.req.url).pathname;
+			if (AUTH_EXEMPT_PATHS.has(path)) return next();
+			const match = (c.req.header("authorization") ?? "").match(/^Bearer\s+(.+)$/i);
+			if (!match) throw new WireError("unauthorized", "Missing or malformed Authorization header.", 401);
+			const token = match[1];
+			if (!(typeof verify === "string" ? token === verify : await verify(token))) throw new WireError("unauthorized", "Invalid bearer token.", 401);
+			return next();
+		});
+	}
+	app.onError((err, c) => {
+		if (err instanceof WireError) {
+			const status = err.status;
+			return c.json({ error: {
+				code: err.code,
+				message: err.message,
+				details: err.details
+			} }, status);
+		}
+		console.error("[agent-eval] unhandled error:", err);
+		return c.json({ error: {
+			code: "internal_error",
+			message: "Internal server error."
+		} }, 500);
+	});
+	app.get("/healthz", (c) => c.json({
+		status: "ok",
+		uptimeSec: (Date.now() - STARTED_AT) / 1e3
+	}));
+	app.get("/v1/version", (c) => c.json(handleVersion()));
+	app.get("/v1/rubrics", (c) => c.json(handleListRubrics()));
+	app.post("/v1/judge", async (c) => {
+		const raw = await c.req.json().catch(() => null);
+		if (raw == null) throw new WireError("validation_error", "Request body must be JSON.", 400);
+		const parsed = JudgeRequestSchema.safeParse(raw);
+		if (!parsed.success) throw new WireError("validation_error", "Request did not match JudgeRequest schema.", 400, parsed.error.issues);
+		const result = await handleJudge(parsed.data, {
+			llm: opts.llm,
+			defaultModel: opts.judgeModel,
+			routeRequirements: opts.llmRouteRequirements
+		});
+		return c.json(result);
+	});
+	app.post("/v1/traces/ingest", async (c) => {
+		const contentType = c.req.header("content-type") ?? "";
+		let payload;
+		if (contentType.includes("application/x-ndjson")) payload = { events: (await c.req.text()).split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => {
+			try {
+				return JSON.parse(line);
+			} catch {
+				throw new WireError("validation_error", "NDJSON line did not parse as JSON.", 400, line.slice(0, 200));
+			}
+		}) };
+		else payload = await c.req.json().catch(() => null);
+		if (payload == null) throw new WireError("validation_error", "Request body must be JSON or NDJSON.", 400);
+		const parsed = TracesIngestRequestSchema.safeParse(payload);
+		if (!parsed.success) throw new WireError("validation_error", "Request did not match TracesIngestRequest schema.", 400, parsed.error.issues);
+		const result = await handleTracesIngest(parsed.data, opts.stores ?? {});
+		return c.json(result);
+	});
+	app.post("/v1/feedback", async (c) => {
+		const raw = await c.req.json().catch(() => null);
+		if (raw == null) throw new WireError("validation_error", "Request body must be JSON.", 400);
+		const parsed = FeedbackTrajectorySchema.safeParse(raw);
+		if (!parsed.success) throw new WireError("validation_error", "Request did not match FeedbackTrajectory schema.", 400, parsed.error.issues);
+		const result = await handleFeedbackIngest(parsed.data, opts.stores ?? {});
+		return c.json(result);
+	});
+	app.get("/openapi.json", (c) => c.json(buildOpenApi(handleVersion().version)));
+	return app;
+}
+function startServer(opts = {}) {
+	const app = createApp(opts);
+	const port = opts.port ?? 5005;
+	const host = opts.host ?? "127.0.0.1";
+	return serve({
+		fetch: app.fetch,
+		port,
+		hostname: host
+	}, ({ address, port: actualPort }) => {
+		console.log(`[agent-eval] serving on http://${address}:${actualPort}`);
+	});
+}
+/**
+* Promise-returning variant of `startServer` that resolves once the server is
+* listening and surfaces the resolved bound port. Use this from smoke tests
+* (`startServerAsync({ port: 0 })`) and any caller that needs to dial back.
+*/
+function startServerAsync(opts = {}) {
+	const app = createApp(opts);
+	const port = opts.port ?? 5005;
+	const host = opts.host ?? "127.0.0.1";
+	return new Promise((resolve, reject) => {
+		let settled = false;
+		let server;
+		server = serve({
+			fetch: app.fetch,
+			port,
+			hostname: host
+		}, ({ address, port: actualPort }) => {
+			if (settled) return;
+			settled = true;
+			console.log(`[agent-eval] serving on http://${address}:${actualPort}`);
+			resolve({
+				server,
+				port: actualPort,
+				host: address,
+				close: () => new Promise((res, rej) => {
+					server.close((err) => err ? rej(err) : res());
+				})
+			});
+		});
+		server.on("error", (err) => {
+			if (settled) return;
+			settled = true;
+			reject(err);
+		});
+	});
+}
+//#endregion
+export { TraceEventSchema as A, HealthResponseSchema as C, RubricDimensionSchema as D, ListRubricsResponseSchema as E, hashRubric as F, TracesIngestResponseSchema as M, VersionResponseSchema as N, RubricInfoSchema as O, WIRE_VERSION as P, FeedbackTrajectorySchema as S, JudgeResultSchema as T, ErrorResponseSchema as _, runRpcBatch as a, FeedbackIngestResponseSchema as b, WireError as c, handleListRubrics as d, handleTracesIngest as f, listBuiltinRubrics as g, getBuiltinRubric as h, dispatchRpc as i, TracesIngestRequestSchema as j, RubricSchema as k, handleFeedbackIngest as l, BUILTIN_RUBRICS as m, startServer as n, runRpcOnce as o, handleVersion as p, startServerAsync as r, buildOpenApi as s, createApp as t, handleJudge as u, FailureModeSchema as v, JudgeRequestSchema as w, FeedbackLabelSchema as x, FeedbackAttemptSchema as y };
+
+//# sourceMappingURL=server-m5D9cvnG.js.map