@tangle-network/agent-eval 0.23.1 → 0.25.0

package/dist/{chunk-6KQG5HAH.js → chunk-5LBB5B3Z.js}

@@ -1,6 +1,6 @@
 import {
   callLlmJson
-} from "./chunk-KAO3Q65R.js";
+} from "./chunk-4S4BM3QQ.js";
 
 // src/wire/schemas.ts
 import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi";
@@ -74,9 +74,119 @@ var HealthResponseSchema = z.object({
   status: z.literal("ok"),
   uptimeSec: z.number()
 }).openapi("HealthResponse");
+var TraceEventSchema = z.object({
+  eventId: z.string().min(1).describe("Stable id for the event. Use ULID or UUID."),
+  runId: z.string().min(1).describe("Run this event belongs to."),
+  spanId: z.string().optional().describe("Span that emitted the event, if any."),
+  kind: z.enum([
+    "log",
+    "error",
+    "budget_decrement",
+    "budget_breach",
+    "state_mutation",
+    "policy_violation",
+    "redaction_applied",
+    "custom"
+  ]).describe("Coarse event category \u2014 matches the TraceSchema v1 EventKind enum."),
+  timestamp: z.number().int().nonnegative().describe("Unix millis. Must be monotonically non-decreasing within a span."),
+  payload: z.record(z.string(), z.unknown()).describe("Free-form payload \u2014 the runtime owns the shape.")
+}).openapi("TraceEvent");
+var TracesIngestRequestSchema = z.object({
+  events: z.array(TraceEventSchema).min(1).max(1e4).describe("Batch of events. Max 10k per call \u2014 bigger streams should be chunked.")
+}).openapi("TracesIngestRequest");
+var TracesIngestResponseSchema = z.object({
+  accepted: z.number().int().nonnegative().describe("Number of events persisted."),
+  rejected: z.number().int().nonnegative().describe("Number of events the store refused \u2014 see `errors[]` for reasons."),
+  errors: z.array(
+    z.object({
+      eventId: z.string().describe("Event id this error applies to."),
+      message: z.string().describe("Why the event was rejected.")
+    })
+  ).default([])
+}).openapi("TracesIngestResponse");
+var FeedbackLabelSchema = z.object({
+  id: z.string().optional(),
+  source: z.enum(["user", "judge", "environment", "metric", "policy", "system"]),
+  kind: z.enum([
+    "approve",
+    "reject",
+    "select",
+    "edit",
+    "rank",
+    "rate",
+    "comment",
+    "metric_outcome",
+    "policy_block",
+    "revision_request"
+  ]),
+  value: z.unknown(),
+  reason: z.string().optional(),
+  severity: z.enum(["info", "warning", "error", "critical"]).optional(),
+  createdAt: z.string().describe("ISO-8601 UTC."),
+  metadata: z.record(z.string(), z.unknown()).optional()
+}).openapi("FeedbackLabel");
+var FeedbackAttemptSchema = z.object({
+  id: z.string().min(1),
+  stepIndex: z.number().int().nonnegative(),
+  artifactType: z.enum([
+    "text",
+    "code",
+    "plan",
+    "research",
+    "action",
+    "ui",
+    "decision",
+    "data",
+    "other"
+  ]),
+  artifact: z.unknown(),
+  options: z.array(z.unknown()).optional(),
+  proposedAction: z.object({
+    type: z.string(),
+    risk: z.enum(["low", "medium", "high"]).optional(),
+    costUsd: z.number().optional(),
+    externalSideEffect: z.boolean().optional(),
+    requiresApproval: z.boolean().optional(),
+    metadata: z.record(z.string(), z.unknown()).optional()
+  }).optional(),
+  feedback: z.array(FeedbackLabelSchema).optional(),
+  createdAt: z.string(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+}).openapi("FeedbackAttempt");
+var FeedbackTrajectorySchema = z.object({
+  id: z.string().min(1).describe("Stable id; idempotency key for the trajectory."),
+  projectId: z.string().optional(),
+  scenarioId: z.string().optional(),
+  task: z.object({
+    intent: z.string().min(1),
+    context: z.unknown().optional()
+  }),
+  attempts: z.array(FeedbackAttemptSchema).default([]),
+  labels: z.array(FeedbackLabelSchema).default([]),
+  outcome: z.object({
+    success: z.boolean().optional(),
+    score: z.number().optional(),
+    metrics: z.record(z.string(), z.number()).optional(),
+    costUsd: z.number().optional(),
+    detail: z.string().optional(),
+    observedAt: z.string().optional(),
+    metadata: z.record(z.string(), z.unknown()).optional()
+  }).optional(),
+  split: z.enum(["train", "dev", "test", "holdout"]).optional(),
+  tags: z.record(z.string(), z.string()).optional(),
+  createdAt: z.string().describe("ISO-8601 UTC."),
+  updatedAt: z.string().optional(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+}).openapi("FeedbackTrajectory");
+var FeedbackIngestResponseSchema = z.object({
+  id: z.string().describe("Trajectory id that was persisted."),
+  persisted: z.boolean().describe("True when the trajectory was saved (idempotent on id).")
+}).openapi("FeedbackIngestResponse");
 var ErrorResponseSchema = z.object({
   error: z.object({
-    code: z.string().describe('Machine-readable code: "validation_error", "rubric_not_found", "judge_error".'),
+    code: z.string().describe(
+      'Machine-readable code: "validation_error", "rubric_not_found", "judge_error".'
+    ),
     message: z.string().describe("Human-readable message."),
     details: z.unknown().optional().describe("Optional structured detail.")
   }).describe("Errors are always wrapped in this shape across all endpoints.")
@@ -247,7 +357,12 @@ function validateJudgeOutput(value, rubric) {
   for (const dim of rubric.dimensions) {
     const score = dimensionRecord[dim.id];
     if (typeof score !== "number" || !Number.isFinite(score) || score < dim.min || score > dim.max) {
-      throw new WireError("judge_error", `Judge returned invalid score for dimension "${dim.id}".`, 500, value);
+      throw new WireError(
+        "judge_error",
+        `Judge returned invalid score for dimension "${dim.id}".`,
+        500,
+        value
+      );
     }
     dimensions[dim.id] = score;
   }
@@ -268,7 +383,12 @@ function validateIdArray(raw, allowed, field, original) {
   const out = [];
   for (const item of raw) {
     if (typeof item !== "string" || !allowed.has(item)) {
-      throw new WireError("judge_error", `Judge returned unknown ${field} id "${String(item)}".`, 500, original);
+      throw new WireError(
+        "judge_error",
+        `Judge returned unknown ${field} id "${String(item)}".`,
+        500,
+        original
+      );
     }
     out.push(item);
   }
@@ -366,12 +486,46 @@ function handleVersion() {
     package: "@tangle-network/agent-eval",
     version: readPackageVersion(),
     wireVersion: WIRE_VERSION,
-    apiSurface: ["judge", "listRubrics", "version"]
+    apiSurface: ["judge", "listRubrics", "version", "feedback.ingest", "traces.ingest"]
   };
 }
+async function handleTracesIngest(req, stores) {
+  if (!stores.traceStore) {
+    throw new WireError(
+      "service_unavailable",
+      "No trace store configured on this server. Pass `traceStore` to `createApp`.",
+      503
+    );
+  }
+  const errors = [];
+  let accepted = 0;
+  for (const event of req.events) {
+    try {
+      await stores.traceStore.appendEvent(event);
+      accepted++;
+    } catch (err) {
+      errors.push({
+        eventId: event.eventId,
+        message: err instanceof Error ? err.message : String(err)
+      });
+    }
+  }
+  return { accepted, rejected: errors.length, errors };
+}
+async function handleFeedbackIngest(req, stores) {
+  if (!stores.feedbackStore) {
+    throw new WireError(
+      "service_unavailable",
+      "No feedback store configured on this server. Pass `feedbackStore` to `createApp`.",
+      503
+    );
+  }
+  await stores.feedbackStore.save(req);
+  return { id: req.id, persisted: true };
+}
 
 // src/wire/openapi.ts
-import { OpenApiGeneratorV31, OpenAPIRegistry } from "@asteasolutions/zod-to-openapi";
+import { OpenAPIRegistry, OpenApiGeneratorV31 } from "@asteasolutions/zod-to-openapi";
 function buildOpenApi(packageVersion) {
   const registry = new OpenAPIRegistry();
   registry.register("JudgeRequest", JudgeRequestSchema);
@@ -380,6 +534,10 @@ function buildOpenApi(packageVersion) {
   registry.register("VersionResponse", VersionResponseSchema);
   registry.register("HealthResponse", HealthResponseSchema);
   registry.register("ErrorResponse", ErrorResponseSchema);
+  registry.register("TracesIngestRequest", TracesIngestRequestSchema);
+  registry.register("TracesIngestResponse", TracesIngestResponseSchema);
+  registry.register("FeedbackTrajectory", FeedbackTrajectorySchema);
+  registry.register("FeedbackIngestResponse", FeedbackIngestResponseSchema);
   registry.registerPath({
     method: "post",
     path: "/v1/judge",
@@ -446,6 +604,69 @@ function buildOpenApi(packageVersion) {
       }
     }
   });
+  registry.registerPath({
+    method: "post",
+    path: "/v1/traces/ingest",
+    summary: "Ingest a batch of production TraceEvents",
+    description: "Append a batch of TraceEvents to the configured TraceStore. Accepts application/json ({events:[...]}) or application/x-ndjson (one event per line). Returns counts of accepted + rejected events.",
+    request: {
+      body: {
+        content: {
+          "application/json": { schema: TracesIngestRequestSchema },
+          "application/x-ndjson": { schema: TracesIngestRequestSchema }
+        }
+      }
+    },
+    responses: {
+      200: {
+        description: "Ingestion summary",
+        content: { "application/json": { schema: TracesIngestResponseSchema } }
+      },
+      400: {
+        description: "Validation error",
+        content: { "application/json": { schema: ErrorResponseSchema } }
+      },
+      401: {
+        description: "Unauthorized (when bearer auth is configured)",
+        content: { "application/json": { schema: ErrorResponseSchema } }
+      },
+      503: {
+        description: "No trace store configured",
+        content: { "application/json": { schema: ErrorResponseSchema } }
+      }
+    }
+  });
+  registry.registerPath({
+    method: "post",
+    path: "/v1/feedback",
+    summary: "Ingest a FeedbackTrajectory from production",
+    description: "Persist a single FeedbackTrajectory. Idempotent on trajectory.id \u2014 re-posting replaces the prior record. Used by production runtimes to forward user \u{1F44D}/\u{1F44E}/edits into the eval substrate.",
+    request: {
+      body: {
+        content: {
+          "application/json": { schema: FeedbackTrajectorySchema }
+        }
+      }
+    },
+    responses: {
+      200: {
+        description: "Persisted",
+        content: { "application/json": { schema: FeedbackIngestResponseSchema } }
+      },
+      400: {
+        description: "Validation error",
+        content: { "application/json": { schema: ErrorResponseSchema } }
+      },
+      401: {
+        description: "Unauthorized (when bearer auth is configured)",
+        content: { "application/json": { schema: ErrorResponseSchema } }
+      },
+      503: {
+        description: "No feedback store configured",
+        content: { "application/json": { schema: ErrorResponseSchema } }
+      }
+    }
+  });
   const generator = new OpenApiGeneratorV31(registry.definitions);
   const doc = generator.generateDocument({
     openapi: "3.1.0",
@@ -494,62 +715,6 @@ Wire-protocol version: ${WIRE_VERSION}. Bumps on breaking changes to request/res
   return doc;
 }
 
-// src/wire/server.ts
-import { serve } from "@hono/node-server";
-import { Hono } from "hono";
-import { cors } from "hono/cors";
-var STARTED_AT = Date.now();
-function createApp() {
-  const app = new Hono();
-  app.use("*", cors());
-  app.onError((err, c) => {
-    if (err instanceof WireError) {
-      return c.json(
-        { error: { code: err.code, message: err.message, details: err.details } },
-        err.status
-      );
-    }
-    console.error("[agent-eval] unhandled error:", err);
-    return c.json(
-      { error: { code: "internal_error", message: "Internal server error." } },
-      500
-    );
-  });
-  app.get(
-    "/healthz",
-    (c) => c.json({ status: "ok", uptimeSec: (Date.now() - STARTED_AT) / 1e3 })
-  );
-  app.get("/v1/version", (c) => c.json(handleVersion()));
-  app.get("/v1/rubrics", (c) => c.json(handleListRubrics()));
-  app.post("/v1/judge", async (c) => {
-    const raw = await c.req.json().catch(() => null);
-    if (raw == null) {
-      throw new WireError("validation_error", "Request body must be JSON.", 400);
-    }
-    const parsed = JudgeRequestSchema.safeParse(raw);
-    if (!parsed.success) {
-      throw new WireError(
-        "validation_error",
-        "Request did not match JudgeRequest schema.",
-        400,
-        parsed.error.issues
-      );
-    }
-    const result = await handleJudge(parsed.data);
-    return c.json(result);
-  });
-  app.get("/openapi.json", (c) => c.json(buildOpenApi(handleVersion().version)));
-  return app;
-}
-function startServer(opts = {}) {
-  const app = createApp();
-  const port = opts.port ?? 5005;
-  const host = opts.host ?? "127.0.0.1";
-  return serve({ fetch: app.fetch, port, hostname: host }, ({ address, port: actualPort }) => {
-    console.log(`[agent-eval] serving on http://${address}:${actualPort}`);
-  });
-}
-
 // src/wire/rpc.ts
 async function dispatchRpc(req) {
   try {
@@ -602,17 +767,19 @@ async function runRpcOnce(method) {
     req = method ? { method, params: body } : body;
   } catch (err) {
     process.stdout.write(
-      JSON.stringify({
+      `${JSON.stringify({
         error: {
           code: "parse_error",
           message: `stdin was not valid JSON: ${err instanceof Error ? err.message : String(err)}`
         }
-      }) + "\n"
+      })}
+`
     );
     return 1;
   }
   const out = await dispatchRpc(req);
-  process.stdout.write(JSON.stringify(out) + "\n");
+  process.stdout.write(`${JSON.stringify(out)}
+`);
   return "error" in out ? 1 : 0;
 }
 async function runRpcBatch(method) {
@@ -626,23 +793,151 @@ async function runRpcBatch(method) {
       req = method ? { method, params: body } : body;
     } catch (err) {
       process.stdout.write(
-        JSON.stringify({
+        `${JSON.stringify({
           error: {
             code: "parse_error",
             message: `line was not valid JSON: ${err instanceof Error ? err.message : String(err)}`
           }
-        }) + "\n"
+        })}
+`
       );
       exitCode = 1;
       continue;
     }
     const out = await dispatchRpc(req);
-    process.stdout.write(JSON.stringify(out) + "\n");
+    process.stdout.write(`${JSON.stringify(out)}
+`);
     if ("error" in out) exitCode = 1;
   }
   return exitCode;
 }
 
+// src/wire/server.ts
+import { serve } from "@hono/node-server";
+import { Hono } from "hono";
+import { cors } from "hono/cors";
+var STARTED_AT = Date.now();
+var AUTH_EXEMPT_PATHS = /* @__PURE__ */ new Set(["/healthz", "/v1/version", "/openapi.json"]);
+function createApp(opts = {}) {
+  const app = new Hono();
+  app.use("*", cors());
+  if (opts.auth) {
+    const verify = opts.auth.bearer;
+    app.use("*", async (c, next) => {
+      const path = new URL(c.req.url).pathname;
+      if (AUTH_EXEMPT_PATHS.has(path)) return next();
+      const raw = c.req.header("authorization") ?? "";
+      const match = raw.match(/^Bearer\s+(.+)$/i);
+      if (!match) {
+        throw new WireError("unauthorized", "Missing or malformed Authorization header.", 401);
+      }
+      const token = match[1];
+      const ok = typeof verify === "string" ? token === verify : await verify(token);
+      if (!ok) {
+        throw new WireError("unauthorized", "Invalid bearer token.", 401);
+      }
+      return next();
+    });
+  }
+  app.onError((err, c) => {
+    if (err instanceof WireError) {
+      const status = err.status;
+      return c.json(
+        { error: { code: err.code, message: err.message, details: err.details } },
+        status
+      );
+    }
+    console.error("[agent-eval] unhandled error:", err);
+    return c.json({ error: { code: "internal_error", message: "Internal server error." } }, 500);
+  });
+  app.get(
+    "/healthz",
+    (c) => c.json({ status: "ok", uptimeSec: (Date.now() - STARTED_AT) / 1e3 })
+  );
+  app.get("/v1/version", (c) => c.json(handleVersion()));
+  app.get("/v1/rubrics", (c) => c.json(handleListRubrics()));
+  app.post("/v1/judge", async (c) => {
+    const raw = await c.req.json().catch(() => null);
+    if (raw == null) {
+      throw new WireError("validation_error", "Request body must be JSON.", 400);
+    }
+    const parsed = JudgeRequestSchema.safeParse(raw);
+    if (!parsed.success) {
+      throw new WireError(
+        "validation_error",
+        "Request did not match JudgeRequest schema.",
+        400,
+        parsed.error.issues
+      );
+    }
+    const result = await handleJudge(parsed.data);
+    return c.json(result);
+  });
+  app.post("/v1/traces/ingest", async (c) => {
+    const contentType = c.req.header("content-type") ?? "";
+    let payload;
+    if (contentType.includes("application/x-ndjson")) {
+      const text = await c.req.text();
+      const events = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => {
+        try {
+          return JSON.parse(line);
+        } catch {
+          throw new WireError(
+            "validation_error",
+            "NDJSON line did not parse as JSON.",
+            400,
+            line.slice(0, 200)
+          );
+        }
+      });
+      payload = { events };
+    } else {
+      payload = await c.req.json().catch(() => null);
+    }
+    if (payload == null) {
+      throw new WireError("validation_error", "Request body must be JSON or NDJSON.", 400);
+    }
+    const parsed = TracesIngestRequestSchema.safeParse(payload);
+    if (!parsed.success) {
+      throw new WireError(
+        "validation_error",
+        "Request did not match TracesIngestRequest schema.",
+        400,
+        parsed.error.issues
+      );
+    }
+    const result = await handleTracesIngest(parsed.data, opts.stores ?? {});
+    return c.json(result);
+  });
+  app.post("/v1/feedback", async (c) => {
+    const raw = await c.req.json().catch(() => null);
+    if (raw == null) {
+      throw new WireError("validation_error", "Request body must be JSON.", 400);
+    }
+    const parsed = FeedbackTrajectorySchema.safeParse(raw);
+    if (!parsed.success) {
+      throw new WireError(
+        "validation_error",
+        "Request did not match FeedbackTrajectory schema.",
+        400,
+        parsed.error.issues
+      );
+    }
+    const result = await handleFeedbackIngest(parsed.data, opts.stores ?? {});
+    return c.json(result);
+  });
+  app.get("/openapi.json", (c) => c.json(buildOpenApi(handleVersion().version)));
+  return app;
+}
+function startServer(opts = {}) {
+  const app = createApp(opts);
+  const port = opts.port ?? 5005;
+  const host = opts.host ?? "127.0.0.1";
+  return serve({ fetch: app.fetch, port, hostname: host }, ({ address, port: actualPort }) => {
+    console.log(`[agent-eval] serving on http://${address}:${actualPort}`);
+  });
+}
+
 export {
   RubricDimensionSchema,
   FailureModeSchema,
@@ -653,6 +948,13 @@ export {
   ListRubricsResponseSchema,
   VersionResponseSchema,
   HealthResponseSchema,
+  TraceEventSchema,
+  TracesIngestRequestSchema,
+  TracesIngestResponseSchema,
+  FeedbackLabelSchema,
+  FeedbackAttemptSchema,
+  FeedbackTrajectorySchema,
+  FeedbackIngestResponseSchema,
   ErrorResponseSchema,
   WIRE_VERSION,
   hashRubric,
@@ -663,11 +965,13 @@ export {
   handleJudge,
   handleListRubrics,
   handleVersion,
+  handleTracesIngest,
+  handleFeedbackIngest,
   buildOpenApi,
-  createApp,
-  startServer,
   dispatchRpc,
   runRpcOnce,
-  runRpcBatch
+  runRpcBatch,
+  createApp,
+  startServer
 };
-//# sourceMappingURL=chunk-6KQG5HAH.js.map
+//# sourceMappingURL=chunk-5LBB5B3Z.js.map