@axiom-lattice/gateway 2.1.73 → 2.1.74

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axiom-lattice/gateway",
-  "version": "2.1.73",
+  "version": "2.1.74",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
   "types": "dist/index.d.ts",
@@ -39,10 +39,11 @@
     "pg": "^8.11.0",
     "redis": "^5.0.1",
     "uuid": "^9.0.1",
-    "@axiom-lattice/core": "2.1.63",
-    "@axiom-lattice/pg-stores": "1.0.53",
-    "@axiom-lattice/protocols": "2.1.32",
-    "@axiom-lattice/queue-redis": "1.0.31"
+    "@axiom-lattice/agent-eval": "2.1.58",
+    "@axiom-lattice/core": "2.1.64",
+    "@axiom-lattice/pg-stores": "1.0.54",
+    "@axiom-lattice/protocols": "2.1.33",
+    "@axiom-lattice/queue-redis": "1.0.32"
   },
   "devDependencies": {
     "@types/jest": "^29.5.14",

package/src/controllers/eval.ts

@@ -0,0 +1,469 @@
+import { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
+import { getStoreLattice } from "@axiom-lattice/core";
+import { EvalStore } from "@axiom-lattice/protocols";
+import { v4 as uuidv4 } from "uuid";
+import { evalRunner, EvalStreamEvent } from "../services/eval-runner";
+
+function getTenantId(request: FastifyRequest): string {
+  const userTenantId = (request as any).user?.tenantId;
+  if (userTenantId) {
+    return userTenantId;
+  }
+  return (request.headers["x-tenant-id"] as string) || "default";
+}
+
+function getEvalStore(): EvalStore {
+  return getStoreLattice("default", "eval").store as EvalStore;
+}
+
+export async function createProject(
+  request: FastifyRequest<{ Body: Record<string, unknown> }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const id = uuidv4();
+    const data = request.body as any;
+    const project = await store.createProject(tenantId, id, {
+      name: data.name,
+      description: data.description,
+      version: data.version,
+      judgeModelConfig: data.judgeModelConfig ?? {},
+      targetServerConfig: data.targetServerConfig ?? {},
+      concurrency: data.concurrency ?? 1,
+      reportConfig: data.reportConfig,
+    });
+    return reply.status(201).send({
+      success: true,
+      message: "Successfully created project",
+      data: project,
+    });
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to create project";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function listProjects(
+  request: FastifyRequest,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    let projects = await store.getProjectsByTenant(tenantId);
+
+    // Seed a default project for new tenants
+    if (projects.length === 0) {
+      try {
+        const { modelLatticeManager } = await import("@axiom-lattice/core");
+        const models = modelLatticeManager.getAllLattices();
+        const first = models[0];
+        const judgeModel = first
+          ? { modelKey: first.key }
+          : { provider: "openai", model: "gpt-4" };
+
+        const host = request.hostname || "localhost";
+        const baseUrl = `http://${host}:${process.env.PORT || 4001}`;
+
+        await store.createProject(tenantId, uuidv4(), {
+          name: "Default",
+          description: "Built-in project for testing eval against this server",
+          judgeModelConfig: judgeModel,
+          targetServerConfig: { base_url: baseUrl, api_key: "" },
+          concurrency: 3,
+        });
+        projects = await store.getProjectsByTenant(tenantId);
+      } catch { /* seed failure is non-fatal */ }
+    }
+
+    return {
+      success: true,
+      message: "Ok",
+      data: { records: projects, total: projects.length },
+    };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to list projects";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function getProject(
+  request: FastifyRequest<{ Params: { pid: string } }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { pid } = request.params;
+    const project = await store.getProjectById(tenantId, pid);
+    if (!project) {
+      return reply.status(404).send({ success: false, message: "Project not found" });
+    }
+    const suites = await store.getSuitesByProject(tenantId, pid);
+    return {
+      success: true,
+      message: "Ok",
+      data: { project, suites },
+    };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to get project";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function updateProject(
+  request: FastifyRequest<{ Params: { pid: string }; Body: Record<string, unknown> }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { pid } = request.params;
+    const existing = await store.getProjectById(tenantId, pid);
+    if (!existing) {
+      return reply.status(404).send({ success: false, message: "Project not found" });
+    }
+    const updated = await store.updateProject(tenantId, pid, request.body as any);
+    return {
+      success: true,
+      message: "Successfully updated project",
+      data: updated,
+    };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to update project";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function deleteProject(
+  request: FastifyRequest<{ Params: { pid: string } }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { pid } = request.params;
+    const existing = await store.getProjectById(tenantId, pid);
+    if (!existing) {
+      return reply.status(404).send({ success: false, message: "Project not found" });
+    }
+    const deleted = await store.deleteProject(tenantId, pid);
+    if (!deleted) {
+      return reply.status(500).send({ success: false, message: "Failed to delete project" });
+    }
+    return { success: true, message: "Successfully deleted project" };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to delete project";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function createSuite(
+  request: FastifyRequest<{ Params: { pid: string }; Body: Record<string, unknown> }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { pid } = request.params;
+    const project = await store.getProjectById(tenantId, pid);
+    if (!project) {
+      return reply.status(404).send({ success: false, message: "Project not found" });
+    }
+    const id = uuidv4();
+    const data = request.body as any;
+    const suite = await store.createSuite(tenantId, pid, id, { name: data.name });
+    return reply.status(201).send({
+      success: true,
+      message: "Successfully created suite",
+      data: suite,
+    });
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to create suite";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function updateSuite(
+  request: FastifyRequest<{ Params: { pid: string; sid: string }; Body: Record<string, unknown> }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { sid } = request.params;
+    const existing = await store.getSuiteById(tenantId, sid);
+    if (!existing) {
+      return reply.status(404).send({ success: false, message: "Suite not found" });
+    }
+    const updated = await store.updateSuite(tenantId, sid, request.body as any);
+    return {
+      success: true,
+      message: "Successfully updated suite",
+      data: updated,
+    };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to update suite";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function deleteSuite(
+  request: FastifyRequest<{ Params: { pid: string; sid: string } }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { sid } = request.params;
+    const existing = await store.getSuiteById(tenantId, sid);
+    if (!existing) {
+      return reply.status(404).send({ success: false, message: "Suite not found" });
+    }
+    const deleted = await store.deleteSuite(tenantId, sid);
+    if (!deleted) {
+      return reply.status(500).send({ success: false, message: "Failed to delete suite" });
+    }
+    return { success: true, message: "Successfully deleted suite" };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to delete suite";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function createCase(
+  request: FastifyRequest<{ Params: { pid: string; sid: string }; Body: Record<string, unknown> }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { sid } = request.params;
+    const suite = await store.getSuiteById(tenantId, sid);
+    if (!suite) {
+      return reply.status(404).send({ success: false, message: "Suite not found" });
+    }
+    const id = uuidv4();
+    const data = request.body as any;
+    const created = await store.createCase(tenantId, sid, id, {
+      inputMessage: data.inputMessage,
+      inputFiles: data.inputFiles,
+      steps: data.steps ?? [],
+      outputType: data.outputType ?? "message_content",
+      contentAssertion: data.contentAssertion ?? "",
+      rubrics: data.rubrics,
+    });
+    return reply.status(201).send({
+      success: true,
+      message: "Successfully created case",
+      data: created,
+    });
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to create case";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function listCasesForSuite(
+  request: FastifyRequest<{ Params: { pid: string; sid: string } }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { sid } = request.params;
+    const cases = await store.getCasesBySuite(tenantId, sid);
+    return {
+      success: true,
+      message: "Ok",
+      data: { records: cases, total: cases.length },
+    };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to list cases";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function updateCase(
+  request: FastifyRequest<{ Params: { pid: string; sid: string; cid: string }; Body: Record<string, unknown> }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string; data?: unknown }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { cid } = request.params;
+    const existing = await store.getCaseById(tenantId, cid);
+    if (!existing) {
+      return reply.status(404).send({ success: false, message: "Case not found" });
+    }
+    const updated = await store.updateCase(tenantId, cid, request.body as any);
+    return {
+      success: true,
+      message: "Successfully updated case",
+      data: updated,
+    };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to update case";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export async function deleteCase(
+  request: FastifyRequest<{ Params: { pid: string; sid: string; cid: string } }>,
+  reply: FastifyReply
+): Promise<{ success: boolean; message: string }> {
+  try {
+    const tenantId = getTenantId(request);
+    const store = getEvalStore();
+    const { cid } = request.params;
+    const existing = await store.getCaseById(tenantId, cid);
+    if (!existing) {
+      return reply.status(404).send({ success: false, message: "Case not found" });
+    }
+    const deleted = await store.deleteCase(tenantId, cid);
+    if (!deleted) {
+      return reply.status(500).send({ success: false, message: "Failed to delete case" });
+    }
+    return { success: true, message: "Successfully deleted case" };
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : "Failed to delete case";
+    return reply.status(500).send({ success: false, message });
+  }
+}
+
+export function registerEvalRoutes(app: FastifyInstance): void {
+  app.post("/api/eval/projects", createProject);
+  app.get("/api/eval/projects", listProjects);
+  app.get("/api/eval/projects/:pid", getProject);
+  app.put("/api/eval/projects/:pid", updateProject);
+  app.delete("/api/eval/projects/:pid", deleteProject);
+
+  app.post("/api/eval/projects/:pid/suites", createSuite);
+  app.put("/api/eval/projects/:pid/suites/:sid", updateSuite);
+  app.delete("/api/eval/projects/:pid/suites/:sid", deleteSuite);
+
+  app.post("/api/eval/projects/:pid/suites/:sid/cases", createCase);
+  app.get("/api/eval/projects/:pid/suites/:sid/cases", listCasesForSuite);
+  app.put("/api/eval/projects/:pid/suites/:sid/cases/:cid", updateCase);
+  app.delete("/api/eval/projects/:pid/suites/:sid/cases/:cid", deleteCase);
+
+  // POST /api/eval/projects/:pid/runs — trigger run
+  app.post("/api/eval/projects/:pid/runs", async (request, reply) => {
+    try {
+      const tenantId = getTenantId(request);
+      const { pid } = request.params as { pid: string };
+      const runId = await evalRunner.startRun(tenantId, pid);
+      reply.status(202).send({ success: true, message: "Run started", data: { run_id: runId } });
+    } catch (err) {
+      const msg = (err as Error).message;
+      const code = msg === "Project not found" ? 404 : msg.includes("already in progress") ? 409 : 500;
+      reply.status(code).send({ success: false, message: msg });
+    }
+  });
+
+  // GET /api/eval/runs — list runs
+  app.get("/api/eval/runs", async (request, reply) => {
+    try {
+      const tenantId = getTenantId(request);
+      const query = request.query as { project_id?: string; status?: string };
+      const runs = await getEvalStore().getRunsByTenant(tenantId, { projectId: query.project_id, status: query.status });
+      reply.send({ success: true, message: "Ok", data: { records: runs, total: runs.length } });
+    } catch (err) {
+      reply.status(500).send({ success: false, message: (err as Error).message });
+    }
+  });
+
+  // GET /api/eval/runs/:rid — get run detail with results
+  app.get("/api/eval/runs/:rid", async (request, reply) => {
+    try {
+      const tenantId = getTenantId(request);
+      const { rid } = request.params as { rid: string };
+      const run = await getEvalStore().getRunById(tenantId, rid);
+      if (!run) return reply.status(404).send({ success: false, message: "Run not found" });
+      const results = await getEvalStore().getResultsByRun(tenantId, rid);
+      reply.send({ success: true, message: "Ok", data: { ...run, results } });
+    } catch (err) {
+      reply.status(500).send({ success: false, message: (err as Error).message });
+    }
+  });
+
+  // GET /api/eval/runs/:rid/stream — SSE progress stream
+  app.get("/api/eval/runs/:rid/stream", async (request, reply) => {
+    reply.hijack();
+    reply.raw.writeHead(200, {
+      "Content-Type": "text/event-stream",
+      "Cache-Control": "no-cache",
+      Connection: "keep-alive",
+      "Access-Control-Allow-Origin": "*",
+    });
+
+    const { rid } = request.params as { rid: string };
+    const emitter = evalRunner.getEventEmitter();
+    const eventKey = `run:${rid}`;
+
+    if (!evalRunner.isRunning(rid)) {
+      const tenantId = getTenantId(request);
+      const run = await getEvalStore().getRunById(tenantId, rid);
+      if (run) {
+        const eventType = run.status === "completed" ? "completed" : "error";
+        reply.raw.write(`event: ${eventType}\ndata: ${JSON.stringify({ passed: run.passedCases, failed: run.failedCases, avgScore: run.avgScore })}\n\n`);
+      }
+      reply.raw.end();
+      return;
+    }
+
+    const handler = (event: EvalStreamEvent): void => {
+      reply.raw.write(`event: ${event.type}\ndata: ${JSON.stringify(event.data)}\n\n`);
+      if (event.type === "completed" || event.type === "error") {
+        emitter.off(eventKey, handler);
+        reply.raw.end();
+      }
+    };
+
+    emitter.on(eventKey, handler);
+
+    request.raw.on("close", () => {
+      emitter.off(eventKey, handler);
+    });
+  });
+
+  // POST /api/eval/runs/:rid/abort — abort run
+  app.post("/api/eval/runs/:rid/abort", async (request, reply) => {
+    try {
+      const { rid } = request.params as { rid: string };
+      const aborted = await evalRunner.abortRun(rid);
+      if (!aborted) return reply.status(404).send({ success: false, message: "Run not found or not running" });
+      reply.send({ success: true, message: "Run aborted" });
+    } catch (err) {
+      reply.status(500).send({ success: false, message: (err as Error).message });
+    }
+  });
+
+  // DELETE /api/eval/runs/:rid — delete run and results
+  app.delete("/api/eval/runs/:rid", async (request, reply) => {
+    try {
+      const tenantId = getTenantId(request);
+      const { rid } = request.params as { rid: string };
+      const deleted = await getEvalStore().deleteRun(tenantId, rid);
+      if (!deleted) return reply.status(404).send({ success: false, message: "Run not found" });
+      reply.send({ success: true, message: "Run deleted" });
+    } catch (err) {
+      reply.status(500).send({ success: false, message: (err as Error).message });
+    }
+  });
+
+  // GET /api/eval/reports/projects/:pid — aggregated report
+  app.get("/api/eval/reports/projects/:pid", async (request, reply) => {
+    try {
+      const tenantId = getTenantId(request);
+      const { pid } = request.params as { pid: string };
+      const report = await getEvalStore().getProjectReport(tenantId, pid);
+      if (!report) return reply.status(404).send({ success: false, message: "Project not found" });
+      reply.send({ success: true, message: "Ok", data: report });
+    } catch (err) {
+      reply.status(500).send({ success: false, message: (err as Error).message });
+    }
+  });
+}

package/src/controllers/workflow-tracking.ts

@@ -27,6 +27,7 @@ interface ApiResponse<T = any> {
 async function getDefinitionsFromAssistants(tenantId: string): Promise<Array<{
   assistantId: string;
   assistantName: string;
+  description?: string;
   topologyEdges: { from: string; to: string; purpose: string }[];
   totalEdges: number;
 }>> {
@@ -51,6 +52,7 @@ async function getDefinitionsFromAssistants(tenantId: string): Promise<Array<{
           results.push({
             assistantId: a.id,
             assistantName: a.name,
+            description: a.description,
             topologyEdges: mw.config.edges,
             totalEdges: mw.config.edges.length,
           });

package/src/routes/index.ts

@@ -37,6 +37,7 @@ import { registerWorkspaceRoutes } from "../controllers/workspace";
 import { registerDatabaseConfigRoutes } from "../controllers/database-configs";
 import { registerMetricsServerConfigRoutes } from "../controllers/metrics-configs";
 import { registerMcpServerConfigRoutes } from "../controllers/mcp-configs";
+import { registerEvalRoutes } from "../controllers/eval";
 import { registerUserRoutes } from "../controllers/users";
 import { registerTenantRoutes } from "../controllers/tenants";
 import { registerAuthRoutes } from "../controllers/auth";
@@ -320,6 +321,8 @@ export const registerLatticeRoutes = (app: FastifyInstance): void => {
 
   registerSandboxProxyRoutes(app);
 
+  registerEvalRoutes(app);
+
   registerWorkspaceRoutes(app);
 
   registerDatabaseConfigRoutes(app);