@axlsdk/studio 0.13.8 → 0.14.0

package/dist/server/index.cjs

@@ -334,27 +334,30 @@ var CostAggregator = class {
 
 // src/server/routes/health.ts
 var import_hono = require("hono");
-var app = new import_hono.Hono();
-app.get("/health", (c) => {
-  const runtime = c.get("runtime");
-  return c.json({
-    ok: true,
-    data: {
-      status: "healthy",
-      workflows: runtime.getWorkflowNames().length,
-      agents: runtime.getAgents().length,
-      tools: runtime.getTools().length
-    }
+function createHealthRoutes(readOnly) {
+  const app6 = new import_hono.Hono();
+  app6.get("/health", (c) => {
+    const runtime = c.get("runtime");
+    return c.json({
+      ok: true,
+      data: {
+        status: "healthy",
+        readOnly,
+        workflows: runtime.getWorkflowNames().length,
+        agents: runtime.getAgents().length,
+        tools: runtime.getTools().length
+      }
+    });
   });
-});
-var health_default = app;
+  return app6;
+}
 
 // src/server/routes/workflows.ts
 var import_hono2 = require("hono");
 var import_axl = require("@axlsdk/axl");
 function createWorkflowRoutes(connMgr) {
-  const app7 = new import_hono2.Hono();
-  app7.get("/workflows", (c) => {
+  const app6 = new import_hono2.Hono();
+  app6.get("/workflows", (c) => {
     const runtime = c.get("runtime");
     const workflows = runtime.getWorkflows().map((w) => ({
       name: w.name,
@@ -363,7 +366,7 @@ function createWorkflowRoutes(connMgr) {
     }));
     return c.json({ ok: true, data: workflows });
   });
-  app7.get("/workflows/:name", (c) => {
+  app6.get("/workflows/:name", (c) => {
     const runtime = c.get("runtime");
     const name = c.req.param("name");
     const workflow = runtime.getWorkflow(name);
@@ -382,7 +385,7 @@ function createWorkflowRoutes(connMgr) {
       }
     });
   });
-  app7.post("/workflows/:name/execute", async (c) => {
+  app6.post("/workflows/:name/execute", async (c) => {
     const runtime = c.get("runtime");
     const name = c.req.param("name");
     const workflow = runtime.getWorkflow(name);
@@ -406,18 +409,18 @@ function createWorkflowRoutes(connMgr) {
     const result = await runtime.execute(name, body.input ?? {}, { metadata: body.metadata });
     return c.json({ ok: true, data: { result } });
   });
-  return app7;
+  return app6;
 }
 
 // src/server/routes/executions.ts
 var import_hono3 = require("hono");
-var app2 = new import_hono3.Hono();
-app2.get("/executions", async (c) => {
+var app = new import_hono3.Hono();
+app.get("/executions", async (c) => {
   const runtime = c.get("runtime");
   const executions = await runtime.getExecutions();
   return c.json({ ok: true, data: executions });
 });
-app2.get("/executions/:id", async (c) => {
+app.get("/executions/:id", async (c) => {
   const runtime = c.get("runtime");
   const id = c.req.param("id");
   const execution = await runtime.getExecution(id);
@@ -429,19 +432,19 @@ app2.get("/executions/:id", async (c) => {
   }
   return c.json({ ok: true, data: execution });
 });
-app2.post("/executions/:id/abort", (c) => {
+app.post("/executions/:id/abort", (c) => {
   const runtime = c.get("runtime");
   const id = c.req.param("id");
   runtime.abort(id);
   return c.json({ ok: true, data: { aborted: true } });
 });
-var executions_default = app2;
+var executions_default = app;
 
 // src/server/routes/sessions.ts
 var import_hono4 = require("hono");
 function createSessionRoutes(connMgr) {
-  const app7 = new import_hono4.Hono();
-  app7.get("/sessions", async (c) => {
+  const app6 = new import_hono4.Hono();
+  app6.get("/sessions", async (c) => {
     const runtime = c.get("runtime");
     const store = runtime.getStateStore();
     if (!store.listSessions) {
@@ -455,7 +458,7 @@ function createSessionRoutes(connMgr) {
     }
     return c.json({ ok: true, data: sessions });
   });
-  app7.get("/sessions/:id", async (c) => {
+  app6.get("/sessions/:id", async (c) => {
     const runtime = c.get("runtime");
     const store = runtime.getStateStore();
     const id = c.req.param("id");
@@ -463,7 +466,7 @@ function createSessionRoutes(connMgr) {
     const handoffHistory = await store.getSessionMeta(id, "handoffHistory");
     return c.json({ ok: true, data: { id, history, handoffHistory: handoffHistory ?? [] } });
   });
-  app7.post("/sessions/:id/send", async (c) => {
+  app6.post("/sessions/:id/send", async (c) => {
     const runtime = c.get("runtime");
     const id = c.req.param("id");
     const body = await c.req.json();
@@ -471,7 +474,7 @@ function createSessionRoutes(connMgr) {
     const result = await session.send(body.workflow, body.message);
     return c.json({ ok: true, data: { result } });
   });
-  app7.post("/sessions/:id/stream", async (c) => {
+  app6.post("/sessions/:id/stream", async (c) => {
     const runtime = c.get("runtime");
     const id = c.req.param("id");
     const body = await c.req.json();
@@ -485,21 +488,21 @@ function createSessionRoutes(connMgr) {
     })();
     return c.json({ ok: true, data: { executionId, streaming: true } });
   });
-  app7.delete("/sessions/:id", async (c) => {
+  app6.delete("/sessions/:id", async (c) => {
     const runtime = c.get("runtime");
     const store = runtime.getStateStore();
     const id = c.req.param("id");
     await store.deleteSession(id);
     return c.json({ ok: true, data: { deleted: true } });
   });
-  return app7;
+  return app6;
 }
 
 // src/server/routes/agents.ts
 var import_hono5 = require("hono");
 var import_axl2 = require("@axlsdk/axl");
-var app3 = new import_hono5.Hono();
-app3.get("/agents", (c) => {
+var app2 = new import_hono5.Hono();
+app2.get("/agents", (c) => {
   const runtime = c.get("runtime");
   const agents = runtime.getAgents().map((a) => ({
     name: a._name,
@@ -518,7 +521,7 @@ app3.get("/agents", (c) => {
   }));
   return c.json({ ok: true, data: agents });
 });
-app3.get("/agents/:name", (c) => {
+app2.get("/agents/:name", (c) => {
   const runtime = c.get("runtime");
   const name = c.req.param("name");
   const agent = runtime.getAgent(name);
@@ -574,13 +577,13 @@ app3.get("/agents/:name", (c) => {
     }
   });
 });
-var agents_default = app3;
+var agents_default = app2;
 
 // src/server/routes/tools.ts
 var import_hono6 = require("hono");
 var import_axl3 = require("@axlsdk/axl");
-var app4 = new import_hono6.Hono();
-app4.get("/tools", (c) => {
+var app3 = new import_hono6.Hono();
+app3.get("/tools", (c) => {
   const runtime = c.get("runtime");
   const tools = runtime.getTools().map((t) => ({
     name: t.name,
@@ -591,7 +594,7 @@ app4.get("/tools", (c) => {
   }));
   return c.json({ ok: true, data: tools });
 });
-app4.get("/tools/:name", (c) => {
+app3.get("/tools/:name", (c) => {
   const runtime = c.get("runtime");
   const name = c.req.param("name");
   const tool = runtime.getTool(name);
@@ -618,7 +621,7 @@ app4.get("/tools/:name", (c) => {
     }
   });
 });
-app4.post("/tools/:name/test", async (c) => {
+app3.post("/tools/:name/test", async (c) => {
   const runtime = c.get("runtime");
   const name = c.req.param("name");
   const tool = runtime.getTool(name);
@@ -633,12 +636,12 @@ app4.post("/tools/:name/test", async (c) => {
   const result = await tool.run(ctx, body.input);
   return c.json({ ok: true, data: { result } });
 });
-var tools_default = app4;
+var tools_default = app3;
 
 // src/server/routes/memory.ts
 var import_hono7 = require("hono");
-var app5 = new import_hono7.Hono();
-app5.get("/memory/:scope", async (c) => {
+var app4 = new import_hono7.Hono();
+app4.get("/memory/:scope", async (c) => {
   const runtime = c.get("runtime");
   const store = runtime.getStateStore();
   const scope = c.req.param("scope");
@@ -648,7 +651,7 @@ app5.get("/memory/:scope", async (c) => {
   const entries = await store.getAllMemory(scope);
   return c.json({ ok: true, data: entries });
 });
-app5.get("/memory/:scope/:key", async (c) => {
+app4.get("/memory/:scope/:key", async (c) => {
   const runtime = c.get("runtime");
   const store = runtime.getStateStore();
   const scope = c.req.param("scope");
@@ -668,7 +671,7 @@ app5.get("/memory/:scope/:key", async (c) => {
   }
   return c.json({ ok: true, data: { key, value } });
 });
-app5.put("/memory/:scope/:key", async (c) => {
+app4.put("/memory/:scope/:key", async (c) => {
   const runtime = c.get("runtime");
   const store = runtime.getStateStore();
   const scope = c.req.param("scope");
@@ -683,7 +686,7 @@ app5.put("/memory/:scope/:key", async (c) => {
   await store.saveMemory(scope, key, body.value);
   return c.json({ ok: true, data: { saved: true } });
 });
-app5.delete("/memory/:scope/:key", async (c) => {
+app4.delete("/memory/:scope/:key", async (c) => {
   const runtime = c.get("runtime");
   const store = runtime.getStateStore();
   const scope = c.req.param("scope");
@@ -697,61 +700,77 @@ app5.delete("/memory/:scope/:key", async (c) => {
   await store.deleteMemory(scope, key);
   return c.json({ ok: true, data: { deleted: true } });
 });
-app5.post("/memory/search", async (c) => {
+app4.post("/memory/search", async (c) => {
   return c.json({
     ok: true,
     data: { results: [], message: "Semantic search requires MemoryManager with vector store" }
   });
 });
-var memory_default = app5;
+var memory_default = app4;
 
 // src/server/routes/decisions.ts
 var import_hono8 = require("hono");
-var app6 = new import_hono8.Hono();
-app6.get("/decisions", async (c) => {
+var app5 = new import_hono8.Hono();
+app5.get("/decisions", async (c) => {
   const runtime = c.get("runtime");
   const decisions = await runtime.getPendingDecisions();
   return c.json({ ok: true, data: decisions });
 });
-app6.post("/decisions/:executionId/resolve", async (c) => {
+app5.post("/decisions/:executionId/resolve", async (c) => {
   const runtime = c.get("runtime");
   const executionId = c.req.param("executionId");
   const body = await c.req.json();
   await runtime.resolveDecision(executionId, body);
   return c.json({ ok: true, data: { resolved: true } });
 });
-var decisions_default = app6;
+var decisions_default = app5;
 
 // src/server/routes/costs.ts
 var import_hono9 = require("hono");
 function createCostRoutes(costAggregator) {
-  const app7 = new import_hono9.Hono();
-  app7.get("/costs", (c) => {
+  const app6 = new import_hono9.Hono();
+  app6.get("/costs", (c) => {
     return c.json({ ok: true, data: costAggregator.getData() });
   });
-  app7.post("/costs/reset", (c) => {
+  app6.post("/costs/reset", (c) => {
     costAggregator.reset();
     return c.json({ ok: true, data: { reset: true } });
   });
-  return app7;
+  return app6;
 }
 
 // src/server/routes/evals.ts
+var import_node_crypto = require("crypto");
 var import_hono10 = require("hono");
 function createEvalRoutes(evalLoader) {
-  const app7 = new import_hono10.Hono();
-  app7.get("/evals", async (c) => {
+  const app6 = new import_hono10.Hono();
+  app6.get("/evals", async (c) => {
     if (evalLoader) await evalLoader();
     const runtime = c.get("runtime");
     const evals = runtime.getRegisteredEvals();
     return c.json({ ok: true, data: evals });
   });
-  app7.get("/evals/history", async (c) => {
+  app6.get("/evals/history", async (c) => {
     const runtime = c.get("runtime");
     const history = await runtime.getEvalHistory();
     return c.json({ ok: true, data: history });
   });
-  app7.post("/evals/:name/run", async (c) => {
+  app6.delete("/evals/history/:id", async (c) => {
+    const runtime = c.get("runtime");
+    const id = c.req.param("id");
+    const deleted = await runtime.deleteEvalResult(id);
+    if (!deleted) {
+      return c.json(
+        {
+          ok: false,
+          error: { code: "NOT_FOUND", message: `Eval history entry "${id}" not found` }
+        },
+        404
+      );
+    }
+    return c.json({ ok: true, data: { id, deleted: true } });
+  });
+  app6.post("/evals/:name/run", async (c) => {
     if (evalLoader) await evalLoader();
     const runtime = c.get("runtime");
     const name = c.req.param("name");
@@ -772,9 +791,8 @@ function createEvalRoutes(evalLoader) {
     }
     try {
       if (runs > 1) {
-        const { randomUUID } = await import("crypto");
         const { aggregateRuns } = await import("@axlsdk/eval");
-        const runGroupId = randomUUID();
+        const runGroupId = (0, import_node_crypto.randomUUID)();
         const results = [];
         for (let r = 0; r < runs; r++) {
           const result2 = await runtime.runRegisteredEval(name, {
@@ -796,7 +814,7 @@ function createEvalRoutes(evalLoader) {
       return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
     }
   });
-  app7.post("/evals/:name/rescore", async (c) => {
+  app6.post("/evals/:name/rescore", async (c) => {
     if (evalLoader) await evalLoader();
     const runtime = c.get("runtime");
     const name = c.req.param("name");
@@ -842,25 +860,146 @@ function createEvalRoutes(evalLoader) {
       return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
     }
   });
-  app7.post("/evals/compare", async (c) => {
+  app6.post("/evals/compare", async (c) => {
     const runtime = c.get("runtime");
     const body = await c.req.json();
+    const validateIdParam = (v, name) => {
+      if (typeof v === "string") return v === "" ? `${name} must be non-empty` : null;
+      if (Array.isArray(v)) {
+        if (v.length === 0) return `${name} must be a non-empty array`;
+        for (const elem of v) {
+          if (typeof elem !== "string" || elem === "") {
+            return `${name} array must contain only non-empty strings`;
+          }
+        }
+        return null;
+      }
+      return `${name} is required (string or string[])`;
+    };
+    const baselineErr = validateIdParam(body.baselineId, "baselineId");
+    const candidateErr = validateIdParam(body.candidateId, "candidateId");
+    if (baselineErr || candidateErr) {
+      return c.json(
+        {
+          ok: false,
+          error: {
+            code: "BAD_REQUEST",
+            message: [baselineErr, candidateErr].filter(Boolean).join("; ")
+          }
+        },
+        400
+      );
+    }
+    const history = await runtime.getEvalHistory();
+    const byId = new Map(history.map((h) => [h.id, h.data]));
+    const missing = [];
+    const resolveOne = (id) => {
+      const data = byId.get(id);
+      if (!data) missing.push(id);
+      return data;
+    };
+    const resolveSelection = (idOrIds) => {
+      if (Array.isArray(idOrIds)) {
+        const unique = Array.from(new Set(idOrIds));
+        if (unique.length === 1) return resolveOne(unique[0]);
+        const results = [];
+        for (const id of unique) {
+          const data = resolveOne(id);
+          if (data) results.push(data);
+        }
+        return results;
+      }
+      return resolveOne(idOrIds);
+    };
+    const baseline = resolveSelection(body.baselineId);
+    const candidate = resolveSelection(body.candidateId);
+    if (missing.length > 0) {
+      return c.json(
+        {
+          ok: false,
+          error: {
+            code: "NOT_FOUND",
+            message: `Eval result(s) not found in history: ${missing.join(", ")}`
+          }
+        },
+        404
+      );
+    }
     try {
-      const result = await runtime.evalCompare(body.baseline, body.candidate, body.options);
+      const result = await runtime.evalCompare(baseline, candidate, body.options);
       return c.json({ ok: true, data: result });
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
+      return c.json({ ok: false, error: { code: "COMPARE_FAILED", message } }, 400);
+    }
+  });
+  app6.post("/evals/import", async (c) => {
+    const runtime = c.get("runtime");
+    const body = await c.req.json();
+    const bad = (message) => c.json({ ok: false, error: { code: "BAD_REQUEST", message } }, 400);
+    if (!body.result || typeof body.result !== "object") {
+      return bad("result is required");
+    }
+    const result = body.result;
+    if (!Array.isArray(result.items)) {
+      return bad("result.items must be an array");
+    }
+    if (typeof result.summary !== "object" || result.summary == null) {
+      return bad("result.summary must be an object");
+    }
+    if (typeof result.dataset !== "string" || !result.dataset) {
+      return bad("result.dataset must be a non-empty string (required for compare)");
     }
+    const summary = result.summary;
+    if (typeof summary.scorers !== "object" || summary.scorers == null) {
+      return bad("result.summary.scorers must be an object");
+    }
+    const summaryScorerNames = Object.keys(summary.scorers);
+    const items = result.items;
+    const summaryScorerSet = new Set(summaryScorerNames);
+    const uncoveredAcrossItems = /* @__PURE__ */ new Set();
+    for (const item of items) {
+      const itemScores = item?.scores;
+      if (itemScores && typeof itemScores === "object") {
+        for (const name of Object.keys(itemScores)) {
+          if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+        }
+      }
+    }
+    if (uncoveredAcrossItems.size > 0) {
+      return bad(
+        `item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
+      );
+    }
+    const trim = (v) => typeof v === "string" && v.trim() !== "" ? v.trim() : void 0;
+    const metadataObj = typeof result.metadata === "object" && result.metadata != null ? result.metadata : {};
+    const workflowsFromMeta = Array.isArray(metadataObj.workflows) ? metadataObj.workflows : [];
+    const primaryWorkflow = workflowsFromMeta.find((w) => typeof w === "string");
+    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? // Legacy fallback: pre-0.14 CLI artifacts had workflow at the top level.
+    trim(result.workflow) ?? "imported";
+    const id = (0, import_node_crypto.randomUUID)();
+    const timestamp = Date.now();
+    const imported = {
+      ...result,
+      id,
+      metadata: typeof result.metadata === "object" && result.metadata != null ? result.metadata : {}
+    };
+    await runtime.saveEvalResult({
+      id,
+      eval: evalName,
+      timestamp,
+      data: imported
+    });
+    return c.json({ ok: true, data: { id, eval: evalName, timestamp } });
   });
-  return app7;
+  return app6;
 }
 
 // src/server/routes/playground.ts
 var import_hono11 = require("hono");
 function createPlaygroundRoutes(connMgr) {
-  const app7 = new import_hono11.Hono();
-  app7.post("/playground/chat", async (c) => {
+  const app6 = new import_hono11.Hono();
+  app6.post("/playground/chat", async (c) => {
     const runtime = c.get("runtime");
     const body = await c.req.json();
     if (!body.message || typeof body.message !== "string" || !body.message.trim()) {
@@ -922,42 +1061,45 @@ function createPlaygroundRoutes(connMgr) {
       data: { sessionId, executionId, streaming: true }
     });
   });
-  return app7;
+  return app6;
 }
 
 // src/server/index.ts
 function createServer(options) {
   const { runtime, staticRoot, basePath = "", readOnly = false } = options;
-  const app7 = new import_hono12.Hono();
+  const app6 = new import_hono12.Hono();
   const connMgr = new ConnectionManager();
   const costAggregator = new CostAggregator(connMgr);
   if (options.cors !== false) {
-    app7.use("*", (0, import_cors.cors)());
+    app6.use("*", (0, import_cors.cors)());
   }
-  app7.use("*", errorHandler);
-  app7.use("*", async (c, next) => {
+  app6.use("*", errorHandler);
+  app6.use("*", async (c, next) => {
     c.set("runtime", runtime);
     await next();
   });
   if (readOnly) {
     const blocked = [
-      "POST /api/workflows",
-      "POST /api/executions",
-      "POST /api/sessions",
-      "DELETE /api/sessions",
-      "PUT /api/memory",
-      "DELETE /api/memory",
-      "POST /api/decisions",
-      "POST /api/costs",
-      "POST /api/tools",
-      "POST /api/evals",
-      "POST /api/playground"
+      /^POST \/api\/workflows(\/|$)/,
+      /^POST \/api\/executions(\/|$)/,
+      /^POST \/api\/sessions(\/|$)/,
+      /^DELETE \/api\/sessions(\/|$)/,
+      /^PUT \/api\/memory(\/|$)/,
+      /^DELETE \/api\/memory(\/|$)/,
+      /^POST \/api\/decisions(\/|$)/,
+      /^POST \/api\/costs(\/|$)/,
+      /^POST \/api\/tools(\/|$)/,
+      /^POST \/api\/evals\/import$/,
+      /^POST \/api\/evals\/[^/]+\/run$/,
+      /^POST \/api\/evals\/[^/]+\/rescore$/,
+      /^DELETE \/api\/evals\/history\/[^/]+$/,
+      /^POST \/api\/playground(\/|$)/
     ];
-    app7.use("/api/*", async (c, next) => {
+    app6.use("/api/*", async (c, next) => {
       const apiIdx = c.req.path.indexOf("/api/");
       const apiPath = apiIdx >= 0 ? c.req.path.slice(apiIdx) : c.req.path;
       const key = `${c.req.method} ${apiPath}`;
-      if (blocked.some((b) => key.startsWith(b))) {
+      if (blocked.some((re) => re.test(key))) {
         return c.json(
           {
             ok: false,
@@ -970,7 +1112,7 @@ function createServer(options) {
     });
   }
   const api = new import_hono12.Hono();
-  api.route("/", health_default);
+  api.route("/", createHealthRoutes(readOnly));
   api.route("/", createWorkflowRoutes(connMgr));
   api.route("/", executions_default);
   api.route("/", createSessionRoutes(connMgr));
@@ -981,7 +1123,7 @@ function createServer(options) {
   api.route("/", createCostRoutes(costAggregator));
   api.route("/", createEvalRoutes(options.evalLoader));
   api.route("/", createPlaygroundRoutes(connMgr));
-  app7.route("/api", api);
+  app6.route("/api", api);
   const traceListener = (event) => {
     const traceEvent = event;
     if (traceEvent.executionId) {
@@ -1022,7 +1164,7 @@ function createServer(options) {
       root: staticRoot,
       rewriteRequestPath: basePath ? (path) => path.startsWith(basePath) ? path.slice(basePath.length) || "/" : path : void 0
     });
-    app7.use("/*", async (c, next) => {
+    app6.use("/*", async (c, next) => {
       const reqPath = c.req.path;
       const resolved = basePath && reqPath.startsWith(basePath) ? reqPath.slice(basePath.length) || "/" : reqPath;
       if (resolved === "/" || resolved === "/index.html" || resolved === "/ws") {
@@ -1031,7 +1173,7 @@ function createServer(options) {
       return staticHandler(c, next);
     });
     if (spaHtml) {
-      app7.get("*", async (c, next) => {
+      app6.get("*", async (c, next) => {
         const resolved = basePath && c.req.path.startsWith(basePath) ? c.req.path.slice(basePath.length) || "/" : c.req.path;
         if (resolved === "/ws") return next();
         return c.html(spaHtml);
@@ -1039,7 +1181,7 @@ function createServer(options) {
     }
   }
   return {
-    app: app7,
+    app: app6,
     connMgr,
     costAggregator,
     /** Create WS handlers. Call before registering static/SPA routes are reached. */