@axlsdk/studio 0.13.5 → 0.13.7

package/dist/cli.cjs

@@ -63,11 +63,18 @@ async function errorHandler(c, next) {
 }
 
 // src/server/ws/connection-manager.ts
+function isBufferedChannel(channel) {
+  return channel.startsWith("execution:");
+}
+var BUFFER_TTL_MS = 3e4;
+var MAX_BUFFER_EVENTS = 500;
 var ConnectionManager = class {
   /** channel -> set of WS connections */
   channels = /* @__PURE__ */ new Map();
   /** ws -> set of subscribed channels (for cleanup) */
   connections = /* @__PURE__ */ new Map();
+  /** channel -> replay buffer for execution streams */
+  buffers = /* @__PURE__ */ new Map();
   maxConnections = 100;
   /** Register a new WS connection. */
   add(ws) {
@@ -90,7 +97,7 @@ var ConnectionManager = class {
     }
     this.connections.delete(ws);
   }
-  /** Subscribe a connection to a channel. No-op if the connection was not added. */
+  /** Subscribe a connection to a channel. Replays buffered events for execution channels. */
   subscribe(ws, channel) {
     if (!this.connections.has(ws)) return;
     let subs = this.channels.get(channel);
@@ -100,6 +107,17 @@ var ConnectionManager = class {
     }
     subs.add(ws);
     this.connections.get(ws).add(channel);
+    const buffer = this.buffers.get(channel);
+    if (buffer) {
+      for (const msg of buffer.events) {
+        try {
+          ws.send(msg);
+        } catch {
+          this.remove(ws);
+          return;
+        }
+      }
+    }
   }
   /** Unsubscribe a connection from a channel. */
   unsubscribe(ws, channel) {
@@ -109,11 +127,30 @@ var ConnectionManager = class {
     }
     this.connections.get(ws)?.delete(channel);
   }
-  /** Broadcast data to all subscribers of a channel. */
+  /** Broadcast data to all subscribers of a channel. Buffers events for execution channels. */
   broadcast(channel, data) {
+    const msg = JSON.stringify({ type: "event", channel, data });
+    if (isBufferedChannel(channel)) {
+      let buffer = this.buffers.get(channel);
+      if (!buffer) {
+        buffer = { events: [], complete: false };
+        this.buffers.set(channel, buffer);
+      }
+      const event = data;
+      const isTerminal = event.type === "done" || event.type === "error";
+      if (buffer.events.length < MAX_BUFFER_EVENTS || isTerminal) {
+        buffer.events.push(msg);
+      }
+      if (isTerminal) {
+        buffer.complete = true;
+        if (buffer.timer) clearTimeout(buffer.timer);
+        buffer.timer = setTimeout(() => {
+          this.buffers.delete(channel);
+        }, BUFFER_TTL_MS);
+      }
+    }
     const subs = this.channels.get(channel);
     if (!subs || subs.size === 0) return;
-    const msg = JSON.stringify({ type: "event", channel, data });
     for (const ws of [...subs]) {
       try {
         ws.send(msg);
@@ -140,13 +177,17 @@ var ConnectionManager = class {
       }
     }
   }
-  /** Close all connections and clear all state. Used during shutdown. */
+  /** Close all connections, clear all state and buffers. Used during shutdown. */
   closeAll() {
     for (const ws of this.connections.keys()) {
       ws.close?.();
     }
+    for (const buffer of this.buffers.values()) {
+      if (buffer.timer) clearTimeout(buffer.timer);
+    }
     this.connections.clear();
     this.channels.clear();
+    this.buffers.clear();
   }
   /** Get the number of active connections. */
   get connectionCount() {
@@ -351,15 +392,8 @@ function createWorkflowRoutes(connMgr) {
       const stream = runtime.stream(name, body.input ?? {}, { metadata: body.metadata });
       const executionId = `stream-${Date.now()}`;
       (async () => {
-        try {
-          for await (const event of stream) {
-            connMgr.broadcastWithWildcard(`execution:${executionId}`, event);
-          }
-        } catch (err) {
-          connMgr.broadcastWithWildcard(`execution:${executionId}`, {
-            type: "error",
-            message: err instanceof Error ? err.message : "Stream error"
-          });
+        for await (const event of stream) {
+          connMgr.broadcastWithWildcard(`execution:${executionId}`, event);
         }
       })();
       return c.json({ ok: true, data: { executionId, streaming: true } });
@@ -440,15 +474,8 @@ function createSessionRoutes(connMgr) {
     const stream = await session.stream(body.workflow, body.message);
     const executionId = `session-${id}-${Date.now()}`;
     (async () => {
-      try {
-        for await (const event of stream) {
-          connMgr.broadcastWithWildcard(`execution:${executionId}`, event);
-        }
-      } catch (err) {
-        connMgr.broadcastWithWildcard(`execution:${executionId}`, {
-          type: "error",
-          message: err instanceof Error ? err.message : "Stream error"
-        });
+      for await (const event of stream) {
+        connMgr.broadcastWithWildcard(`execution:${executionId}`, event);
       }
     })();
     return c.json({ ok: true, data: { executionId, streaming: true } });
@@ -730,8 +757,80 @@ function createEvalRoutes(evalLoader) {
         404
       );
     }
+    let runs = 1;
+    try {
+      const body = await c.req.json().catch(() => ({}));
+      if (typeof body.runs === "number" && Number.isFinite(body.runs) && body.runs > 1) {
+        runs = Math.min(Math.floor(body.runs), 25);
+      }
+    } catch {
+    }
+    try {
+      if (runs > 1) {
+        const { randomUUID } = await import("crypto");
+        const { aggregateRuns } = await import("@axlsdk/eval");
+        const runGroupId = randomUUID();
+        const results = [];
+        for (let r = 0; r < runs; r++) {
+          const result2 = await runtime.runRegisteredEval(name, {
+            metadata: { runGroupId, runIndex: r }
+          });
+          results.push(result2);
+        }
+        const typedResults = results;
+        const aggregate = aggregateRuns(typedResults);
+        const first = typedResults[0];
+        const result = { ...first, _multiRun: { aggregate, allRuns: typedResults } };
+        return c.json({ ok: true, data: result });
+      } else {
+        const result = await runtime.runRegisteredEval(name);
+        return c.json({ ok: true, data: result });
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
+    }
+  });
+  app7.post("/evals/:name/rescore", async (c) => {
+    if (evalLoader) await evalLoader();
+    const runtime = c.get("runtime");
+    const name = c.req.param("name");
+    const body = await c.req.json();
+    if (!body.resultId || typeof body.resultId !== "string") {
+      return c.json(
+        { ok: false, error: { code: "BAD_REQUEST", message: "resultId is required" } },
+        400
+      );
+    }
+    const entry = runtime.getRegisteredEval(name);
+    if (!entry) {
+      return c.json(
+        { ok: false, error: { code: "NOT_FOUND", message: `Eval "${name}" not found` } },
+        404
+      );
+    }
+    const history = await runtime.getEvalHistory();
+    const historyEntry = history.find((h) => h.id === body.resultId);
+    if (!historyEntry) {
+      return c.json(
+        { ok: false, error: { code: "NOT_FOUND", message: `Result "${body.resultId}" not found` } },
+        404
+      );
+    }
     try {
-      const result = await runtime.runRegisteredEval(name);
+      const { rescore } = await import("@axlsdk/eval");
+      const config = entry.config;
+      const result = await rescore(
+        historyEntry.data,
+        config.scorers,
+        runtime
+      );
+      await runtime.saveEvalResult({
+        id: result.id,
+        eval: name,
+        timestamp: Date.now(),
+        data: result
+      });
       return c.json({ ok: true, data: result });
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
@@ -742,7 +841,7 @@ function createEvalRoutes(evalLoader) {
     const runtime = c.get("runtime");
     const body = await c.req.json();
     try {
-      const result = await runtime.evalCompare(body.baseline, body.candidate);
+      const result = await runtime.evalCompare(body.baseline, body.candidate, body.options);
       return c.json({ ok: true, data: result });
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
@@ -759,26 +858,57 @@ function createPlaygroundRoutes(connMgr) {
   app7.post("/playground/chat", async (c) => {
     const runtime = c.get("runtime");
     const body = await c.req.json();
-    const workflowName = body.workflow ?? runtime.getWorkflowNames()[0];
-    if (!workflowName) {
+    if (!body.message || typeof body.message !== "string" || !body.message.trim()) {
+      return c.json(
+        {
+          ok: false,
+          error: {
+            code: "INVALID_INPUT",
+            message: "message is required and must be a non-empty string"
+          }
+        },
+        400
+      );
+    }
+    const agents = runtime.getAgents();
+    const agent = body.agent ? agents.find((a) => a._name === body.agent) : agents[0];
+    if (!agent) {
       return c.json(
-        { ok: false, error: { code: "NO_WORKFLOW", message: "No workflows registered" } },
+        {
+          ok: false,
+          error: { code: "NO_AGENT", message: `Agent "${body.agent ?? ""}" not found` }
+        },
         400
       );
     }
     const sessionId = body.sessionId ?? `playground-${Date.now()}`;
-    const session = runtime.session(sessionId);
-    const stream = await session.stream(workflowName, body.message);
     const executionId = `playground-${sessionId}-${Date.now()}`;
+    const store = runtime.getStateStore();
+    const history = await store.getSession(sessionId);
+    history.push({ role: "user", content: body.message });
+    const ctx = runtime.createContext({
+      sessionHistory: history,
+      onToken: (token) => {
+        connMgr.broadcastWithWildcard(`execution:${executionId}`, {
+          type: "token",
+          data: token
+        });
+      }
+    });
     (async () => {
       try {
-        for await (const event of stream) {
-          connMgr.broadcastWithWildcard(`execution:${executionId}`, event);
-        }
+        const result = await ctx.ask(agent, body.message);
+        const resultText = typeof result === "string" ? result : JSON.stringify(result);
+        history.push({ role: "assistant", content: resultText });
+        await store.saveSession(sessionId, history);
+        connMgr.broadcastWithWildcard(`execution:${executionId}`, {
+          type: "done",
+          data: resultText
+        });
       } catch (err) {
         connMgr.broadcastWithWildcard(`execution:${executionId}`, {
           type: "error",
-          message: err instanceof Error ? err.message : "Stream error"
+          message: err instanceof Error ? err.message : String(err)
         });
       }
     })();
@@ -890,19 +1020,24 @@ function createServer(options) {
     app7.use("/*", async (c, next) => {
       const reqPath = c.req.path;
       const resolved = basePath && reqPath.startsWith(basePath) ? reqPath.slice(basePath.length) || "/" : reqPath;
-      if (resolved === "/" || resolved === "/index.html") {
+      if (resolved === "/" || resolved === "/index.html" || resolved === "/ws") {
         return next();
       }
       return staticHandler(c, next);
     });
     if (spaHtml) {
-      app7.get("*", (c) => c.html(spaHtml));
+      app7.get("*", async (c, next) => {
+        const resolved = basePath && c.req.path.startsWith(basePath) ? c.req.path.slice(basePath.length) || "/" : c.req.path;
+        if (resolved === "/ws") return next();
+        return c.html(spaHtml);
+      });
     }
   }
   return {
     app: app7,
     connMgr,
     costAggregator,
+    /** Create WS handlers. Call before registering static/SPA routes are reached. */
     createWsHandlers: () => createWsHandlers(connMgr),
     traceListener
   };