@axlsdk/studio 0.19.1 → 0.20.1

package/README.md

@@ -89,7 +89,9 @@ The `--conditions` flag is useful in monorepos where workspace packages use the
 
 ## Panels
 
-Eight panels, all live over WebSocket. The screenshots below are placeholders — see [`docs/assets/CAPTURE.md`](../../docs/assets/CAPTURE.md) for how to record the real thing.
+Eight panels, all live over WebSocket. The captures below use representative seeded
+runs; see [`docs/assets/CAPTURE.md`](../../docs/assets/CAPTURE.md) for how to regenerate
+them.
 
 ### Agent Playground
 
@@ -101,7 +103,12 @@ Chat with any registered agent in real time. Streaming tokens, tool calls with e
 
 ### Trace Explorer
 
-Waterfall visualization of execution traces. Filter by type, agent, or tool; see token counts, cost per step, and duration. The Stats view shows the event-type distribution, top tools, and retry stacks.
+Waterfall visualization of execution traces. Filter by type, agent, or tool; see token
+counts, cost per step, and duration. Tool rows distinguish calls that succeeded,
+failed, were denied, were cancelled, or were rejected before they started. Studio also
+keeps older trace history readable and identifies incomplete runs instead of presenting
+missing events as a complete story. The Stats view shows the event-type distribution,
+top tools, and retry stacks.
 
 > **Studio vs `ctx.events`.** Studio consumes the same `AxlEvent` firehose via `runtime.on('trace', …)` — every event from every execution. Inside a workflow handler, `ctx.events` is the in-handler counterpart (per-context, scoped to the current workflow). The two coexist: Studio is for cross-execution observability and replay; `ctx.events` is for in-handler streaming UIs. See [`docs/observability.md`](../../docs/observability.md#observation-paths).
 
@@ -115,7 +122,14 @@ Track spending across agents, models, workflows, and embedders with time-window
 
 ### Eval Runner
 
-Run evaluations from the UI, watch items stream in, and drill into per-item scores, timing, cost, and LLM-judge reasoning. Compare two runs (baseline vs candidate) with bootstrap-CI significance, a score-distribution chart, and an item-level diff table. The History tab groups multi-run results and tracks mean scores across runs. Toggle **Capture traces** to render each item's events inline. Requires `@axlsdk/eval` as an optional peer dependency.
+Run evaluations from the UI, watch items stream in, and drill into per-item scores,
+timing, cost, and LLM-judge reasoning. Compare two runs (baseline vs candidate) with
+bootstrap-CI significance, a score-distribution chart, and an item-level diff table.
+The History tab groups multi-run results and tracks mean scores across runs. Toggle
+**Capture traces** to render each item's events inline. For tool-using workflows, those
+events show whether a call failed, was denied, was cancelled, or never started, giving
+you better evidence for debugging and custom scorers. Studio does not turn those states
+into scores automatically. Requires `@axlsdk/eval` as an optional peer dependency.
 
 Amber banners flag runs you shouldn't fully trust — a scorer that failed on too many items, a subset run, or annotations dropped by the dataset schema — so a thinned or misconfigured run can't quietly look clean.
 

package/dist/{chunk-AYUKXMYW.js → chunk-KQ3HBRTC.js}

@@ -12,15 +12,14 @@ import { cors } from "hono/cors";
 import { serveStatic } from "@hono/node-server/serve-static";
 
 // src/server/redact.ts
-import { redactEvent } from "@axlsdk/axl";
+import { redactHistoricalEvent } from "@axlsdk/axl";
 var REDACTED = "[redacted]";
 var SAFE_ERROR_NAMES = /* @__PURE__ */ new Set([
   "QuorumNotMet",
   "NoConsensus",
   "TimeoutError",
   "MaxTurnsError",
-  "BudgetExceededError",
-  "ToolDenied"
+  "BudgetExceededError"
 ]);
 function redactErrorMessage(err, redact) {
   const raw = err instanceof Error ? err.message : String(err);
@@ -88,7 +87,7 @@ function redactSessionHistory(history, redact) {
 }
 function redactStreamEvent(event, redact) {
   if (!redact) return event;
-  return redactEvent(event);
+  return redactHistoricalEvent(event);
 }
 function redactEvalItem(item) {
   const scrubbed = {
@@ -151,31 +150,29 @@ function redactPendingDecisionList(decisions, redact) {
 }
 
 // src/server/middleware/error-handler.ts
-async function errorHandler(c, next) {
-  try {
-    await next();
-  } catch (err) {
-    const rawMessage = err instanceof Error ? err.message : String(err);
-    const code = err.code ?? "INTERNAL_ERROR";
-    let status = 500;
-    if ("status" in err) {
-      const errStatus = err.status;
-      if (typeof errStatus === "number" && errStatus >= 400 && errStatus < 600) {
-        status = errStatus;
-      }
-    } else if (code === "NOT_FOUND" || rawMessage.includes("not found") || rawMessage.includes("not registered")) {
-      status = 404;
-    } else if (code === "VALIDATION_ERROR" || rawMessage.includes("Expected") || rawMessage.includes("invalid")) {
-      status = 400;
-    }
-    const runtime = c.get("runtime");
-    const redactOn = runtime?.isRedactEnabled?.() ?? false;
-    const body = {
-      ok: false,
-      error: { code, message: redactErrorMessage(err, redactOn) }
-    };
-    return c.json(body, status);
+function errorHandler(err, c) {
+  const rawMessage = err.message;
+  const code = err.code ?? "INTERNAL_ERROR";
+  let status = 500;
+  if ("status" in err) {
+    const errStatus = err.status;
+    if (typeof errStatus === "number" && errStatus >= 400 && errStatus < 600) {
+      status = errStatus;
+    }
+  } else if (code === "CROSS_PROCESS_RESUME_UNSUPPORTED") {
+    status = 409;
+  } else if (code === "NOT_FOUND" || code === "PENDING_DECISION_NOT_FOUND" || rawMessage.includes("not found") || rawMessage.includes("not registered")) {
+    status = 404;
+  } else if (code === "VALIDATION_ERROR" || code === "INVALID_HUMAN_DECISION" || rawMessage.includes("Expected") || rawMessage.includes("invalid")) {
+    status = 400;
   }
+  const runtime = c.get("runtime");
+  const redactOn = runtime?.isRedactEnabled?.() ?? false;
+  const body = {
+    ok: false,
+    error: { code, message: redactErrorMessage(err, redactOn) }
+  };
+  return c.json(body, status);
 }
 
 // src/server/ws/connection-manager.ts
@@ -1091,19 +1088,58 @@ function reduceTraceStats(acc, event) {
   const eventTypeCounts = { ...acc.eventTypeCounts };
   eventTypeCounts[event.type] = (eventTypeCounts[event.type] ?? 0) + 1;
   const byTool = { ...acc.byTool };
-  if (event.type === "tool_call_end" || event.type === "tool_denied" || event.type === "tool_approval") {
+  if (event.type === "tool_call_start" || event.type === "tool_call_end" || event.type === "tool_call_rejected" || event.type === "tool_denied" || event.type === "tool_approval") {
     const toolName = event.tool;
-    const prev = byTool[toolName] ?? { calls: 0, denied: 0, approved: 0 };
-    const isDeniedEvent = event.type === "tool_denied";
-    const isApprovalEvent = event.type === "tool_approval";
-    const eventData = isDeniedEvent || isApprovalEvent ? event.data : void 0;
-    const isApproved = isDeniedEvent && eventData?.approved === true || isApprovalEvent && eventData?.approved === true;
-    const isDenied = isDeniedEvent && !eventData?.approved || isApprovalEvent && eventData?.approved === false;
-    byTool[toolName] = {
-      calls: prev.calls + (event.type === "tool_call_end" ? 1 : 0),
-      denied: prev.denied + (isDenied ? 1 : 0),
-      approved: prev.approved + (isApproved ? 1 : 0)
+    const prev = byTool[toolName] ?? {
+      accepted: 0,
+      succeeded: 0,
+      failed: 0,
+      failedByPhase: {},
+      denied: 0,
+      cancelled: 0,
+      rejected: 0,
+      approved: 0,
+      legacy: { calls: 0, denied: 0, approved: 0 }
+    };
+    const next = {
+      ...prev,
+      failedByPhase: { ...prev.failedByPhase },
+      legacy: { ...prev.legacy }
     };
+    if (event.schemaVersion === 2) {
+      if (event.type === "tool_call_start") next.accepted++;
+      if (event.type === "tool_call_rejected") next.rejected++;
+      if (event.type === "tool_approval" && event.data?.approved) next.approved++;
+      if (event.type === "tool_call_end") {
+        switch (event.data.outcome.status) {
+          case "succeeded":
+            next.succeeded++;
+            break;
+          case "failed": {
+            next.failed++;
+            const phase = event.data.outcome.failure.phase;
+            next.failedByPhase[phase] = (next.failedByPhase[phase] ?? 0) + 1;
+            break;
+          }
+          case "denied":
+            next.denied++;
+            break;
+          case "cancelled":
+            next.cancelled++;
+            break;
+        }
+      }
+    } else if (event.type === "tool_call_end") {
+      next.legacy.calls++;
+    } else if (event.type === "tool_denied") {
+      const approved = event.data?.approved === true;
+      if (approved) next.legacy.approved++;
+      else next.legacy.denied++;
+    } else if (event.type === "tool_approval") {
+      if (event.data?.approved === true) next.legacy.approved++;
+      else if (event.data?.approved === false) next.legacy.denied++;
+    }
+    byTool[toolName] = next;
   }
   const retryByAgent = { ...acc.retryByAgent };
   if (event.agent && event.type === "agent_call_end") {
@@ -1583,7 +1619,14 @@ app4.get("/decisions", async (c) => {
 app4.post("/decisions/:executionId/resolve", async (c) => {
   const runtime = c.get("runtime");
   const executionId = c.req.param("executionId");
-  const body = await c.req.json();
+  let body;
+  try {
+    body = await c.req.json();
+  } catch {
+    const error48 = new Error("Invalid JSON request body");
+    Object.assign(error48, { code: "INVALID_HUMAN_DECISION", status: 400 });
+    throw error48;
+  }
   await runtime.resolveDecision(executionId, body);
   return c.json({ ok: true, data: { resolved: true } });
 });
@@ -15923,6 +15966,7 @@ function createPlaygroundRoutes(connMgr) {
     (async () => {
       let stepCounter = Number.MAX_SAFE_INTEGER - 1;
       const terminalFields = () => ({
+        schemaVersion: 2,
         executionId,
         step: stepCounter++,
         timestamp: Date.now()
@@ -16039,7 +16083,7 @@ function createServer(options) {
   if (options.cors !== false) {
     app5.use("*", cors());
   }
-  app5.use("*", errorHandler);
+  app5.onError(errorHandler);
   app5.use("*", async (c, next) => {
     c.set("runtime", runtime);
     await next();
@@ -16195,4 +16239,4 @@ export {
   EvalAggregator,
   createServer
 };
-//# sourceMappingURL=chunk-AYUKXMYW.js.map
+//# sourceMappingURL=chunk-KQ3HBRTC.js.map