@axlsdk/eval 0.17.1 → 0.17.3

package/README.md

@@ -355,6 +355,48 @@ for (const item of results.items) {
 
 `captureTraces` strips verbose-mode `agent_call_end.data.messages` snapshots and high-volume events (`token`, `partial_object`) from the captured array to keep memory bounded. The structural events you'd want for debugging — `agent_call_start`/`agent_call_end`, `tool_call_*`, gate events, `pipeline`, `verify`, `handoff_*` — are all retained.
 
+#### Per-item live observation with `ctx.events`
+
+`captureTraces` collects events for post-hoc inspection on `EvalItem.traces`. For **live** per-item observation — e.g., printing each item's `partial_object` snapshots as they stream during eval execution — use `ctx.events` inside your top-level `executeWorkflow` export. The two are complementary: `captureTraces` records the structural timeline; `ctx.events` is the live firehose.
+
+```typescript
+// evals/extract.eval.ts
+import { defineEval, dataset, scorer } from '@axlsdk/eval';
+import type { AxlRuntime } from '@axlsdk/axl';
+import { z } from 'zod';
+import { extractor } from '../src/agents/extractor.js'; // your agent
+
+const extractSchema = z.object({ id: z.string(), summary: z.string() });
+
+export default defineEval({
+  workflow: 'extract-fields',
+  dataset: dataset({ items: [{ id: 'a', text: 'hello world' }] }),
+  scorers: [scorer({ name: 'has-id', score: ({ output }) => (output.id ? 1 : 0) })],
+});
+
+// Top-level export, NOT nested inside defineEval. The eval runner
+// imports both the default-exported config AND this named function.
+export async function executeWorkflow(
+  input: { id: string; text: string },
+  runtime: AxlRuntime,
+) {
+  const ctx = runtime.createContext({ signal: AbortSignal.timeout(30_000) });
+  // Allocate the bus before the first ctx.ask() — the streaming gate
+  // is per-ask, and a late subscription leaves the in-flight ask
+  // through the non-streaming code path.
+  const events = ctx.events;
+  void (async () => {
+    for await (const partial of events.partialObjects) {
+      console.log(`[${input.id}] attempt ${partial.attempt}:`, partial.object);
+    }
+  })().catch((err) => console.error('observer failed:', err));
+  const output = await ctx.ask(extractor, input.text, { schema: extractSchema });
+  return { output };
+}
+```
+
+The `signal` from `createContext` auto-disposes the bus on timeout, so the iterator terminates cleanly even if the eval item never emits a workflow terminal. See [`ctx.events`](../../docs/api-reference.md#ctxevents) for the full type reference.
+
 ## Comparing Results
 
 Compare two runs to detect regressions and improvements. Runs must use the same dataset and scorers.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.17.1",
+  "version": "0.17.3",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.17.1"
+    "@axlsdk/axl": "0.17.3"
   },
   "peerDependencies": {
     "zod": "^4.0.0",