@ariaflowagents/core 0.8.1 → 0.9.1

package/dist/capabilities/index.d.ts

@@ -0,0 +1,154 @@
+import type { ZodTypeAny } from 'zod';
+/**
+ * Backend-agnostic tool declaration. Produced by capabilities,
+ * consumed by adapters (Gemini, AI SDK, LiveKit).
+ */
+export interface ToolDeclaration {
+    name: string;
+    description: string;
+    parameters: ZodTypeAny;
+    execute: (args: any, options?: unknown) => Promise<unknown>;
+}
+export interface PromptSection {
+    /**
+     * Semantic role that determines ordering in the final prompt.
+     * Built-in roles: 'role' > 'task' > 'state' > 'context' > 'extraction' > 'routing' > 'policy'.
+     * Custom roles sort after built-in ones.
+     */
+    role: string;
+    content: string;
+}
+/**
+ * Structured payload for the realtime model when an extraction submit
+ * advances collection but does not yet complete the node.
+ */
+export interface ExtractionToolResponseEnvelope {
+    accepted: true;
+    fieldsReceived: string[];
+    fieldsStillRequired: string[];
+    instruction: string;
+}
+/** Optional metadata when the flow transitions to a new node (reconfigure). */
+export interface FlowReconfigureTransition {
+    from: string;
+    to: string;
+    /** Extraction submit tool that triggered transition to `to`, when applicable. */
+    triggerTool?: string;
+}
+/**
+ * Returned by `Capability.processToolResult()` to tell the host what to do.
+ */
+export type CapabilityAction = {
+    type: 'continue';
+    toolResponseOverride?: ExtractionToolResponseEnvelope;
+} | {
+    type: 'reconfigure';
+    transition?: FlowReconfigureTransition;
+} | {
+    type: 'handoff';
+    targetAgent: string;
+    reason?: string;
+} | {
+    type: 'end';
+    reason?: string;
+} | {
+    type: 'extraction-complete';
+    data: Record<string, unknown>;
+};
+/**
+ * A Capability is a self-contained unit of agent behavior that exposes
+ * its features as tools and prompt sections. Any LLM backend that supports
+ * tool calling can drive it.
+ *
+ * The Capability interface follows the Component pattern: each feature
+ * (flows, triage, extraction, handoffs, guardrails) is a capability that
+ * plugs into a CapabilityHost. The host collects tools, builds prompts,
+ * and routes tool results — the LLM backend is irrelevant.
+ */
+export interface Capability {
+    /** What tools does this capability expose for the current state? */
+    getTools(): ToolDeclaration[];
+    /** What does this capability contribute to the system prompt? */
+    getPromptSections(): PromptSection[];
+    /**
+     * A tool was called and executed. Does this capability handle the result?
+     *
+     * @param toolName - The name of the tool that was called
+     * @param args - The arguments passed to the tool
+     * @param result - The result returned by the tool's execute()
+     * @returns An action telling the host what to do, or null if this
+     *          capability doesn't handle this tool.
+     */
+    processToolResult(toolName: string, args: unknown, result: unknown): CapabilityAction | null;
+}
+/**
+ * Collects capabilities and exposes a unified interface for any LLM backend.
+ *
+ * Usage:
+ * ```typescript
+ * const host = new CapabilityHost()
+ *   .use(new FlowCapability({ flow, initialNode: 'greeting' }))
+ *   .use(new ExtractionCapability({ schema }))
+ *   .use(new GuardrailCapability(policies));
+ *
+ * // For Gemini Live:
+ * const tools = toGeminiDeclarations(host.getAllTools());
+ * const prompt = host.getSystemPrompt(basePrompt);
+ *
+ * // For AI SDK:
+ * const tools = toAISDKTools(host.getAllTools());
+ * streamText({ system: prompt, tools });
+ * ```
+ */
+export declare class CapabilityHost {
+    private capabilities;
+    private regularTools;
+    private _version;
+    private _lastCheckedVersion;
+    /** Add a capability (flow, triage, extraction, etc.). */
+    use(capability: Capability): this;
+    /** Add regular tools not managed by any capability. */
+    addTools(tools: ToolDeclaration[]): this;
+    /**
+     * Collect all tools from all capabilities + regular tools.
+     *
+     * Deduplicates by name — capability-provided tools take priority over
+     * regular (agent-level) tools. This prevents duplicate function
+     * declarations that providers like Gemini Live reject.
+     */
+    getAllTools(): ToolDeclaration[];
+    /**
+     * Build complete system prompt from base prompt + all capability sections.
+     * Sections are ordered by role priority.
+     */
+    getSystemPrompt(basePrompt?: string): string;
+    /**
+     * Route a tool result through capabilities. First capability to claim it wins.
+     * If no capability claims it, returns `{ type: 'continue' }` (regular tool).
+     */
+    processToolResult(toolName: string, args: unknown, result: unknown): CapabilityAction;
+    /** Mark the current state as "seen" for reconfigure detection. */
+    markConfigured(): void;
+    /** Bump version to signal that tools or prompt have changed. */
+    notifyChanged(): void;
+    /** True if tools or prompt changed since last `markConfigured()` call. */
+    get needsReconfigure(): boolean;
+    /** Number of registered capabilities. */
+    get capabilityCount(): number;
+}
+export { FlowCapability } from './FlowCapability.js';
+export type { FlowCapabilityConfig, FlowCapabilityState, FlowEvent } from './FlowCapability.js';
+export { TriageCapability } from './TriageCapability.js';
+export { ExtractionCapability } from './ExtractionCapability.js';
+export type { ExtractionCapabilityConfig } from './ExtractionCapability.js';
+export { HandoffCapability } from './HandoffCapability.js';
+export type { HandoffTarget } from './HandoffCapability.js';
+export { GuardrailCapability } from './GuardrailCapability.js';
+export { AutoRetrieveCapability } from './AutoRetrieveCapability.js';
+export type { AutoRetrieveCapabilityConfig, RetrieveProvider } from './AutoRetrieveCapability.js';
+export { toGeminiDeclarations } from './adapters/gemini.js';
+export type { GeminiFunctionDeclaration } from './adapters/gemini.js';
+export { toAISDKTools } from './adapters/ai-sdk.js';
+export { DefaultLivePromptAssembler, DEFAULT_VOICE_RULES, DEFAULT_GUARDRAILS } from './LivePromptAssembler.js';
+export type { LivePromptAssembler, LivePromptContext, DefaultLivePromptAssemblerConfig, } from './LivePromptAssembler.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/capabilities/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/capabilities/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,KAAK,CAAC;AAItC;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7D;AAeD,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAID;;;GAGG;AACH,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,IAAI,CAAC;IACf,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,+EAA+E;AAC/E,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GACxB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,oBAAoB,CAAC,EAAE,8BAA8B,CAAA;CAAE,GAC3E;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,CAAC,EAAE,yBAAyB,CAAA;CAAE,GAC/D;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAAC;AAInE;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,oEAAoE;IACpE,QAAQ,IAAI,eAAe,EAAE,CAAC;IAE9B,iEAAiE;IACjE,iBAAiB,IAAI,aAAa,EAAE,CAAC;IAErC;;;;;;;;OAQG;IACH,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC;CAC9F;AAID;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,yDAAyD;IACzD,GAAG,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMjC,uDAAuD;IACvD,QAAQ,CAAC,KAAK,EAAE,eAAe,EAAE,GAAG,IAAI;IAMxC;;;;;;OAMG;IACH,WAAW,IAAI,eAAe,EAAE;IAOhC;;;OAGG;IACH,eAAe,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM;IAK5C;;;OAGG;IACH,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,gBAAgB;IAQrF,kEAAkE;IAClE,cAAc,IAAI,IAAI;IAItB,gEAAgE;IAChE,aAAa,IAAI,IAAI;IAIrB,0EAA0E;IAC1E,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,yCAAyC;IACzC,IAAI,eAAe,IAAI,MAAM,CAE5B;CACF;AA6BD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChG,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,YAAY,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,YAAY,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,YAAY,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAClG,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,YAAY,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC/G,YAAY,EACV,mBAAmB,EACnB,iBAAiB,EACjB,gCAAgC,GACjC,MAAM,0BAA0B,CAAC"}

package/dist/capabilities/index.js

@@ -0,0 +1,128 @@
+// ─── Prompt Section ──────────────────────────────────────────────────────────
+/** Ordering priority for prompt sections (lower = earlier in prompt). */
+const SECTION_ORDER = {
+    role: 0,
+    task: 1,
+    state: 2,
+    context: 3,
+    extraction: 4,
+    routing: 5,
+    policy: 6,
+};
+// ─── CapabilityHost ──────────────────────────────────────────────────────────
+/**
+ * Collects capabilities and exposes a unified interface for any LLM backend.
+ *
+ * Usage:
+ * ```typescript
+ * const host = new CapabilityHost()
+ *   .use(new FlowCapability({ flow, initialNode: 'greeting' }))
+ *   .use(new ExtractionCapability({ schema }))
+ *   .use(new GuardrailCapability(policies));
+ *
+ * // For Gemini Live:
+ * const tools = toGeminiDeclarations(host.getAllTools());
+ * const prompt = host.getSystemPrompt(basePrompt);
+ *
+ * // For AI SDK:
+ * const tools = toAISDKTools(host.getAllTools());
+ * streamText({ system: prompt, tools });
+ * ```
+ */
+export class CapabilityHost {
+    capabilities = [];
+    regularTools = [];
+    _version = 0;
+    _lastCheckedVersion = 0;
+    /** Add a capability (flow, triage, extraction, etc.). */
+    use(capability) {
+        this.capabilities.push(capability);
+        this._version++;
+        return this;
+    }
+    /** Add regular tools not managed by any capability. */
+    addTools(tools) {
+        this.regularTools.push(...tools);
+        this._version++;
+        return this;
+    }
+    /**
+     * Collect all tools from all capabilities + regular tools.
+     *
+     * Deduplicates by name — capability-provided tools take priority over
+     * regular (agent-level) tools. This prevents duplicate function
+     * declarations that providers like Gemini Live reject.
+     */
+    getAllTools() {
+        const capabilityTools = this.capabilities.flatMap(c => c.getTools());
+        const seen = new Set(capabilityTools.map(t => t.name));
+        const dedupedRegular = this.regularTools.filter(t => !seen.has(t.name));
+        return [...capabilityTools, ...dedupedRegular];
+    }
+    /**
+     * Build complete system prompt from base prompt + all capability sections.
+     * Sections are ordered by role priority.
+     */
+    getSystemPrompt(basePrompt) {
+        const sections = this.capabilities.flatMap(c => c.getPromptSections());
+        return assemblePromptSections(basePrompt, sections);
+    }
+    /**
+     * Route a tool result through capabilities. First capability to claim it wins.
+     * If no capability claims it, returns `{ type: 'continue' }` (regular tool).
+     */
+    processToolResult(toolName, args, result) {
+        for (const cap of this.capabilities) {
+            const action = cap.processToolResult(toolName, args, result);
+            if (action)
+                return action;
+        }
+        return { type: 'continue' };
+    }
+    /** Mark the current state as "seen" for reconfigure detection. */
+    markConfigured() {
+        this._lastCheckedVersion = this._version;
+    }
+    /** Bump version to signal that tools or prompt have changed. */
+    notifyChanged() {
+        this._version++;
+    }
+    /** True if tools or prompt changed since last `markConfigured()` call. */
+    get needsReconfigure() {
+        return this._version !== this._lastCheckedVersion;
+    }
+    /** Number of registered capabilities. */
+    get capabilityCount() {
+        return this.capabilities.length;
+    }
+}
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+function assemblePromptSections(basePrompt, sections) {
+    const parts = [];
+    if (basePrompt) {
+        parts.push(basePrompt);
+    }
+    // Sort sections by role priority
+    const sorted = [...sections].sort((a, b) => {
+        const orderA = SECTION_ORDER[a.role] ?? 100;
+        const orderB = SECTION_ORDER[b.role] ?? 100;
+        return orderA - orderB;
+    });
+    for (const section of sorted) {
+        if (section.content.trim()) {
+            parts.push(section.content);
+        }
+    }
+    return parts.join('\n\n');
+}
+// ─── Re-exports ──────────────────────────────────────────────────────────────
+export { FlowCapability } from './FlowCapability.js';
+export { TriageCapability } from './TriageCapability.js';
+export { ExtractionCapability } from './ExtractionCapability.js';
+export { HandoffCapability } from './HandoffCapability.js';
+export { GuardrailCapability } from './GuardrailCapability.js';
+export { AutoRetrieveCapability } from './AutoRetrieveCapability.js';
+export { toGeminiDeclarations } from './adapters/gemini.js';
+export { toAISDKTools } from './adapters/ai-sdk.js';
+export { DefaultLivePromptAssembler, DEFAULT_VOICE_RULES, DEFAULT_GUARDRAILS } from './LivePromptAssembler.js';
+//# sourceMappingURL=index.js.map

package/dist/capabilities/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/capabilities/index.ts"],"names":[],"mappings":"AAeA,gFAAgF;AAEhF,yEAAyE;AACzE,MAAM,aAAa,GAA2B;IAC5C,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,OAAO,EAAE,CAAC;IACV,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;CACV,CAAC;AA0EF,gFAAgF;AAEhF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,cAAc;IACjB,YAAY,GAAiB,EAAE,CAAC;IAChC,YAAY,GAAsB,EAAE,CAAC;IACrC,QAAQ,GAAG,CAAC,CAAC;IACb,mBAAmB,GAAG,CAAC,CAAC;IAEhC,yDAAyD;IACzD,GAAG,CAAC,UAAsB;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uDAAuD;IACvD,QAAQ,CAAC,KAAwB;QAC/B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACH,WAAW;QACT,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,eAAe,EAAE,GAAG,cAAc,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,UAAmB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC;QACvE,OAAO,sBAAsB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;IAED;;;OAGG;IACH,iBAAiB,CAAC,QAAgB,EAAE,IAAa,EAAE,MAAe;QAChE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,GAAG,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC5B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9B,CAAC;IAED,kEAAkE;IAClE,cAAc;QACZ,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC3C,CAAC;IAED,gEAAgE;IAChE,aAAa;QACX,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;IAED,0EAA0E;IAC1E,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,mBAAmB,CAAC;IACpD,CAAC;IAED,yCAAyC;IACzC,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IAClC,CAAC;CACF;AAED,gFAAgF;AAEhF,SAAS,sBAAsB,CAAC,UAA8B,EAAE,QAAyB;IACvF,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzB,CAAC;IAED,iCAAiC;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzC,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QAC5C,OAAO,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,gFAAgF;AAEhF,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAErE,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAE5D,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/eval/EvalRunner.d.ts

@@ -0,0 +1,12 @@
+import type { LanguageModel } from 'ai';
+import type { HarnessHooks } from '../types/index.js';
+import type { EvalScenario, ScenarioScore } from './types.js';
+export declare class EvalRunner {
+    private config;
+    constructor(config: {
+        model: LanguageModel;
+        hooks?: HarnessHooks;
+    });
+    runText(scenario: EvalScenario): Promise<ScenarioScore>;
+}
+//# sourceMappingURL=EvalRunner.d.ts.map

package/dist/eval/EvalRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalRunner.d.ts","sourceRoot":"","sources":["../../src/eval/EvalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAa,MAAM,YAAY,CAAC;AAGzE,qBAAa,UAAU;IACT,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,KAAK,CAAC,EAAE,YAAY,CAAA;KAAE;IAEpE,OAAO,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC;CAoE9D"}

package/dist/eval/EvalRunner.js

@@ -0,0 +1,64 @@
+import { Runtime } from '../runtime/Runtime.js';
+import { getFlowState } from '../runtime/pipeline/sessionUtils.js';
+import { aggregateScores, scoreTurn } from './scoring.js';
+export class EvalRunner {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async runText(scenario) {
+        const agent = scenario.agent;
+        const runtime = new Runtime({
+            agents: [agent],
+            defaultAgentId: agent.id,
+            defaultModel: this.config.model,
+            hooks: this.config.hooks,
+        });
+        let sessionId;
+        const turnScores = [];
+        for (let i = 0; i < scenario.turns.length; i++) {
+            const turn = scenario.turns[i];
+            const startTime = Date.now();
+            let response = '';
+            const toolsCalled = [];
+            const transitions = [];
+            for await (const part of runtime.stream({ input: turn.input, sessionId })) {
+                if (part.type === 'text-delta') {
+                    response += part.text;
+                }
+                if (part.type === 'tool-call') {
+                    toolsCalled.push(part.toolName);
+                }
+                if (part.type === 'flow-transition') {
+                    transitions.push({ from: part.from, to: part.to });
+                }
+                if (part.type === 'done') {
+                    sessionId = part.sessionId;
+                }
+            }
+            const latencyMs = Date.now() - startTime;
+            let extractionSnapshot;
+            if (sessionId && turn.expect?.extractionFields) {
+                const session = await runtime.getSession(sessionId);
+                const flow = session ? getFlowState(session, agent.id) : undefined;
+                const data = flow?.context?.collectedData;
+                if (data && typeof data === 'object') {
+                    extractionSnapshot = { ...data };
+                }
+            }
+            const checks = scoreTurn(turn.expect, response, toolsCalled, transitions, latencyMs, extractionSnapshot);
+            turnScores.push({
+                turnIndex: i,
+                input: turn.input,
+                response: response.trim(),
+                passed: checks.length === 0 ? true : checks.every(c => c.passed),
+                checks,
+                latencyMs,
+                toolsCalled,
+                flowTransitions: transitions,
+            });
+        }
+        return aggregateScores(scenario.name, scenario.mode, turnScores);
+    }
+}
+//# sourceMappingURL=EvalRunner.js.map

package/dist/eval/EvalRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalRunner.js","sourceRoot":"","sources":["../../src/eval/EvalRunner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAGnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,OAAO,UAAU;IACD;IAApB,YAAoB,MAAsD;QAAtD,WAAM,GAAN,MAAM,CAAgD;IAAG,CAAC;IAE9E,KAAK,CAAC,OAAO,CAAC,QAAsB;QAClC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC;YAC1B,MAAM,EAAE,CAAC,KAAK,CAAC;YACf,cAAc,EAAE,KAAK,CAAC,EAAE;YACxB,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YAC/B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;SACzB,CAAC,CAAC;QAEH,IAAI,SAA6B,CAAC;QAClC,MAAM,UAAU,GAAgB,EAAE,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,QAAQ,GAAG,EAAE,CAAC;YAClB,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,MAAM,WAAW,GAAwC,EAAE,CAAC;YAE5D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;gBAC1E,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAC/B,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC;gBACxB,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC9B,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClC,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;oBACpC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrD,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBACzB,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,IAAI,kBAAuD,CAAC;YAC5D,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC;gBAC/C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBACpD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACnE,MAAM,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,aAAa,CAAC;gBAC1C,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACrC,kBAAkB,GAAG,EAAE,GAAG,IAAI,EAA6B,CAAC;gBAC9D,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,SAAS,CACtB,IAAI,CAAC,MAAM,EACX,QAAQ,EACR,WAAW,EACX,WAAW,EACX,SAAS,EACT,kBAAkB,CACnB,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC;gBACd,SAAS,EAAE,CAAC;gBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;gBAChE,MAAM;gBACN,SAAS;gBACT,WAAW;gBACX,eAAe,EAAE,WAAW;aAC7B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACnE,CAAC;CACF"}

package/dist/eval/scoring.d.ts

@@ -0,0 +1,15 @@
+import type { EvalTurn, ScenarioScore, TurnScore } from './types.js';
+/**
+ * Score a single turn against expectations.
+ * @param extractionSnapshot — flow `context.collectedData` after the turn (optional).
+ */
+export declare function scoreTurn(expect: EvalTurn['expect'], response: string, toolsCalled: string[], transitions: Array<{
+    from: string;
+    to: string;
+}>, latencyMs: number, extractionSnapshot?: Record<string, unknown>): Array<{
+    name: string;
+    passed: boolean;
+    detail: string;
+}>;
+export declare function aggregateScores(scenarioName: string, mode: 'text' | 'voice', turnScores: TurnScore[]): ScenarioScore;
+//# sourceMappingURL=scoring.d.ts.map

package/dist/eval/scoring.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../../src/eval/scoring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAUrE;;;GAGG;AACH,wBAAgB,SAAS,CACvB,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAC1B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,WAAW,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,EAChD,SAAS,EAAE,MAAM,EACjB,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC3C,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAoG1D;AAsBD,wBAAgB,eAAe,CAC7B,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,GAAG,OAAO,EACtB,UAAU,EAAE,SAAS,EAAE,GACtB,aAAa,CA8Bf"}

package/dist/eval/scoring.js

@@ -0,0 +1,152 @@
+function valuesEqual(a, b) {
+    if (a === b)
+        return true;
+    if (typeof a === 'object' && a !== null && typeof b === 'object' && b !== null) {
+        return JSON.stringify(a) === JSON.stringify(b);
+    }
+    return false;
+}
+/**
+ * Score a single turn against expectations.
+ * @param extractionSnapshot — flow `context.collectedData` after the turn (optional).
+ */
+export function scoreTurn(expect, response, toolsCalled, transitions, latencyMs, extractionSnapshot) {
+    const checks = [];
+    if (!expect) {
+        return checks;
+    }
+    const toolSet = new Set(toolsCalled);
+    if (expect.toolCalls) {
+        for (const tool of expect.toolCalls) {
+            const passed = toolSet.has(tool);
+            checks.push({
+                name: `tool:${tool}`,
+                passed,
+                detail: passed ? `Observed tool call "${tool}"` : `Missing tool call "${tool}"; got: ${[...toolSet].join(', ') || '(none)'}`,
+            });
+        }
+    }
+    if (expect.noToolCalls) {
+        for (const tool of expect.noToolCalls) {
+            const passed = !toolSet.has(tool);
+            checks.push({
+                name: `no-tool:${tool}`,
+                passed,
+                detail: passed
+                    ? `Tool "${tool}" was not called`
+                    : `Tool "${tool}" was called but should not have been`,
+            });
+        }
+    }
+    if (expect.flowTransition) {
+        const { from, to } = expect.flowTransition;
+        const passed = transitions.some(t => t.from === from && t.to === to);
+        checks.push({
+            name: 'transition',
+            passed,
+            detail: passed
+                ? `Saw transition ${from} → ${to}`
+                : `Expected transition ${from} → ${to}; saw: ${transitions.length ? transitions.map(t => `${t.from}→${t.to}`).join('; ') : '(none)'}`,
+        });
+    }
+    if (expect.extractionFields && extractionSnapshot) {
+        for (const [key, expectedVal] of Object.entries(expect.extractionFields)) {
+            const actual = extractionSnapshot[key];
+            const passed = valuesEqual(actual, expectedVal);
+            checks.push({
+                name: `extraction:${key}`,
+                passed,
+                detail: passed
+                    ? `Field "${key}" matches expected`
+                    : `Field "${key}": expected ${JSON.stringify(expectedVal)}, got ${JSON.stringify(actual)}`,
+            });
+        }
+    }
+    else if (expect.extractionFields && !extractionSnapshot) {
+        checks.push({
+            name: 'extraction',
+            passed: false,
+            detail: 'Expected extractionFields checks but no flow extraction snapshot was available',
+        });
+    }
+    if (expect.responseContains) {
+        for (const phrase of expect.responseContains) {
+            const passed = response.includes(phrase);
+            checks.push({
+                name: `contains:${phrase}`,
+                passed,
+                detail: passed ? `Response contains "${phrase}"` : `Response missing substring "${phrase}"`,
+            });
+        }
+    }
+    if (expect.responseNotContains) {
+        for (const phrase of expect.responseNotContains) {
+            const passed = !response.includes(phrase);
+            checks.push({
+                name: `notContains:${phrase}`,
+                passed,
+                detail: passed ? `Response does not contain "${phrase}"` : `Response should not contain "${phrase}"`,
+            });
+        }
+    }
+    if (expect.maxLatencyMs !== undefined) {
+        const passed = latencyMs <= expect.maxLatencyMs;
+        checks.push({
+            name: 'latency',
+            passed,
+            detail: passed
+                ? `Latency ${latencyMs}ms within ${expect.maxLatencyMs}ms`
+                : `Latency ${latencyMs}ms exceeds max ${expect.maxLatencyMs}ms`,
+        });
+    }
+    return checks;
+}
+function accuracyForPrefix(turns, prefixes) {
+    let total = 0;
+    let passed = 0;
+    for (const t of turns) {
+        for (const c of t.checks) {
+            if (prefixes.some(p => c.name.startsWith(p))) {
+                total++;
+                if (c.passed)
+                    passed++;
+            }
+        }
+    }
+    return total === 0 ? 1 : passed / total;
+}
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const pos = (p / 100) * (sorted.length - 1);
+    return sorted[Math.round(pos)];
+}
+export function aggregateScores(scenarioName, mode, turnScores) {
+    const totalTurns = turnScores.length;
+    const passedTurns = turnScores.filter(t => t.passed).length;
+    const failedTurns = totalTurns - passedTurns;
+    const passRate = totalTurns === 0 ? 1 : passedTurns / totalTurns;
+    const latencies = turnScores.map(t => t.latencyMs).sort((a, b) => a - b);
+    const sum = latencies.reduce((a, b) => a + b, 0);
+    const avgLatencyMs = totalTurns === 0 ? 0 : sum / totalTurns;
+    const toolCallAccuracy = accuracyForPrefix(turnScores, ['tool:', 'no-tool:']);
+    const extractionAccuracy = accuracyForPrefix(turnScores, ['extraction:']);
+    return {
+        scenario: scenarioName,
+        mode,
+        passed: failedTurns === 0,
+        turnScores,
+        aggregate: {
+            passRate,
+            totalTurns,
+            passedTurns,
+            failedTurns,
+            avgLatencyMs,
+            p50LatencyMs: percentile(latencies, 50),
+            p95LatencyMs: percentile(latencies, 95),
+            toolCallAccuracy,
+            extractionAccuracy,
+        },
+    };
+}
+//# sourceMappingURL=scoring.js.map

package/dist/eval/scoring.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.js","sourceRoot":"","sources":["../../src/eval/scoring.ts"],"names":[],"mappings":"AAEA,SAAS,WAAW,CAAC,CAAU,EAAE,CAAU;IACzC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/E,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CACvB,MAA0B,EAC1B,QAAgB,EAChB,WAAqB,EACrB,WAAgD,EAChD,SAAiB,EACjB,kBAA4C;IAE5C,MAAM,MAAM,GAA6D,EAAE,CAAC;IAC5E,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,QAAQ,IAAI,EAAE;gBACpB,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,uBAAuB,IAAI,GAAG,CAAC,CAAC,CAAC,sBAAsB,IAAI,WAAW,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,EAAE;aAC7H,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,WAAW,IAAI,EAAE;gBACvB,MAAM;gBACN,MAAM,EAAE,MAAM;oBACZ,CAAC,CAAC,SAAS,IAAI,kBAAkB;oBACjC,CAAC,CAAC,SAAS,IAAI,uCAAuC;aACzD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC;QAC3C,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY;YAClB,MAAM;YACN,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,kBAAkB,IAAI,MAAM,EAAE,EAAE;gBAClC,CAAC,CAAC,uBAAuB,IAAI,MAAM,EAAE,UACnC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAC9E,EAAE;SACL,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,gBAAgB,IAAI,kBAAkB,EAAE,CAAC;QAClD,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACzE,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,cAAc,GAAG,EAAE;gBACzB,MAAM;gBACN,MAAM,EAAE,MAAM;oBACZ,CAAC,CAAC,UAAU,GAAG,oBAAoB;oBACnC,CAAC,CAAC,UAAU,GAAG,eAAe,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;aAC7F,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,CAAC,gBAAgB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1D,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,gFAAgF;SACzF,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC5B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,YAAY,MAAM,EAAE;gBAC1B,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,sBAAsB,MAAM,GAAG,CAAC,CAAC,CAAC,+BAA+B,MAAM,GAAG;aAC5F,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,eAAe,MAAM,EAAE;gBAC7B,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,8BAA8B,MAAM,GAAG,CAAC,CAAC,CAAC,gCAAgC,MAAM,GAAG;aACrG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAS;YACf,MAAM;YACN,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,WAAW,SAAS,aAAa,MAAM,CAAC,YAAY,IAAI;gBAC1D,CAAC,CAAC,WAAW,SAAS,kBAAkB,MAAM,CAAC,YAAY,IAAI;SAClE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB,EAAE,QAAkB;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,CAAC,MAAM;oBAAE,MAAM,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,YAAoB,EACpB,IAAsB,EACtB,UAAuB;IAEvB,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;IACrC,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5D,MAAM,WAAW,GAAG,UAAU,GAAG,WAAW,CAAC;IAC7C,MAAM,QAAQ,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,UAAU,CAAC;IAEjE,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,YAAY,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC;IAE7D,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC9E,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,UAAU,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IAE1E,OAAO;QACL,QAAQ,EAAE,YAAY;QACtB,IAAI;QACJ,MAAM,EAAE,WAAW,KAAK,CAAC;QACzB,UAAU;QACV,SAAS,EAAE;YACT,QAAQ;YACR,UAAU;YACV,WAAW;YACX,WAAW;YACX,YAAY;YACZ,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC;YACvC,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC;YACvC,gBAAgB;YAChB,kBAAkB;SACnB;KACF,CAAC;AACJ,CAAC"}

package/dist/eval/types.d.ts

@@ -0,0 +1,59 @@
+import type { AgentConfig, FlowAgentConfig } from '../types/index.js';
+export interface EvalScenario {
+    name: string;
+    description?: string;
+    agent: AgentConfig | FlowAgentConfig;
+    turns: EvalTurn[];
+    mode: 'text' | 'voice';
+}
+export interface EvalTurn {
+    input: string;
+    audioFixture?: string;
+    expect?: {
+        toolCalls?: string[];
+        noToolCalls?: string[];
+        flowTransition?: {
+            from: string;
+            to: string;
+        };
+        extractionFields?: Record<string, unknown>;
+        responseContains?: string[];
+        responseNotContains?: string[];
+        maxLatencyMs?: number;
+    };
+}
+export interface ScenarioScore {
+    scenario: string;
+    mode: 'text' | 'voice';
+    passed: boolean;
+    turnScores: TurnScore[];
+    aggregate: {
+        passRate: number;
+        totalTurns: number;
+        passedTurns: number;
+        failedTurns: number;
+        avgLatencyMs: number;
+        p50LatencyMs: number;
+        p95LatencyMs: number;
+        toolCallAccuracy: number;
+        extractionAccuracy: number;
+    };
+}
+export interface TurnScore {
+    turnIndex: number;
+    input: string;
+    response: string;
+    passed: boolean;
+    checks: Array<{
+        name: string;
+        passed: boolean;
+        detail: string;
+    }>;
+    latencyMs: number;
+    toolsCalled: string[];
+    flowTransitions: Array<{
+        from: string;
+        to: string;
+    }>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEtE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,WAAW,GAAG,eAAe,CAAC;IACrC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE;QACP,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,cAAc,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9C,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC3C,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;CACH;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACtD"}

package/dist/eval/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/eval/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":""}

package/dist/flows/FlowGraph.d.ts

@@ -1,4 +1,4 @@
-import type { ContextStrategy, FlowConfig, FlowNodeConfig, FlowTransitionConfig } from '../types/index.js';
+import type { ContextStrategy, ExtractionNodeConfig, FlowConfig, FlowNodeConfig, FlowTransitionConfig } from '../types/index.js';
 import { FlowNode } from './FlowNode.js';
 export declare class FlowGraph {
     private nodes;
@@ -15,6 +15,8 @@ export declare class FlowGraph {
 export declare class FlowGraphBuilder {
     private graph;
     node(id: string, config: Omit<FlowNodeConfig, 'id'>): this;
+    /** Add an extraction node that loops until a Zod schema is fully satisfied. */
+    extractionNode(id: string, config: Omit<ExtractionNodeConfig, 'id' | 'nodeType'>): this;
     transition(from: string, to: string, config?: Omit<FlowTransitionConfig, 'from' | 'to'>): this;
     defaultRolePrompt(prompt: string): this;
     contextStrategy(strategy: ContextStrategy): this;