@falai/agent 2.2.4 → 2.4.1

package/dist/core/ResponsePipeline.js

@@ -1,13 +1,13 @@
 /**
  * Response processing utilities shared between respond() and respondStream() methods
  */
-import { createSession, enterStep, mergeCollected, completeCurrentFlow, logger, historyToEvents, serializeToolResult, } from "../utils";
+import { createSession, enterStep, mergeCollected, logger, historyToEvents, eventsToHistory, getLastMessageFromHistory, } from "../utils";
 import { enterFlow } from "../utils/session";
 import { createTemplateContext } from "../utils/template";
 import { FlowConfigurationError } from "../core/Step";
 import { evaluateBranches, createAiConditionEvaluator } from "./BranchEvaluator";
 import { DirectiveChainTracker } from "./DirectiveChainTracker";
-import { DirectiveBus } from "./DirectiveBus";
+import { ResponseGenerationError } from "./ResponseGenerationError";
 /**
  * Position fields on a Directive that represent a navigation decision.
  * When any of these is set, the directive "wins" the turn's position decision
@@ -29,16 +29,13 @@ export function hasDirectivePositionField(directive) {
  * Shared response processing logic between respond() and respondStream() methods
  */
 export class ResponsePipeline {
-    constructor(options, getFlows, tools, flowRouter, updateContext, updateData, updateCollectedData, toolManager, signalProcessor) {
+    constructor(options, getFlows, flowRouter, signalCoordinator, updateCollectedData, getSchema) {
         this.options = options;
         this.getFlows = getFlows;
-        this.tools = tools;
         this.flowRouter = flowRouter;
-        this.updateContext = updateContext;
-        this.updateData = updateData;
+        this.signalCoordinator = signalCoordinator;
         this.updateCollectedData = updateCollectedData;
-        this.toolManager = toolManager;
-        this.signalProcessor = signalProcessor;
+        this.getSchema = getSchema;
     }
     /**
      * Create a fresh chain tracker for the current turn.
@@ -58,87 +55,30 @@ export class ResponsePipeline {
         }
         return this._chainTracker;
     }
-    /**
-     * Create a fresh DirectiveBus for a new turn.
-     * The bus collects directives from hooks, tools, and branches during the turn
-     * and merges them at phase boundaries via Algorithm 4.
-     */
-    createDirectiveBus() {
-        return new DirectiveBus();
-    }
-    // ──────────────────────────────────────────────────────────────────────────
-    // Signal pipeline phases
-    // ──────────────────────────────────────────────────────────────────────────
-    /**
-     * PRE-SIGNAL PHASE — Evaluates pre/both signals in parallel with routing.
-     *
-     * Delegates to `signalProcessor.runPreSignalPhase(...)` when configured.
-     * When no signal processor is present (zero-cost path), returns a stable
-     * empty shape so callers don't need to branch.
-     *
-     * @requirements 2.1, 2.3, 8.6, 13.3
-     */
-    async runPreSignalPhase(session, context, history) {
-        if (!this.signalProcessor) {
-            return { firings: [], updatedSession: session, mergedDirective: undefined };
-        }
-        const result = await this.signalProcessor.runPreSignalPhase({ session, history, context });
-        return {
-            firings: result.firings,
-            updatedSession: result.updatedSession,
-            mergedDirective: result.mergedDirective,
-        };
-    }
-    /**
-     * POST-SIGNAL PHASE — Evaluates post/both signals after finalize/onComplete.
-     *
-     * Delegates to `signalProcessor.runPostSignalPhase(...)` when configured.
-     * When no signal processor is present, returns a stable empty shape.
-     *
-     * Post-phase signals see the complete turn result: assistant message in history,
-     * collected data, tool results. Position directives from this phase set
-     * `session.pendingDirective` (no mid-turn re-entry per D6 decision).
-     *
-     * Pre-LLM-only fields (`appendPrompt`, `injectTools`, `halt`) are already
-     * dropped inside `runPostSignalPhase` per Phase 4.5 — this seam does NOT
-     * re-introduce them.
-     *
-     * @requirements 9.1, 9.2, 9.3, 9.4
-     */
-    async runPostSignalPhase(session, context, history) {
-        if (!this.signalProcessor) {
-            return { firings: [], updatedSession: session, mergedDirective: undefined };
-        }
-        const result = await this.signalProcessor.runPostSignalPhase({ session, history, context });
-        return {
-            firings: result.firings,
-            updatedSession: result.updatedSession,
-            mergedDirective: result.mergedDirective,
-        };
-    }
     /**
      * Prepare context and session for response generation
      */
     async prepareResponseContext(params) {
         const { contextOverride, session } = params;
-        // Get current context (may fetch from provider)
-        let currentContext = await this.getContext();
+        let currentContext = params.currentContext;
+        let contextAfterHook;
         // Call beforeRespond hook if configured
         if (this.options.hooks?.beforeRespond && currentContext !== undefined) {
             currentContext = await this.options.hooks.beforeRespond(currentContext);
-            // Update stored context with the result from beforeRespond
-            this.context = currentContext;
+            // Surface the hook result so the caller can sync it back to the agent
+            contextAfterHook = currentContext;
         }
         // Merge context with override
         const effectiveContext = {
             ...currentContext,
             ...contextOverride,
         };
-        // Initialize or get session (use current session if available)
-        const targetSession = session || this.currentSession || createSession();
+        // Initialize or get session (use the live session if available)
+        const targetSession = session || params.currentSession || createSession();
         return {
             effectiveContext,
             session: targetSession,
+            contextAfterHook,
         };
     }
     /**
@@ -158,7 +98,7 @@ export class ResponsePipeline {
         // Check for pending directive from previous flow completion or external dispatch
         if (targetSession.pendingDirective) {
             const directive = targetSession.pendingDirective;
-            logger.debug(`[ResponseHandler] Applying pending directive at start of turn`);
+            logger.debug(`[ResponsePipeline] Applying pending directive at start of turn`);
             // Track directive chain depth (Requirement 22.1)
             const tracker = this.chainTracker;
             tracker.record(directive, "pending");
@@ -182,7 +122,7 @@ export class ResponsePipeline {
                 if (flowTarget) {
                     const targetFlow = flows.find((r) => r.id === flowTarget || r.title === flowTarget);
                     if (targetFlow) {
-                        logger.debug(`[ResponseHandler] Pending directive goTo → flow: ${targetFlow.title}`);
+                        logger.debug(`[ResponsePipeline] Pending directive goTo → flow: ${targetFlow.title}`);
                         targetSession = enterFlow(targetSession, targetFlow.id, targetFlow.title);
                         // Merge initial data if available
                         if (targetFlow.initialData) {
@@ -291,7 +231,7 @@ export class ResponsePipeline {
             completedFlows = orchestration.completedFlows || [];
             // Log if flow is complete
             if (isFlowComplete) {
-                logger.debug(`[ResponseHandler] Flow complete: all required data collected or last step reached`);
+                logger.debug(`[ResponsePipeline] Flow complete: all required data collected or last step reached`);
             }
         }
         return {
@@ -307,7 +247,7 @@ export class ResponsePipeline {
      * Determine next step and update session
      */
     async determineNextStep(params) {
-        const { selectedFlow, selectedStep, session, isFlowComplete, busDirective } = params;
+        const { selectedFlow, selectedStep, session, isFlowComplete, context, busDirective } = params;
         if (!selectedFlow) {
             return { nextStep: undefined, session };
         }
@@ -317,7 +257,7 @@ export class ResponsePipeline {
         // complete, abort, reset), that decision wins the turn. Do NOT evaluate
         // branches or linear/AI selection — the caller applies the bus directive.
         if (hasDirectivePositionField(busDirective)) {
-            logger.debug(`[ResponseHandler] Directive bus winner has position field — skipping branch evaluation and linear/AI selection`);
+            logger.debug(`[ResponsePipeline] Directive bus winner has position field — skipping branch evaluation and linear/AI selection`);
             return { nextStep: undefined, session };
         }
         // STEP 1 (Algorithm 1): branches win over linear chain AND flow completion.
@@ -328,8 +268,7 @@ export class ResponsePipeline {
                 ? selectedFlow.getStep(session.currentStep.id)
                 : undefined;
             if (currentStep?.branches && currentStep.branches.length > 0) {
-                const contextToUse = this.getStoredContext();
-                const branchResult = await this.evaluateStepBranches(currentStep, selectedFlow, session, contextToUse);
+                const branchResult = await this.evaluateStepBranches(currentStep, selectedFlow, session, context);
                 if (branchResult) {
                     return branchResult;
                 }
@@ -349,13 +288,12 @@ export class ResponsePipeline {
             const currentStep = session.currentFlow?.id === selectedFlow.id && session.currentStep
                 ? selectedFlow.getStep(session.currentStep.id)
                 : undefined;
-            const contextToUse = this.getStoredContext();
             // Get candidate steps based on current position in the flow
             const candidates = await this.flowRouter.getCandidateStepsWithConditions(selectedFlow, currentStep, // Pass current step instead of undefined to maintain progression
-            createTemplateContext({ data: session.data, session, context: contextToUse }));
+            createTemplateContext({ data: session.data, session, context }));
             if (candidates.length > 0) {
                 nextStep = candidates[0].step;
-                logger.debug(`[ResponseHandler] Using first valid step: ${nextStep.id}${currentStep ? ' (progressing from ' + currentStep.id + ')' : ' for new flow'}`);
+                logger.debug(`[ResponsePipeline] Using first valid step: ${nextStep.id}${currentStep ? ' (progressing from ' + currentStep.id + ')' : ' for new flow'}`);
             }
             else {
                 // Fallback to initial step even if it should be skipped
@@ -368,7 +306,7 @@ export class ResponsePipeline {
             const sessionData = session.data || {};
             const missingRequires = nextStep.requires.filter(field => sessionData[String(field)] === undefined);
             if (missingRequires.length > 0) {
-                logger.debug(`[ResponseHandler] Cannot enter step "${nextStep.id}": missing required fields [${missingRequires.join(', ')}]. Staying at current step.`);
+                logger.debug(`[ResponsePipeline] Cannot enter step "${nextStep.id}": missing required fields [${missingRequires.join(', ')}]. Staying at current step.`);
                 // Stay at current step - don't enter the next one
                 const currentStepId = session.currentStep?.id;
                 if (currentStepId && selectedFlow) {
@@ -382,7 +320,7 @@ export class ResponsePipeline {
         }
         // Update session with next step
         const updatedSession = enterStep(session, nextStep.id, nextStep.description);
-        logger.debug(`[ResponseHandler] Entered step: ${nextStep.id}`);
+        logger.debug(`[ResponsePipeline] Entered step: ${nextStep.id}`);
         return { nextStep, session: updatedSession };
     }
     /**
@@ -421,7 +359,7 @@ export class ResponsePipeline {
         const localStep = selectedFlow.getStep(result);
         if (localStep) {
             const updatedSession = enterStep(session, localStep.id, localStep.description);
-            logger.debug(`[ResponseHandler] Branch resolved to local step: ${localStep.id}`);
+            logger.debug(`[ResponsePipeline] Branch resolved to local step: ${localStep.id}`);
             return { nextStep: localStep, session: updatedSession };
         }
         // 2. Look up in agent's flow registry
@@ -429,7 +367,7 @@ export class ResponsePipeline {
         const targetFlow = flows.find(f => f.id === result || f.title === result);
         if (targetFlow) {
             // Treat as applyDirective({ goTo: result }) — enter the target flow
-            logger.debug(`[ResponseHandler] Branch resolved to flow: ${targetFlow.title}`);
+            logger.debug(`[ResponsePipeline] Branch resolved to flow: ${targetFlow.title}`);
             const updatedSession = enterFlow(session, targetFlow.id, targetFlow.title);
             return { nextStep: undefined, session: updatedSession, flowChanged: targetFlow };
         }
@@ -467,7 +405,7 @@ export class ResponsePipeline {
                     updatedSession = enterStep(updatedSession, stepTarget);
                     // Try to resolve the target step instance for the caller
                     const targetStepInstance = targetFlow.getStep(stepTarget);
-                    logger.debug(`[ResponseHandler] Branch directive goToStep → ${flowTarget}.${stepTarget}`);
+                    logger.debug(`[ResponsePipeline] Branch directive goToStep → ${flowTarget}.${stepTarget}`);
                     return { nextStep: targetStepInstance || undefined, session: updatedSession, flowChanged: targetFlow };
                 }
                 throw new FlowConfigurationError(`[FlowConfigurationError] Branch directive goToStep targets unknown flow: "${flowTarget}" does not match any flow id or title. ` +
@@ -477,7 +415,7 @@ export class ResponsePipeline {
             const targetStep = selectedFlow.getStep(stepTarget);
             if (targetStep) {
                 updatedSession = enterStep(updatedSession, targetStep.id, targetStep.description);
-                logger.debug(`[ResponseHandler] Branch directive goToStep → ${targetStep.id}`);
+                logger.debug(`[ResponsePipeline] Branch directive goToStep → ${targetStep.id}`);
                 return { nextStep: targetStep, session: updatedSession };
             }
             throw new FlowConfigurationError(`[FlowConfigurationError] Branch directive goToStep targets unknown step: "${stepTarget}" does not exist in flow "${selectedFlow.id}". ` +
@@ -496,7 +434,7 @@ export class ResponsePipeline {
                     if (typeof directive.goTo === 'object' && directive.goTo.step) {
                         updatedSession = enterStep(updatedSession, directive.goTo.step);
                     }
-                    logger.debug(`[ResponseHandler] Branch directive goTo → ${targetFlow.title}`);
+                    logger.debug(`[ResponsePipeline] Branch directive goTo → ${targetFlow.title}`);
                     return { nextStep: undefined, session: updatedSession, flowChanged: targetFlow };
                 }
                 throw new FlowConfigurationError(`[FlowConfigurationError] Branch directive goTo targets unknown flow: "${flowTarget}" does not match any flow id or title. ` +
@@ -504,11 +442,11 @@ export class ResponsePipeline {
             }
         }
         if (directive.complete) {
-            logger.debug(`[ResponseHandler] Branch directive complete`);
+            logger.debug(`[ResponsePipeline] Branch directive complete`);
             return { nextStep: undefined, session: updatedSession };
         }
         if (directive.abort) {
-            logger.debug(`[ResponseHandler] Branch directive abort`);
+            logger.debug(`[ResponsePipeline] Branch directive abort`);
             return { nextStep: undefined, session: updatedSession };
         }
         if (directive.reset) {
@@ -525,7 +463,7 @@ export class ResponsePipeline {
             // Reset to initial step
             const initialStep = selectedFlow.initialStep;
             updatedSession = enterStep(updatedSession, initialStep.id, initialStep.description);
-            logger.debug(`[ResponseHandler] Branch directive reset → ${initialStep.id}`);
+            logger.debug(`[ResponsePipeline] Branch directive reset → ${initialStep.id}`);
             return { nextStep: initialStep, session: updatedSession };
         }
         // Directive with only non-position fields (e.g., just dataUpdate/contextUpdate/reply)
@@ -533,407 +471,420 @@ export class ResponsePipeline {
         return { nextStep: undefined, session: updatedSession };
     }
     /**
-     * Execute tool calls and handle results
+     * TURN ROUTING + STEP SELECTION — the single entry point for deciding which
+     * flow and step a turn renders. Runs the routing-skip optimization, the
+     * pre-signal phase (parallel with routing when a processor is configured),
+     * pre-extraction, and next-step determination.
      */
-    async executeToolCalls(params) {
-        const { toolCalls, selectedFlow, context, session, history, isStreaming = false, } = params;
-        if (toolCalls.length === 0) {
-            return { session, toolCalls: undefined };
-        }
-        logger.debug(`[ResponseHandler] Executing ${toolCalls.length} ${isStreaming ? "streaming " : ""}tool calls`);
-        let updatedSession = session;
-        const executedToolCalls = [];
-        const toolResults = new Map();
-        for (const toolCall of toolCalls) {
-            const tool = this.findAvailableTool(toolCall.toolName, selectedFlow);
-            if (!tool) {
-                logger.warn(`[ToolExecutionError] Tool not found: "${toolCall.toolName}" is not registered in any scope (transient, step, flow, or agent). Skipping this tool call. Register the tool or check the tool name.`);
-                continue;
+    async routeAndSelectStep(params) {
+        try {
+            // Create a fresh chain tracker for this turn (Requirement 22.1)
+            this.createChainTracker();
+            // ROUTING SKIP OPTIMIZATION (Requirements 20.1, 20.2, 20.3):
+            // When the current step has collect fields AND pre-extraction populates at least
+            // one of those fields, skip FlowRouter.decideFlowAndStep for this turn.
+            const routingSkipResult = await this.attemptRoutingSkipForCollect(params);
+            if (routingSkipResult) {
+                // Even when routing is skipped, run pre-signal phase if processor is present
+                if (this.signalCoordinator.enabled) {
+                    const signalResult = await this.signalCoordinator.runPrePhase(params.session, params.context, params.history);
+                    // If signal halts, override the routing skip result
+                    if (signalResult.mergedDirective?.halt) {
+                        return {
+                            ...routingSkipResult,
+                            session: signalResult.updatedSession,
+                            signalFirings: signalResult.firings,
+                            signalHalted: true,
+                            signalHaltReply: signalResult.mergedDirective.reply,
+                        };
+                    }
+                    // If signal has position fields, override routing skip result
+                    if (hasDirectivePositionField(signalResult.mergedDirective)) {
+                        return this.signalCoordinator.applyPositionDirective(signalResult);
+                    }
+                    // Non-position directive: propagate for pre-LLM augmentation
+                    return {
+                        ...routingSkipResult,
+                        session: signalResult.updatedSession,
+                        signalFirings: signalResult.firings,
+                        signalPreDirective: signalResult.mergedDirective || undefined,
+                    };
+                }
+                return routingSkipResult;
             }
-            // Use ToolManager for unified tool execution
-            let result;
-            if (this.toolManager) {
-                result = await this.toolManager.executeTool({
-                    tool,
-                    context,
-                    updateContext: this.updateContext,
-                    updateData: this.updateCollectedData || (async () => { }),
-                    history,
-                    data: updatedSession.data,
+            // ── PARALLEL PRE-SIGNAL PHASE + ROUTING (Algorithm 5) ────────────────
+            // When signalProcessor is present, run pre-signals in parallel with routing.
+            // When absent, call the router directly (zero overhead, preserve current behavior).
+            if (this.signalCoordinator.enabled) {
+                // Run pre-signal phase in parallel with routing (Requirement 8.1)
+                const [signalResult, routingResult] = await Promise.all([
+                    this.signalCoordinator.runPrePhase(params.session, params.context, params.history),
+                    this.handleRoutingAndStepSelection({
+                        session: params.session,
+                        history: params.history,
+                        context: params.context,
+                        signal: params.signal,
+                    }),
+                ]);
+                // ── Requirement 8.2: halt → discard routing, skip LLM ────────────
+                if (signalResult.mergedDirective?.halt) {
+                    return {
+                        selectedFlow: undefined,
+                        selectedStep: undefined,
+                        session: signalResult.updatedSession,
+                        isFlowComplete: false,
+                        signalFirings: signalResult.firings,
+                        signalHalted: true,
+                        signalHaltReply: signalResult.mergedDirective.reply,
+                    };
+                }
+                // ── Requirement 8.3: position directive → discard routing, apply signal position ──
+                if (hasDirectivePositionField(signalResult.mergedDirective)) {
+                    return this.signalCoordinator.applyPositionDirective(signalResult);
+                }
+                // ── Requirement 8.4: non-position directive → use routing, propagate augmentation ──
+                // ── Requirement 8.5: no directive → use routing as-is ─────────────
+                let updatedSession = signalResult.updatedSession;
+                // Apply data/context updates from signal to the routed session
+                if (signalResult.mergedDirective?.dataUpdate) {
+                    updatedSession = mergeCollected(updatedSession, signalResult.mergedDirective.dataUpdate);
+                }
+                // Use routing result for flow/step, but carry signal session state
+                // Merge routing session changes on top of signal session
+                const routingSession = routingResult.session;
+                updatedSession = {
+                    ...updatedSession,
+                    currentFlow: routingSession.currentFlow,
+                    currentStep: routingSession.currentStep,
+                    flowHistory: routingSession.flowHistory,
+                    pendingDirective: routingSession.pendingDirective,
+                };
+                const isFlowComplete = routingResult.isFlowComplete;
+                // PRE-EXTRACTION: same logic as below — extract data from user message
+                if (routingResult.selectedFlow && !isFlowComplete) {
+                    if (this.shouldPreExtractData(routingResult.selectedFlow)) {
+                        logger.debug(`[ResponsePipeline] Pre-extracting data for flow: ${routingResult.selectedFlow.title}`);
+                        const extractedData = await this.preExtractFlowData({
+                            route: routingResult.selectedFlow,
+                            history: params.history,
+                            context: params.context,
+                            session: updatedSession,
+                            signal: params.signal,
+                        });
+                        if (extractedData && Object.keys(extractedData).length > 0) {
+                            logger.debug(`[ResponsePipeline] Pre-extracted data:`, extractedData);
+                            updatedSession = mergeCollected(updatedSession, extractedData);
+                            await this.updateCollectedData(extractedData);
+                        }
+                    }
+                }
+                // Determine next step
+                const stepResult = await this.determineNextStep({
+                    selectedFlow: routingResult.selectedFlow,
+                    selectedStep: routingResult.selectedStep,
+                    session: updatedSession,
+                    isFlowComplete,
+                    context: params.context,
                 });
+                return {
+                    selectedFlow: stepResult.flowChanged || routingResult.selectedFlow,
+                    selectedStep: stepResult.nextStep,
+                    responseDirectives: routingResult.responseDirectives,
+                    session: stepResult.session,
+                    isFlowComplete: stepResult.flowChanged ? false : isFlowComplete,
+                    signalFirings: signalResult.firings,
+                    signalPreDirective: signalResult.mergedDirective || undefined,
+                };
             }
-            else {
-                // Fallback: execute tool directly if ToolManager not available
-                throw new Error(`ToolManager not available for tool execution: ${toolCall.toolName}`);
-            }
-            executedToolCalls.push(toolCall);
-            // Store the actual tool result data for history
-            toolResults.set(toolCall.toolName, serializeToolResult(result));
-            // Update context with tool results
-            if (result.contextUpdate) {
-                await this.updateContext(result.contextUpdate);
-            }
-            // Update collected data with tool results
-            if (result.dataUpdate) {
-                updatedSession = await this.updateData(updatedSession, result.dataUpdate);
-                logger.debug(`[ResponseHandler] ${isStreaming ? "Streaming " : ""}Tool updated collected data:`, result.dataUpdate);
-            }
-            logger.debug(`[ResponseHandler] Executed ${isStreaming ? "streaming " : ""}tool: ${String(tool.id || tool.id || 'unknown')} (success: ${result.success})`);
-        }
-        return {
-            session: updatedSession,
-            toolCalls: executedToolCalls.length > 0 ? executedToolCalls : undefined,
-            toolResults,
-        };
-    }
-    /**
-     * Execute tool loop for follow-up tool calls
-     */
-    async executeToolLoop(params) {
-        const { initialToolCalls, selectedFlow, nextStep, responsePrompt, history, context, session, responseSchema, isStreaming = false, } = params;
-        const MAX_TOOL_LOOPS = 5;
-        let toolLoopCount = 0;
-        let currentToolCalls = initialToolCalls;
-        let hasToolCalls = currentToolCalls && currentToolCalls.length > 0;
-        let currentSession = session;
-        // Track tool results across loop iterations for history
-        let currentToolResults = new Map();
-        while (hasToolCalls && toolLoopCount < MAX_TOOL_LOOPS) {
-            toolLoopCount++;
-            logger.debug(`[ResponseHandler] Starting ${isStreaming ? "streaming " : ""}tool loop ${toolLoopCount}/${MAX_TOOL_LOOPS}`);
-            // Add tool execution results to history so AI knows what happened
-            const toolResultItems = [];
-            for (const toolCall of currentToolCalls || []) {
-                const tool = this.findAvailableTool(toolCall.toolName, selectedFlow);
-                if (tool) {
-                    toolResultItems.push({
-                        role: "assistant",
-                        content: null,
-                        tool_calls: [{
-                                id: toolCall.toolName,
-                                name: toolCall.toolName,
-                                arguments: toolCall.arguments,
-                            }],
-                    });
-                    toolResultItems.push({
-                        role: "tool",
-                        tool_call_id: toolCall.toolName,
-                        name: toolCall.toolName,
-                        content: currentToolResults.get(toolCall.toolName) || "Tool executed successfully",
+            // ── No signal processor: existing behavior (zero overhead) ────────────
+            const routingResult = await this.handleRoutingAndStepSelection({
+                session: params.session,
+                history: params.history,
+                context: params.context,
+                signal: params.signal,
+            });
+            let updatedSession = routingResult.session;
+            const isFlowComplete = routingResult.isFlowComplete;
+            // PRE-EXTRACTION: If entering a flow that collects data, extract data from user message first
+            // This allows us to skip steps whose data is already provided
+            if (routingResult.selectedFlow && !isFlowComplete) {
+                // Always pre-extract when flow collects data (not just on new flow entry)
+                // This ensures step selection has the most up-to-date data
+                if (this.shouldPreExtractData(routingResult.selectedFlow)) {
+                    logger.debug(`[ResponsePipeline] Pre-extracting data for flow: ${routingResult.selectedFlow.title}`);
+                    const extractedData = await this.preExtractFlowData({
+                        route: routingResult.selectedFlow,
+                        history: params.history,
+                        context: params.context,
+                        session: updatedSession,
+                        signal: params.signal,
                     });
+                    if (extractedData && Object.keys(extractedData).length > 0) {
+                        logger.debug(`[ResponsePipeline] Pre-extracted data:`, extractedData);
+                        // Merge pre-extracted data into session before step selection
+                        updatedSession = mergeCollected(updatedSession, extractedData);
+                        // Also update agent's collected data
+                        await this.updateCollectedData(extractedData);
+                    }
                 }
             }
-            // Create updated history with tool results
-            const updatedHistory = [...history, ...toolResultItems];
-            // Make follow-up AI call to see if more tools are needed
-            const followUpResult = await this.options.provider.generateMessage({
-                prompt: responsePrompt,
-                history: updatedHistory,
-                context,
-                tools: this.collectAvailableTools(selectedFlow, nextStep),
-                parameters: {
-                    jsonSchema: responseSchema,
-                    schemaName: isStreaming ? "tool_followup_streaming" : "tool_followup",
-                },
+            // Determine next step using pipeline method for consistency
+            const stepResult = await this.determineNextStep({
+                selectedFlow: routingResult.selectedFlow,
+                selectedStep: routingResult.selectedStep,
+                session: updatedSession, // Use updated session with pre-extracted data
+                isFlowComplete, // Use updated completion status
+                context: params.context,
             });
-            // Check if follow-up call has more tool calls
-            const followUpToolCalls = followUpResult.structured?.toolCalls;
-            hasToolCalls = followUpToolCalls && followUpToolCalls.length > 0;
-            if (hasToolCalls) {
-                logger.debug(`[ResponseHandler] Follow-up call produced ${followUpToolCalls.length} additional tool calls`);
-                // Execute the follow-up tool calls
-                const toolResult = await this.executeToolCalls({
-                    toolCalls: followUpToolCalls,
-                    selectedFlow,
-                    context,
-                    session: currentSession,
-                    history: historyToEvents(updatedHistory),
-                    isStreaming,
-                });
-                currentSession = toolResult.session;
-                currentToolCalls = followUpToolCalls;
-                currentToolResults = toolResult.toolResults || new Map();
-            }
-            else {
-                logger.debug(`[ResponseHandler] ${isStreaming ? "Streaming " : ""}Tool loop completed after ${toolLoopCount} iterations`);
-                // Update final toolCalls from follow-up result if no more tools
-                currentToolCalls = followUpToolCalls || [];
-                break;
-            }
-        }
-        if (toolLoopCount >= MAX_TOOL_LOOPS) {
-            logger.warn(`[ResponseGenerationError] ${isStreaming ? "Streaming t" : "T"}ool loop limit reached: ${toolLoopCount} iterations hit the cap (${MAX_TOOL_LOOPS}). Stopping tool execution. Increase MAX_TOOL_LOOPS or reduce recursive tool calls.`);
-        }
-        return {
-            session: currentSession,
-            toolCalls: currentToolCalls,
-        };
-    }
-    /**
-     * Handle data collection from structured response
-     */
-    async handleDataCollection(params) {
-        const { structured, nextStep, session } = params;
-        if (!structured || !nextStep.collect) {
-            return { session };
-        }
-        const collectedData = {};
-        // The structured response includes both base fields and collected extraction fields
-        const structuredData = structured;
-        for (const field of nextStep.collect) {
-            const fieldKey = field;
-            if (fieldKey in structuredData) {
-                collectedData[fieldKey] = structuredData[fieldKey];
-            }
-        }
-        let updatedSession = session;
-        if (Object.keys(collectedData).length > 0) {
-            // Update agent-level collected data with validation if available
-            if (this.updateCollectedData) {
-                await this.updateCollectedData(collectedData);
-            }
-            // Update session with validated data
-            updatedSession = await this.updateData(session, collectedData);
-            logger.debug(`[ResponseHandler] Collected data:`, collectedData);
+            return {
+                selectedFlow: stepResult.flowChanged || routingResult.selectedFlow,
+                selectedStep: stepResult.nextStep, // Use the determined next step
+                responseDirectives: routingResult.responseDirectives,
+                session: stepResult.session,
+                // If a branch changed the flow, the original isFlowComplete no longer applies
+                isFlowComplete: stepResult.flowChanged ? false : isFlowComplete,
+            };
         }
-        return {
-            session: updatedSession,
-            collectedData,
-        };
-    }
-    /**
-     * Handle context updates from structured response
-     */
-    async handleContextUpdate(structured) {
-        if (structured &&
-            typeof structured === "object" &&
-            "contextUpdate" in structured) {
-            await this.updateContext(structured
-                .contextUpdate);
+        catch (error) {
+            throw ResponseGenerationError.fromError(error, 'routing_optimization', params);
         }
     }
     /**
-     * Handle flow completion: pure state transition.
+     * RENDER-STEP RESOLUTION — resolve the step a flow response will render,
+     * shared by the streaming and non-streaming paths. When no step was
+     * pre-selected: branches win over the linear chain, then candidate steps,
+     * then the initial-step fallback. Enforces `requires` (stays at the current
+     * step when required fields are missing) and enters the resolved step.
      *
-     * Releases the session to idle (`currentFlow`/`currentStep` cleared),
-     * marks the active `flowHistory` entry completed, and (when `onComplete`
-     * is set) wires `pendingDirective` for the next turn. The framework
-     * emits no message of its own at the completion boundary; callers that
-     * need closing copy should add a final interactive step.
+     * Returns `flowTransition: true` when a branch resolved to a flow
+     * transition or completion — there is no local step to render and the
+     * caller handles the transition.
      */
-    async handleFlowCompletion(params) {
-        const { selectedFlow, session, context } = params;
-        // Check for onComplete transition
-        const transitionConfig = await selectedFlow.evaluateOnComplete({ data: session.data }, context);
-        // Release to idle. When the flow is reentrant, scrub its owned
-        // fields so a future re-selection doesn't short-circuit on stale data.
-        const ownedFields = selectedFlow.reentrant
-            ? [
-                ...(selectedFlow.requiredFields ?? []),
-                ...(selectedFlow.optionalFields ?? []),
-            ]
-            : undefined;
-        let updatedSession = completeCurrentFlow(session, {
-            clearOwnedFields: ownedFields,
-        });
-        if (transitionConfig) {
-            // Extract target from directive's goTo field
-            const goToTarget = typeof transitionConfig.goTo === 'string'
-                ? transitionConfig.goTo
-                : transitionConfig.goTo?.flow;
-            // Find target flow by ID or title
-            const targetFlow = goToTarget ? this.getFlows().find((r) => r.id === goToTarget ||
-                r.title === goToTarget) : undefined;
-            if (targetFlow) {
-                updatedSession = {
-                    ...updatedSession,
-                    pendingDirective: {
-                        goTo: targetFlow.id,
-                    },
-                };
-                logger.debug(`[ResponseHandler] Flow ${selectedFlow.title} completed with pending directive to: ${targetFlow.title}`);
-                return { session: updatedSession, hasTransition: true };
-            }
-            else if (goToTarget) {
-                logger.warn(`[FlowConfigurationError] onComplete target not found: flow "${selectedFlow.title}" completed but onComplete target "${goToTarget}" does not match any flow. ` +
-                    `Fix the onComplete value to reference an existing flow id/title, or remove onComplete to release the session to idle.`);
-            }
-        }
-        logger.debug(`[ResponseHandler] Flow ${selectedFlow.title} completed; session released to idle.`);
-        return { session: updatedSession, hasTransition: false };
-    }
-    /**
-     * Find an available tool by name for the given flow using ToolManager
-     * Delegates to ToolManager for unified tool resolution
-     */
-    findAvailableTool(toolName, flow) {
-        // Use ToolManager for unified tool resolution if available
-        if (this.toolManager) {
-            return this.toolManager.find(toolName, undefined, undefined, flow);
-        }
-        // Fallback to legacy resolution if ToolManager not available
-        logger.warn(`[ResponsePipeline] ToolManager not available, using legacy tool resolution for: ${toolName}`);
-        // Check flow-level tools first (if flow provided)
-        if (flow) {
-            const flowTool = flow
-                .getTools()
-                .find((tool) => tool.id === toolName || tool.id === toolName);
-            if (flowTool)
-                return flowTool;
-        }
-        // Fall back to agent-level tools
-        return this.tools.find((tool) => tool.id === toolName || tool.id === toolName);
-    }
-    /**
-     * Collect all available tools for the given flow and step context using ToolManager
-     * Delegates to ToolManager for unified tool resolution and deduplication
-     */
-    collectAvailableTools(flow, step) {
-        // Use ToolManager for unified tool collection if available
-        if (this.toolManager) {
-            const availableTools = this.toolManager.getAvailable(undefined, step, flow);
-            return availableTools.map((tool) => ({
-                id: tool.id,
-                description: tool.description,
-                parameters: tool.parameters,
-            }));
-        }
-        // Fallback to legacy collection logic if ToolManager not available
-        logger.warn(`[ResponsePipeline] ToolManager not available, using legacy tool collection`);
-        const availableTools = new Map();
-        // Add agent-level tools
-        this.tools.forEach((tool) => {
-            availableTools.set(tool.id, tool);
-        });
-        // Add flow-level tools (these take precedence)
-        if (flow) {
-            flow.getTools().forEach((tool) => {
-                availableTools.set(tool.id, tool);
-            });
+    async resolveRenderStep(params) {
+        const { selectedFlow, selectedStep, context } = params;
+        let session = params.session;
+        // Determine next step
+        let nextStep;
+        if (selectedStep) {
+            nextStep = selectedStep;
         }
-        // Filter by step-level allowed tools if specified
-        if (step?.tools) {
-            const allowedToolIds = new Set();
-            const stepTools = [];
-            for (const toolRef of step.tools) {
-                if (typeof toolRef === "string") {
-                    // Reference to registered tool
-                    allowedToolIds.add(toolRef);
+        else {
+            // Determine current step from session if we're already in this flow
+            const isInSameFlow = session.currentFlow?.id === selectedFlow.id;
+            const currentStep = isInSameFlow && session.currentStep
+                ? selectedFlow.getStep(session.currentStep.id)
+                : undefined;
+            logger.debug(`[ResponsePipeline] Step determination: flow match=${isInSameFlow}, currentFlow=${session.currentFlow?.id}, selectedFlow=${selectedFlow.id}, currentStep=${currentStep?.id || 'none'}`);
+            // STEP 1 (Algorithm 1): branches win over linear chain
+            if (currentStep?.branches && currentStep.branches.length > 0) {
+                const branchResult = await this.evaluateStepBranches(currentStep, selectedFlow, session, context);
+                if (branchResult) {
+                    if (branchResult.nextStep) {
+                        nextStep = branchResult.nextStep;
+                        session = branchResult.session;
+                    }
+                    else {
+                        // Flow transition or completion — no local step to render
+                        return { nextStep: undefined, session: branchResult.session, flowTransition: true };
+                    }
+                }
+            }
+            if (!nextStep) {
+                // Get candidate steps based on current position in the flow
+                const candidates = await this.flowRouter.getCandidateStepsWithConditions(selectedFlow, currentStep, // Pass current step instead of undefined to maintain progression
+                createTemplateContext({ data: session.data, session, context }));
+                logger.debug(`[ResponsePipeline] Found ${candidates.length} candidate steps${currentStep ? ' from current step ' + currentStep.id : ' (new flow entry)'}`);
+                if (candidates.length > 0) {
+                    nextStep = candidates[0].step;
+                    logger.debug(`[ResponsePipeline] Using first valid step: ${nextStep.id}${currentStep ? ' (progressing from ' + currentStep.id + ')' : ' for new flow'}`);
                 }
                 else {
-                    // Inline tool definition
-                    if (toolRef.id) {
-                        allowedToolIds.add(toolRef.id);
-                        stepTools.push(toolRef);
-                    }
+                    // Fallback to initial step even if it should be skipped
+                    nextStep = selectedFlow.initialStep;
+                    logger.warn(`[FlowConfigurationError] No valid steps found: all candidates were skipped in flow. Falling back to initial step "${nextStep.id}". Review step skip conditions.`);
                 }
             }
-            // If step specifies tools, only include those
-            if (allowedToolIds.size > 0) {
-                const filteredTools = new Map();
-                for (const toolId of Array.from(allowedToolIds)) {
-                    const tool = availableTools.get(toolId);
-                    if (tool) {
-                        filteredTools.set(toolId, tool);
+        }
+        // Update session with next step
+        // If the next step has requires fields that are missing, stay at the previous step
+        if (nextStep.requires && nextStep.requires.length > 0) {
+            const sessionData = session.data || {};
+            const missingRequires = nextStep.requires.filter(field => sessionData[String(field)] === undefined);
+            if (missingRequires.length > 0) {
+                const warning = `[FlowConfigurationError] Cannot advance to step "${nextStep.description || nextStep.id}": ` +
+                    `missing required fields [${missingRequires.join(', ')}]. Staying at current step. Ensure preceding steps collect these fields.`;
+                logger.warn(warning);
+                console.warn(warning);
+                // Stay at the current step - don't enter the next one
+                const currentStepId = session.currentStep?.id;
+                if (currentStepId) {
+                    const currentStepInstance = selectedFlow.getStep(currentStepId);
+                    if (currentStepInstance) {
+                        nextStep = currentStepInstance;
+                        logger.debug(`[ResponsePipeline] Staying at current step: ${nextStep.id} due to missing requires`);
                     }
                 }
-                // Add inline tools
-                stepTools.forEach((tool) => {
-                    if (tool.id) {
-                        filteredTools.set(tool.id, tool);
-                    }
-                });
-                availableTools.clear();
-                filteredTools.forEach((tool, id) => availableTools.set(id, tool));
+            }
+            else {
+                session = enterStep(session, nextStep.id, nextStep.description);
+                logger.debug(`[ResponsePipeline] Entered step: ${nextStep.id}`);
             }
         }
-        // Convert to the format expected by AI providers
-        return Array.from(availableTools.values()).map((tool) => ({
-            id: tool.id,
-            description: tool.description,
-            parameters: tool.parameters,
-        }));
-    }
-    /**
-     * Get current context (fetches from provider if configured)
-     */
-    async getContext() {
-        // If context provider is configured, use it to fetch fresh context
-        if (this.options.contextProvider) {
-            return await this.options.contextProvider();
+        else {
+            session = enterStep(session, nextStep.id, nextStep.description);
+            logger.debug(`[ResponsePipeline] Entered step: ${nextStep.id}`);
         }
-        // Otherwise return the stored context
-        return this.context;
-    }
-    // Setters for context and current session (needed for beforeRespond hook)
-    setContext(context) {
-        this.context = context;
-    }
-    setCurrentSession(session) {
-        this.currentSession = session;
-    }
-    getStoredContext() {
-        return this.context;
-    }
-    getCurrentSession() {
-        return this.currentSession;
+        return { nextStep, session, flowTransition: false };
     }
     /**
-     * Handle cross-flow completion evaluation and notifications
-     * This method evaluates all flows for completion and can trigger completion handlers
+     * Routing skip optimization (Requirements 20.1, 20.2, 20.3):
+     * When the current step declares `collect` fields AND pre-extraction populates
+     * at least one of those fields from the user's message, skip routing for this turn.
+     *
+     * Returns the routing result if the skip applies, or undefined to fall through
+     * to normal routing.
      */
-    async handleCrossFlowCompletion(params) {
-        const { flows, session, context } = params;
-        // Evaluate all flows for completion
-        const completedFlows = [];
-        const pendingTransitions = [];
-        for (const flow of flows) {
-            if (flow.isComplete(session.data || {})) {
-                completedFlows.push(flow);
-                // Check for onComplete transitions
-                const transitionConfig = await flow.evaluateOnComplete({ data: session.data }, context);
-                if (transitionConfig) {
-                    pendingTransitions.push({ flow, transitionConfig });
-                }
-                logger.debug(`[ResponsePipeline] Flow completed: ${flow.title} ` +
-                    `(${Math.round(flow.getCompletionProgress(session.data || {}) * 100)}%)`);
+    async attemptRoutingSkipForCollect(params) {
+        const { session } = params;
+        // Only applies when we already have a current flow and step
+        if (!session.currentFlow || !session.currentStep) {
+            return undefined;
+        }
+        // Also skip this optimization if there's a pending directive (it takes priority)
+        if (session.pendingDirective) {
+            return undefined;
+        }
+        // Look up the actual Flow and Step objects to access `collect`
+        const currentFlow = this.getFlows().find((f) => f.id === session.currentFlow?.id);
+        if (!currentFlow) {
+            return undefined;
+        }
+        const currentStep = currentFlow.getStep(session.currentStep.id);
+        if (!currentStep || !currentStep.collect || currentStep.collect.length === 0) {
+            return undefined;
+        }
+        // We have a step with collect fields. Run pre-extraction to see if the
+        // user's message populates any of them.
+        const collectFields = currentStep.collect;
+        // Snapshot current data for comparison
+        const dataBefore = { ...session.data };
+        // Run pre-extraction against the current flow
+        const extractedData = await this.preExtractFlowData({
+            route: currentFlow,
+            history: params.history,
+            context: params.context,
+            session,
+            signal: params.signal,
+        });
+        if (!extractedData || Object.keys(extractedData).length === 0) {
+            return undefined;
+        }
+        // Determine which collect fields were newly populated by pre-extraction
+        const populatedCollectFields = [];
+        for (const field of collectFields) {
+            const key = field;
+            const hadValue = dataBefore[field] !== undefined && dataBefore[field] !== null;
+            const hasNewValue = extractedData[field] !== undefined && extractedData[field] !== null;
+            if (hasNewValue && !hadValue) {
+                populatedCollectFields.push(key);
             }
         }
-        // Log completion status for all flows
-        if (completedFlows.length > 0) {
-            logger.debug(`[ResponsePipeline] Cross-flow completion evaluation: ` +
-                `${completedFlows.length}/${flows.length} flows complete`);
-        }
+        if (populatedCollectFields.length === 0) {
+            // Pre-extraction didn't populate any declared collect field — no skip
+            return undefined;
+        }
+        // ROUTING SKIP: pre-extraction populated collect fields → retain current flow/step
+        logger.debug(`[ResponsePipeline] Routing skip: pre-extraction populated collect fields [${populatedCollectFields.join(', ')}] for step "${currentStep.id}" — skipping FlowRouter`);
+        // Merge extracted data into session
+        const updatedSession = mergeCollected(session, extractedData);
+        await this.updateCollectedData(extractedData);
+        // Determine next step using pipeline method for consistency
+        // Pass the current flow/step as the routing result (retained)
+        const stepResult = await this.determineNextStep({
+            selectedFlow: currentFlow,
+            selectedStep: currentStep,
+            session: updatedSession,
+            isFlowComplete: false,
+            context: params.context,
+        });
         return {
-            session,
-            completedFlows,
-            pendingTransitions,
+            selectedFlow: stepResult.flowChanged || currentFlow,
+            selectedStep: stepResult.nextStep,
+            responseDirectives: undefined,
+            session: stepResult.session,
+            isFlowComplete: false,
         };
     }
     /**
-     * Update data flow to ensure agent-level data consistency
-     * This method ensures that data updates are properly validated and propagated
+     * Check if a flow should pre-extract data before determining the initial step
      */
-    async updateDataFlow(params) {
-        const { session, dataUpdate, flows } = params;
-        // Update session data
-        const updatedSession = await this.updateData(session, dataUpdate);
-        // Update agent-level data if handler is available
-        if (this.updateCollectedData) {
-            await this.updateCollectedData(dataUpdate);
-        }
-        // Evaluate flow completions after data update
-        const completionResults = await this.handleCrossFlowCompletion({
-            flows,
-            session: updatedSession,
-            context: this.context,
-            history: [],
-        });
-        // Log any newly completed flows
-        if (completionResults.completedFlows.length > 0) {
-            logger.debug(`[ResponsePipeline] Data update resulted in ${completionResults.completedFlows.length} completed flows`);
+    shouldPreExtractData(flow) {
+        // Pre-extract if flow has declared required or optional fields
+        if (flow.requiredFields && flow.requiredFields.length > 0) {
+            return true;
+        }
+        if (flow.optionalFields && flow.optionalFields.length > 0) {
+            return true;
+        }
+        // Pre-extract if any step in the flow collects data
+        const steps = flow.getAllSteps();
+        const hasDataCollectionSteps = steps.some(step => step.collect && step.collect.length > 0);
+        return hasDataCollectionSteps;
+    }
+    /**
+     * Pre-extract data from user message when entering a flow
+     * This allows skipping steps whose data is already provided
+     */
+    async preExtractFlowData(params) {
+        const { route: flow, history, signal } = params;
+        // Build a schema for data extraction based on flow's fields
+        const extractionSchema = this.getSchema();
+        if (!extractionSchema) {
+            logger.warn(`[ResponsePipeline] No schema available for pre-extraction`);
+            return {};
+        }
+        // Get last user message
+        const lastMessage = getLastMessageFromHistory(history);
+        // Build extraction prompt
+        const extractionPrompt = [
+            `Extract any relevant information from the user's message that matches the following data fields.`,
+            `Only extract information that is explicitly stated or clearly implied.`,
+            ``,
+            `User's message: "${lastMessage}"`,
+            ``,
+            `Extract data for these fields if present:`,
+        ];
+        // Add field descriptions
+        if (flow.requiredFields) {
+            extractionPrompt.push(`Required fields: ${flow.requiredFields.join(', ')}`);
+        }
+        if (flow.optionalFields) {
+            extractionPrompt.push(`Optional fields: ${flow.optionalFields.join(', ')}`);
+        }
+        extractionPrompt.push(``, `Return ONLY the extracted data as JSON. If no data can be extracted, return an empty object {}.`);
+        // Convert Event[] to HistoryItem[] for provider call
+        const historyItems = eventsToHistory(history);
+        // Call AI to extract data
+        try {
+            const result = await this.options.provider.generateMessage({
+                prompt: extractionPrompt.join('\n'),
+                history: historyItems,
+                context: {}, // Passed as empty object so AI doesn't "extract" from context
+                // NOTE: context is intentionally NOT passed here.
+                // Passing context caused the AI to "extract" data from the lead's context
+                // (e.g., name, sector, city) instead of from what the user actually said.
+                signal,
+                parameters: {
+                    jsonSchema: extractionSchema,
+                    schemaName: 'data_extraction',
+                },
+            });
+            return result.structured || {};
+        }
+        catch (error) {
+            logger.error(`[ResponsePipeline] Pre-extraction failed:`, error);
+            return {};
         }
-        return completionResults.session;
     }
 }
 //# sourceMappingURL=ResponsePipeline.js.map