@falai/agent 0.9.2 → 1.0.1

package/dist/{src/core → core}/ResponseModal.js

@@ -6,7 +6,10 @@ import { EventKind, MessageRole } from "../types";
 import { Step } from "./Step";
 import { ResponseEngine } from "./ResponseEngine";
 import { ResponsePipeline } from "./ResponsePipeline";
+import { BatchExecutor } from "./BatchExecutor";
+import { BatchPromptBuilder } from "./BatchPromptBuilder";
 import { cloneDeep, mergeCollected, enterStep, getLastMessageFromHistory, render, logger, historyToEvents } from "../utils";
+import { createTemplateContext } from "../utils/template";
 import { END_ROUTE_ID } from "../constants";
 /**
  * Error class for response generation failures
@@ -48,6 +51,10 @@ export class ResponseModal {
         // Initialize response pipeline with agent dependencies
         this.responsePipeline = new ResponsePipeline(this.agent.getAgentOptions(), () => this.agent.getRoutes(), // Pass a function to get routes dynamically
         this.agent.getTools(), this.agent.getRoutingEngine(), this.agent.updateContext.bind(this.agent), this.agent.getUpdateDataMethod(), this.agent.updateCollectedData.bind(this.agent), this.getToolManager());
+        // Initialize batch executor for multi-step execution
+        this.batchExecutor = new BatchExecutor();
+        // Initialize batch prompt builder for combined prompts
+        this.batchPromptBuilder = new BatchPromptBuilder();
     }
     /**
      * Generate a non-streaming response using unified logic
@@ -268,6 +275,7 @@ export class ResponseModal {
                 throw ResponseGenerationError.fromError(error, 'step_preparation', params, { session, effectiveContext });
             }
             // PHASE 2: ROUTING + STEP SELECTION - Determine which route and step to use
+            // Also performs pre-extraction and batch determination
             let routingResult;
             try {
                 routingResult = await this.handleUnifiedRoutingAndStepSelection({
@@ -288,6 +296,9 @@ export class ResponseModal {
                 selectedStep: routingResult.selectedStep,
                 responseDirectives: routingResult.responseDirectives,
                 isRouteComplete: routingResult.isRouteComplete,
+                batchSteps: routingResult.batchSteps,
+                batchStoppedReason: routingResult.batchStoppedReason,
+                batchStoppedAtStep: routingResult.batchStoppedAtStep,
             };
         }
         catch (error) {
@@ -313,31 +324,156 @@ export class ResponseModal {
                 context: params.context,
                 signal: params.signal,
             });
+            let updatedSession = routingResult.session;
+            let isRouteComplete = routingResult.isRouteComplete;
+            // PRE-EXTRACTION: If entering a route that collects data, extract data from user message first
+            // This allows us to skip steps whose data is already provided
+            // Requirement 3.1: Perform Pre_Extraction before determining the Batch
+            if (routingResult.selectedRoute && !isRouteComplete) {
+                // Always pre-extract when route collects data (not just on new route entry)
+                // This ensures batch determination has the most up-to-date data
+                if (this.shouldPreExtractData(routingResult.selectedRoute)) {
+                    logger.debug(`[ResponseModal] Pre-extracting data for route: ${routingResult.selectedRoute.title}`);
+                    const extractedData = await this.preExtractRouteData({
+                        route: routingResult.selectedRoute,
+                        history: params.history,
+                        context: params.context,
+                        session: updatedSession,
+                        signal: params.signal,
+                    });
+                    if (extractedData && Object.keys(extractedData).length > 0) {
+                        logger.debug(`[ResponseModal] Pre-extracted data:`, extractedData);
+                        // Requirement 3.3: Merge pre-extracted data into session before batch determination
+                        updatedSession = mergeCollected(updatedSession, extractedData);
+                        // Also update agent's collected data
+                        await this.agent.updateCollectedData(extractedData);
+                        // Re-check route completion after pre-extraction
+                        const allRequiredFieldsCollected = routingResult.selectedRoute.isComplete(updatedSession.data || {});
+                        if (allRequiredFieldsCollected) {
+                            logger.debug(`[ResponseModal] Route ${routingResult.selectedRoute.title} completed after pre-extraction`);
+                            isRouteComplete = true;
+                        }
+                    }
+                }
+            }
+            // BATCH DETERMINATION: Use BatchExecutor to determine which steps can execute together
+            // Requirement 3.4: Pre-extraction results affect batch determination
+            let batchSteps;
+            let batchStoppedReason;
+            let batchStoppedAtStep;
+            if (routingResult.selectedRoute && !isRouteComplete) {
+                // Determine current step position for batch determination
+                const currentStep = routingResult.selectedStep ||
+                    (updatedSession.currentStep ? routingResult.selectedRoute.getStep(updatedSession.currentStep.id) : undefined);
+                logger.debug(`[ResponseModal] Determining batch starting from step: ${currentStep?.id || 'initial'}`);
+                const batchResult = await this.batchExecutor.determineBatch({
+                    route: routingResult.selectedRoute,
+                    currentStep,
+                    sessionData: updatedSession.data || {},
+                    context: params.context,
+                });
+                batchSteps = batchResult.steps;
+                batchStoppedReason = batchResult.stoppedReason;
+                batchStoppedAtStep = batchResult.stoppedAtStep;
+                logger.debug(`[ResponseModal] Batch determined: ${batchSteps.length} steps, stopped reason: ${batchStoppedReason}`);
+            }
             // Determine next step using pipeline method for consistency
-            const stepResult = this.responsePipeline.determineNextStep({
+            const stepResult = await this.responsePipeline.determineNextStep({
                 selectedRoute: routingResult.selectedRoute,
                 selectedStep: routingResult.selectedStep,
-                session: routingResult.session,
-                isRouteComplete: routingResult.isRouteComplete,
+                session: updatedSession, // Use updated session with pre-extracted data
+                isRouteComplete, // Use updated completion status
             });
             return {
                 selectedRoute: routingResult.selectedRoute,
                 selectedStep: stepResult.nextStep, // Use the determined next step
                 responseDirectives: routingResult.responseDirectives,
                 session: stepResult.session,
-                isRouteComplete: routingResult.isRouteComplete,
+                isRouteComplete, // Use updated completion status
+                batchSteps,
+                batchStoppedReason,
+                batchStoppedAtStep,
             };
         }
         catch (error) {
             throw ResponseGenerationError.fromError(error, 'routing_optimization', params);
         }
     }
+    /**
+     * Check if a route should pre-extract data before determining the initial step
+     * @private
+     */
+    shouldPreExtractData(route) {
+        // Pre-extract if route has declared required or optional fields
+        if (route.requiredFields && route.requiredFields.length > 0) {
+            return true;
+        }
+        if (route.optionalFields && route.optionalFields.length > 0) {
+            return true;
+        }
+        // Pre-extract if any step in the route collects data
+        const steps = route.getAllSteps();
+        const hasDataCollectionSteps = steps.some(step => step.collect && step.collect.length > 0);
+        return hasDataCollectionSteps;
+    }
+    /**
+     * Pre-extract data from user message when entering a route
+     * This allows skipping steps whose data is already provided
+     * @private
+     */
+    async preExtractRouteData(params) {
+        const { route, history, context, signal } = params;
+        // Build a schema for data extraction based on route's fields
+        const extractionSchema = this.agent.getSchema();
+        if (!extractionSchema) {
+            logger.warn(`[ResponseModal] No schema available for pre-extraction`);
+            return {};
+        }
+        // Get last user message
+        const lastMessage = getLastMessageFromHistory(history);
+        // Build extraction prompt
+        const extractionPrompt = [
+            `Extract any relevant information from the user's message that matches the following data fields.`,
+            `Only extract information that is explicitly stated or clearly implied.`,
+            ``,
+            `User's message: "${lastMessage}"`,
+            ``,
+            `Extract data for these fields if present:`,
+        ];
+        // Add field descriptions
+        if (route.requiredFields) {
+            extractionPrompt.push(`Required fields: ${route.requiredFields.join(', ')}`);
+        }
+        if (route.optionalFields) {
+            extractionPrompt.push(`Optional fields: ${route.optionalFields.join(', ')}`);
+        }
+        extractionPrompt.push(``, `Return ONLY the extracted data as JSON. If no data can be extracted, return an empty object {}.`);
+        // Call AI to extract data
+        const agentOptions = this.agent.getAgentOptions();
+        try {
+            const result = await agentOptions.provider.generateMessage({
+                prompt: extractionPrompt.join('\n'),
+                history,
+                context,
+                signal,
+                parameters: {
+                    jsonSchema: extractionSchema,
+                    schemaName: 'data_extraction',
+                },
+            });
+            return result.structured || {};
+        }
+        catch (error) {
+            logger.error(`[ResponseModal] Pre-extraction failed:`, error);
+            return {};
+        }
+    }
     /**
      * Unified response generation for non-streaming responses
      * @private
      */
     async generateUnifiedResponse(responseContext) {
-        const { effectiveContext, session: initialSession, history, selectedRoute, selectedStep, responseDirectives, isRouteComplete } = responseContext;
+        const { effectiveContext, session: initialSession, history, selectedRoute, selectedStep, responseDirectives, isRouteComplete, batchSteps, batchStoppedReason, } = responseContext;
         let session = initialSession;
         // Get last user message (needed for both route and completion handling)
         // Convert HistoryItem[] to Event[] for internal processing
@@ -345,35 +481,79 @@ export class ResponseModal {
         const lastMessageText = getLastMessageFromHistory(historyEvents);
         let message;
         let toolCalls = undefined;
+        let executedSteps;
+        let stoppedReason;
         if (selectedRoute && !isRouteComplete) {
-            // Handle normal route processing
-            const result = await this.processRouteResponse({
-                selectedRoute,
-                selectedStep,
-                responseDirectives,
-                session,
-                history,
-                context: effectiveContext,
-                lastMessageText,
-                historyEvents,
-                signal: responseContext.history ? undefined : undefined, // TODO: Fix signal passing
-            });
-            message = result.message;
-            toolCalls = result.toolCalls;
-            session = result.session;
+            // Check if we have batch steps to execute
+            if (batchSteps && batchSteps.length > 0) {
+                // BATCH EXECUTION: Execute multiple steps in a single LLM call
+                logger.debug(`[ResponseModal] Executing batch of ${batchSteps.length} steps`);
+                const batchResult = await this.executeBatchResponse({
+                    selectedRoute,
+                    batchSteps,
+                    responseDirectives,
+                    session,
+                    history,
+                    context: effectiveContext,
+                    historyEvents,
+                });
+                message = batchResult.message;
+                toolCalls = batchResult.toolCalls;
+                session = batchResult.session;
+                executedSteps = batchResult.executedSteps;
+                stoppedReason = batchStoppedReason;
+            }
+            else {
+                // SINGLE STEP EXECUTION: Fall back to single-step processing
+                // This happens when batch determination returns empty (first step needs input)
+                const result = await this.processRouteResponse({
+                    selectedRoute,
+                    selectedStep,
+                    responseDirectives,
+                    session,
+                    history,
+                    context: effectiveContext,
+                    lastMessageText,
+                    historyEvents,
+                    signal: undefined,
+                });
+                message = result.message;
+                toolCalls = result.toolCalls;
+                session = result.session;
+                // Track executed step for single-step execution
+                if (selectedStep) {
+                    executedSteps = [{
+                            id: selectedStep.id,
+                            routeId: selectedRoute.id,
+                        }];
+                }
+                stoppedReason = batchStoppedReason || 'needs_input';
+            }
         }
         else if (isRouteComplete && selectedRoute) {
             // Handle route completion
-            message = await this.handleRouteCompletion({
-                selectedRoute,
-                session,
-                context: effectiveContext,
-                lastMessageText,
-                historyEvents,
-            });
-            // Set step to END_ROUTE marker
-            session = enterStep(session, END_ROUTE_ID, "Route completed");
-            logger.debug(`[ResponseModal] Route ${selectedRoute.title} completed. Entered END_ROUTE step.`);
+            logger.debug(`[ResponseModal] Generating completion message for route: ${selectedRoute.title}`);
+            try {
+                message = await this.handleRouteCompletion({
+                    selectedRoute,
+                    session,
+                    context: effectiveContext,
+                    lastMessageText,
+                    historyEvents,
+                    signal: undefined,
+                });
+                // Set step to END_ROUTE marker
+                session = enterStep(session, END_ROUTE_ID, "Route completed");
+                stoppedReason = 'route_complete';
+                logger.debug(`[ResponseModal] Route ${selectedRoute.title} completed. Entered END_ROUTE step.`);
+            }
+            catch (error) {
+                logger.error(`[ResponseModal] Error generating completion message:`, error);
+                // Fallback to simple completion message
+                message = `Thank you! I've recorded all the information for your ${selectedRoute.title.toLowerCase()}.`;
+                session = enterStep(session, END_ROUTE_ID, "Route completed");
+                stoppedReason = 'route_complete';
+            }
         }
         else {
             // Fallback: No routes defined, generate a simple response
@@ -382,37 +562,275 @@ export class ResponseModal {
                 context: effectiveContext,
                 session,
             });
+            // For fallback responses, set empty executedSteps and no stoppedReason
+            // since there's no route/step execution happening
+            executedSteps = [];
+            stoppedReason = undefined;
         }
+        // Ensure response structure completeness (Requirement 8.1, 8.2, 8.3)
+        // - executedSteps: array of steps executed (empty array if none)
+        // - stoppedReason: why execution stopped (undefined for fallback)
+        // - session.currentStep: reflects final step position
         return {
             message,
             session,
             toolCalls,
             isRouteComplete,
+            executedSteps: executedSteps || [],
+            stoppedReason,
+        };
+    }
+    /**
+     * Execute a batch of steps with a single LLM call
+     *
+     * This method:
+     * 1. Executes all prepare hooks for steps in the batch (in order)
+     * 2. Builds a combined prompt using BatchPromptBuilder
+     * 3. Makes a single LLM call
+     * 4. Collects data from the response for all steps
+     * 5. Executes all finalize hooks for steps in the batch (in order)
+     *
+     * @private
+     * **Validates: Requirements 1.1, 4.4, 5.1, 5.2**
+     */
+    async executeBatchResponse(params) {
+        const { selectedRoute, batchSteps, history, context, historyEvents, signal } = params;
+        let session = params.session;
+        logger.debug(`[ResponseModal] Starting batch execution for ${batchSteps.length} steps`);
+        // Create hook executor function
+        const executeHook = async (hook, hookContext, data, step) => {
+            // Find the route for this step
+            const route = selectedRoute;
+            // Convert StepOptions to Step if needed for executePrepareFinalize
+            const stepInstance = step?.id ? route.getStep(step.id) : undefined;
+            await this.executePrepareFinalize(hook, hookContext, data, route, stepInstance);
+        };
+        // PHASE 1: Execute all prepare hooks (Requirement 5.1)
+        logger.debug(`[ResponseModal] Executing prepare hooks for batch`);
+        const prepareResult = await this.batchExecutor.executePrepareHooks({
+            steps: batchSteps,
+            context,
+            data: session.data,
+            executeHook,
+        });
+        if (!prepareResult.success) {
+            // Prepare hook failed - return error response
+            logger.error(`[ResponseModal] Prepare hook failed:`, prepareResult.error);
+            throw new ResponseGenerationError(`Prepare hook failed: ${prepareResult.error?.message}`, {
+                phase: 'prepare_hooks',
+                context: {
+                    stepId: prepareResult.error?.stepId,
+                    executedSteps: prepareResult.executedSteps,
+                }
+            });
+        }
+        // PHASE 2: Build combined prompt using BatchPromptBuilder (Requirement 4.4)
+        logger.debug(`[ResponseModal] Building batch prompt`);
+        const batchPromptResult = await this.batchPromptBuilder.buildBatchPrompt({
+            steps: batchSteps,
+            route: selectedRoute,
+            history: historyEvents,
+            context,
+            session,
+            agentOptions: this.agent.getAgentOptions(),
+        });
+        logger.debug(`[ResponseModal] Batch prompt built with ${batchPromptResult.stepCount} steps, collecting: ${batchPromptResult.collectFields.join(', ')}`);
+        // Build response schema for batch (includes all collect fields)
+        const responseSchema = this.buildBatchResponseSchema(batchPromptResult.collectFields);
+        // Collect available tools for AI (from all steps in batch)
+        const availableTools = this.collectBatchAvailableTools(selectedRoute, batchSteps);
+        // PHASE 3: Make single LLM call (Requirement 4.4)
+        logger.debug(`[ResponseModal] Making LLM call for batch`);
+        const agentOptions = this.agent.getAgentOptions();
+        const result = await agentOptions.provider.generateMessage({
+            prompt: batchPromptResult.prompt,
+            history: historyEvents,
+            context,
+            tools: availableTools,
+            signal,
+            parameters: responseSchema ? { jsonSchema: responseSchema, schemaName: "batch_response" } : undefined,
+        });
+        let message = result.structured?.message || result.message;
+        let toolCalls = result.structured?.toolCalls;
+        logger.debug(`[ResponseModal] LLM response received for batch`);
+        // Execute tools if any
+        if (toolCalls && toolCalls.length > 0) {
+            const toolResult = await this.executeUnifiedToolLoop({
+                toolCalls,
+                context,
+                session,
+                history,
+                selectedRoute,
+                responsePrompt: batchPromptResult.prompt,
+                availableTools,
+                responseSchema,
+                signal,
+            });
+            session = toolResult.session;
+            toolCalls = toolResult.finalToolCalls;
+            if (toolResult.finalMessage) {
+                message = toolResult.finalMessage;
+            }
+        }
+        // PHASE 4: Collect data from response for all steps (Requirement 6.1, 6.2, 6.3)
+        logger.debug(`[ResponseModal] Collecting batch data`);
+        const collectResult = this.batchExecutor.collectBatchData({
+            steps: batchSteps,
+            llmResponse: result.structured || {},
+            session,
+            schema: this.agent.getSchema(),
+        });
+        session = collectResult.session;
+        if (collectResult.collectedData && Object.keys(collectResult.collectedData).length > 0) {
+            // Update agent's collected data
+            await this.agent.updateCollectedData(collectResult.collectedData);
+            logger.debug(`[ResponseModal] Batch collected data:`, collectResult.collectedData);
+        }
+        if (collectResult.validationErrors && collectResult.validationErrors.length > 0) {
+            logger.warn(`[ResponseModal] Batch data validation errors:`, collectResult.validationErrors);
+        }
+        // Update session to final step position
+        const lastStep = batchSteps[batchSteps.length - 1];
+        if (lastStep?.id) {
+            session = enterStep(session, lastStep.id, lastStep.description);
+            logger.debug(`[ResponseModal] Updated session to final batch step: ${lastStep.id}`);
+        }
+        // PHASE 5: Execute all finalize hooks (Requirement 5.2)
+        logger.debug(`[ResponseModal] Executing finalize hooks for batch`);
+        const finalizeResult = await this.batchExecutor.executeFinalizeHooks({
+            steps: batchSteps,
+            context,
+            data: session.data,
+            executeHook,
+        });
+        if (finalizeResult.errors && finalizeResult.errors.length > 0) {
+            // Log finalize errors but don't fail (Requirement 5.5)
+            logger.warn(`[ResponseModal] Some finalize hooks failed:`, finalizeResult.errors);
+        }
+        // Build executed steps list
+        const executedSteps = batchSteps
+            .filter(step => step.id)
+            .map(step => ({
+            id: step.id,
+            routeId: selectedRoute.id,
+        }));
+        logger.debug(`[ResponseModal] Batch execution complete. Executed ${executedSteps.length} steps`);
+        return {
+            message,
+            toolCalls,
+            session,
+            executedSteps,
+        };
+    }
+    /**
+     * Build response schema for batch execution
+     * @private
+     */
+    buildBatchResponseSchema(collectFields) {
+        const properties = {
+            message: {
+                type: "string",
+                description: "Your response to the user",
+            },
+        };
+        const agentSchema = this.agent.getSchema();
+        // Add collect fields to schema, using agent schema definitions when available
+        for (const field of collectFields) {
+            if (agentSchema?.properties && agentSchema.properties[field]) {
+                properties[field] = agentSchema.properties[field];
+            }
+            else {
+                // Dynamic fallback when no agent schema is defined
+                properties[field] = {
+                    type: "string",
+                    description: `Collected value for ${field}`,
+                };
+            }
+        }
+        return {
+            type: "object",
+            properties,
+            required: ["message"],
+            additionalProperties: true,
         };
     }
+    /**
+     * Collect available tools from all steps in the batch
+     * @private
+     */
+    collectBatchAvailableTools(route, batchSteps) {
+        const availableTools = new Map();
+        // Add agent-level tools
+        this.agent.getTools().forEach((tool) => {
+            availableTools.set(tool.id, tool);
+        });
+        // Add route-level tools
+        route.getTools().forEach((tool) => {
+            availableTools.set(tool.id, tool);
+        });
+        // Add step-level tools from all batch steps
+        for (const step of batchSteps) {
+            if (step.tools) {
+                for (const toolRef of step.tools) {
+                    if (typeof toolRef === "string") {
+                        // Reference to registered tool - already in availableTools
+                    }
+                    else if (typeof toolRef === 'object' && 'id' in toolRef && toolRef.id) {
+                        // Inline tool definition
+                        availableTools.set(toolRef.id, toolRef);
+                    }
+                }
+            }
+        }
+        // Convert to the format expected by AI providers
+        return Array.from(availableTools.values()).map((tool) => ({
+            id: tool.id,
+            name: tool.name || tool.id,
+            description: tool.description,
+            parameters: tool.parameters,
+        }));
+    }
     /**
      * Unified streaming response generation
      * @private
      */
     async *generateUnifiedStreamingResponse(responseContext) {
-        const { effectiveContext, session: initialSession, history, selectedRoute, selectedStep, responseDirectives, isRouteComplete } = responseContext;
+        const { effectiveContext, session: initialSession, history, selectedRoute, selectedStep, responseDirectives, isRouteComplete, batchSteps, batchStoppedReason, } = responseContext;
         const session = initialSession;
         // Get last user message (needed for both route and completion handling)
         // Convert HistoryItem[] to Event[] for internal processing
         const historyEvents = historyToEvents(history);
         const lastMessageText = getLastMessageFromHistory(historyEvents);
         if (selectedRoute && !isRouteComplete) {
-            // Handle normal route processing with streaming
-            yield* this.processRouteStreamingResponse({
-                selectedRoute,
-                selectedStep,
-                responseDirectives,
-                session,
-                history,
-                context: effectiveContext,
-                lastMessageText,
-                historyEvents,
-            });
+            // Check if we have batch steps to execute
+            if (batchSteps && batchSteps.length > 0) {
+                // BATCH EXECUTION: Execute multiple steps with streaming
+                // Note: For streaming, we still use batch execution but stream the response
+                logger.debug(`[ResponseModal] Streaming batch execution for ${batchSteps.length} steps`);
+                yield* this.streamBatchResponse({
+                    selectedRoute,
+                    batchSteps,
+                    responseDirectives,
+                    session,
+                    history,
+                    context: effectiveContext,
+                    historyEvents,
+                    batchStoppedReason,
+                });
+            }
+            else {
+                // SINGLE STEP EXECUTION: Fall back to single-step streaming
+                yield* this.processRouteStreamingResponse({
+                    selectedRoute,
+                    selectedStep,
+                    responseDirectives,
+                    session,
+                    history,
+                    context: effectiveContext,
+                    lastMessageText,
+                    historyEvents,
+                });
+            }
         }
         else if (isRouteComplete && selectedRoute) {
             // Handle route completion streaming
@@ -433,6 +851,114 @@ export class ResponseModal {
             });
         }
     }
+    /**
+     * Stream a batch response with multiple steps
+     *
+     * Similar to executeBatchResponse but streams the LLM response.
+     *
+     * @private
+     */
+    async *streamBatchResponse(params) {
+        const { selectedRoute, batchSteps, context, historyEvents, batchStoppedReason, signal } = params;
+        let session = params.session;
+        // Create hook executor function
+        const executeHook = async (hook, hookContext, data, step) => {
+            const route = selectedRoute;
+            const stepInstance = step?.id ? route.getStep(step.id) : undefined;
+            await this.executePrepareFinalize(hook, hookContext, data, route, stepInstance);
+        };
+        // PHASE 1: Execute all prepare hooks
+        const prepareResult = await this.batchExecutor.executePrepareHooks({
+            steps: batchSteps,
+            context,
+            data: session.data,
+            executeHook,
+        });
+        if (!prepareResult.success) {
+            // Yield error chunk
+            yield {
+                delta: "",
+                accumulated: "",
+                done: true,
+                session,
+                error: new ResponseGenerationError(`Prepare hook failed: ${prepareResult.error?.message}`, { phase: 'prepare_hooks' }),
+            };
+            return;
+        }
+        // PHASE 2: Build combined prompt
+        const batchPromptResult = await this.batchPromptBuilder.buildBatchPrompt({
+            steps: batchSteps,
+            route: selectedRoute,
+            history: historyEvents,
+            context,
+            session,
+            agentOptions: this.agent.getAgentOptions(),
+        });
+        const responseSchema = this.buildBatchResponseSchema(batchPromptResult.collectFields);
+        const availableTools = this.collectBatchAvailableTools(selectedRoute, batchSteps);
+        // PHASE 3: Stream LLM response
+        const agentOptions = this.agent.getAgentOptions();
+        const stream = agentOptions.provider.generateMessageStream({
+            prompt: batchPromptResult.prompt,
+            history: historyEvents,
+            context,
+            tools: availableTools,
+            signal,
+            parameters: responseSchema ? { jsonSchema: responseSchema, schemaName: "batch_stream_response" } : undefined,
+        });
+        // Build executed steps list
+        const executedSteps = batchSteps
+            .filter(step => step.id)
+            .map(step => ({
+            id: step.id,
+            routeId: selectedRoute.id,
+        }));
+        // Stream chunks
+        for await (const chunk of stream) {
+            // On final chunk, collect data and execute finalize hooks
+            if (chunk.done) {
+                // Collect data from response
+                if (chunk.structured) {
+                    const collectResult = this.batchExecutor.collectBatchData({
+                        steps: batchSteps,
+                        llmResponse: chunk.structured,
+                        session,
+                        schema: this.agent.getSchema(),
+                    });
+                    session = collectResult.session;
+                    if (collectResult.collectedData && Object.keys(collectResult.collectedData).length > 0) {
+                        await this.agent.updateCollectedData(collectResult.collectedData);
+                    }
+                }
+                // Update session to final step position
+                const lastStep = batchSteps[batchSteps.length - 1];
+                if (lastStep?.id) {
+                    session = enterStep(session, lastStep.id, lastStep.description);
+                }
+                // Execute finalize hooks
+                await this.batchExecutor.executeFinalizeHooks({
+                    steps: batchSteps,
+                    context,
+                    data: session.data,
+                    executeHook,
+                });
+                // Finalize session
+                await this.finalizeSession(session, context);
+            }
+            yield {
+                delta: chunk.delta,
+                accumulated: chunk.accumulated,
+                done: chunk.done,
+                session,
+                toolCalls: chunk.structured?.toolCalls,
+                isRouteComplete: false,
+                executedSteps: chunk.done ? executedSteps : undefined,
+                stoppedReason: chunk.done ? batchStoppedReason : undefined,
+                metadata: chunk.metadata,
+                structured: chunk.structured,
+            };
+        }
+    }
     /**
        * Execute prepare function for current step if available
        * @private
@@ -478,12 +1004,20 @@ export class ResponseModal {
             nextStep = selectedStep;
         }
         else {
-            // New route or no step selected - get initial step or first valid step
+            // Determine current step from session if we're already in this route
+            const isInSameRoute = session.currentRoute?.id === selectedRoute.id;
+            const currentStep = isInSameRoute && session.currentStep
+                ? selectedRoute.getStep(session.currentStep.id)
+                : undefined;
+            logger.debug(`[ResponseModal] Step determination: route match=${isInSameRoute}, currentRoute=${session.currentRoute?.id}, selectedRoute=${selectedRoute.id}, currentStep=${currentStep?.id || 'none'}`);
+            // Get candidate steps based on current position in the route
             const routingEngine = this.agent.getRoutingEngine();
-            const candidates = routingEngine.getCandidateSteps(selectedRoute, undefined, session.data || {});
+            const candidates = await routingEngine.getCandidateStepsWithConditions(selectedRoute, currentStep, // Pass current step instead of undefined to maintain progression
+            createTemplateContext({ data: session.data, session, context }));
+            logger.debug(`[ResponseModal] Found ${candidates.length} candidate steps${currentStep ? ' from current step ' + currentStep.id : ' (new route entry)'}`);
             if (candidates.length > 0) {
                 nextStep = candidates[0].step;
-                logger.debug(`[ResponseModal] Using first valid step: ${nextStep.id} for new route`);
+                logger.debug(`[ResponseModal] Using first valid step: ${nextStep.id}${currentStep ? ' (progressing from ' + currentStep.id + ')' : ' for new route'}`);
             }
             else {
                 // Fallback to initial step even if it should be skipped
@@ -492,16 +1026,42 @@ export class ResponseModal {
             }
         }
         // Update session with next step
-        session = enterStep(session, nextStep.id, nextStep.description);
-        logger.debug(`[ResponseModal] Entered step: ${nextStep.id}`);
+        // If the next step has requires fields that are missing, stay at the previous step
+        if (nextStep.requires && nextStep.requires.length > 0) {
+            const sessionData = session.data || {};
+            const missingRequires = nextStep.requires.filter(field => sessionData[String(field)] === undefined);
+            if (missingRequires.length > 0) {
+                const warning = `[Agent] Cannot advance to step "${nextStep.description || nextStep.id}": ` +
+                    `missing required fields [${missingRequires.join(', ')}]. Staying at current step.`;
+                logger.warn(warning);
+                console.warn(warning);
+                // Stay at the current step - don't enter the next one
+                const currentStepId = session.currentStep?.id;
+                if (currentStepId) {
+                    const currentStepInstance = selectedRoute.getStep(currentStepId);
+                    if (currentStepInstance) {
+                        nextStep = currentStepInstance;
+                        logger.debug(`[ResponseModal] Staying at current step: ${nextStep.id} due to missing requires`);
+                    }
+                }
+            }
+            else {
+                session = enterStep(session, nextStep.id, nextStep.description);
+                logger.debug(`[ResponseModal] Entered step: ${nextStep.id}`);
+            }
+        }
+        else {
+            session = enterStep(session, nextStep.id, nextStep.description);
+            logger.debug(`[ResponseModal] Entered step: ${nextStep.id}`);
+        }
         // Build response schema for this route (with collect fields from step)
         const responseSchema = this.responseEngine.responseSchemaForRoute(selectedRoute, nextStep, this.agent.getSchema());
         // Build response prompt
         const responsePrompt = await this.responseEngine.buildResponsePrompt({
             route: selectedRoute,
             currentStep: nextStep,
-            rules: selectedRoute.getRules(),
-            prohibitions: selectedRoute.getProhibitions(),
+            rules: [...this.agent.getRules(), ...selectedRoute.getRules()],
+            prohibitions: [...this.agent.getProhibitions(), ...selectedRoute.getProhibitions()],
             directives: responseDirectives,
             history: historyEvents, // Use Event[] for buildResponsePrompt
             lastMessage: lastMessageText, // Use string for buildResponsePrompt
@@ -526,6 +1086,14 @@ export class ResponseModal {
         });
         let message = result.structured?.message || result.message;
         let toolCalls = result.structured?.toolCalls;
+        // Debug: Log initial AI response
+        logger.debug(`[ResponseModal] Initial AI response:`, {
+            hasMessage: !!message,
+            messageLength: message?.length || 0,
+            hasToolCalls: !!toolCalls,
+            toolCallsCount: toolCalls?.length || 0,
+            toolNames: toolCalls?.map(tc => tc.toolName) || [],
+        });
         // Execute tools with unified loop handling
         const toolResult = await this.executeUnifiedToolLoop({
             toolCalls,
@@ -560,11 +1128,17 @@ export class ResponseModal {
             nextStep = selectedStep;
         }
         else {
+            // Determine current step from session if we're already in this route
+            const currentStep = session.currentRoute?.id === selectedRoute.id && session.currentStep
+                ? selectedRoute.getStep(session.currentStep.id)
+                : undefined;
+            // Get candidate steps based on current position in the route
             const routingEngine = this.agent.getRoutingEngine();
-            const candidates = routingEngine.getCandidateSteps(selectedRoute, undefined, session.data || {});
+            const candidates = await routingEngine.getCandidateStepsWithConditions(selectedRoute, currentStep, // Pass current step instead of undefined to maintain progression
+            createTemplateContext({ data: session.data, session, context }));
             if (candidates.length > 0) {
                 nextStep = candidates[0].step;
-                logger.debug(`[ResponseModal] Using first valid step: ${nextStep.id} for new route`);
+                logger.debug(`[ResponseModal] Using first valid step: ${nextStep.id}${currentStep ? ' (progressing from ' + currentStep.id + ')' : ' for new route'}`);
             }
             else {
                 nextStep = selectedRoute.initialStep;
@@ -572,15 +1146,40 @@ export class ResponseModal {
             }
         }
         // Update session with next step
-        session = enterStep(session, nextStep.id, nextStep.description);
-        logger.debug(`[ResponseModal] Entered step: ${nextStep.id}`);
+        // If the next step has requires fields that are missing, stay at the previous step
+        if (nextStep.requires && nextStep.requires.length > 0) {
+            const sessionData = session.data || {};
+            const missingRequires = nextStep.requires.filter(field => sessionData[String(field)] === undefined);
+            if (missingRequires.length > 0) {
+                const warning = `[Agent] Cannot advance to step "${nextStep.description || nextStep.id}": ` +
+                    `missing required fields [${missingRequires.join(', ')}]. Staying at current step.`;
+                logger.warn(warning);
+                console.warn(warning);
+                const currentStepId = session.currentStep?.id;
+                if (currentStepId) {
+                    const currentStepInstance = selectedRoute.getStep(currentStepId);
+                    if (currentStepInstance) {
+                        nextStep = currentStepInstance;
+                        logger.debug(`[ResponseModal] Staying at current step: ${nextStep.id} due to missing requires`);
+                    }
+                }
+            }
+            else {
+                session = enterStep(session, nextStep.id, nextStep.description);
+                logger.debug(`[ResponseModal] Entered step: ${nextStep.id}`);
+            }
+        }
+        else {
+            session = enterStep(session, nextStep.id, nextStep.description);
+            logger.debug(`[ResponseModal] Entered step: ${nextStep.id}`);
+        }
         // Build response schema and prompt (same as non-streaming)
         const responseSchema = this.responseEngine.responseSchemaForRoute(selectedRoute, nextStep, this.agent.getSchema());
         const responsePrompt = await this.responseEngine.buildResponsePrompt({
             route: selectedRoute,
             currentStep: nextStep,
-            rules: selectedRoute.getRules(),
-            prohibitions: selectedRoute.getProhibitions(),
+            rules: [...this.agent.getRules(), ...selectedRoute.getRules()],
+            prohibitions: [...this.agent.getProhibitions(), ...selectedRoute.getProhibitions()],
             directives: responseDirectives,
             history: historyEvents, // Use Event[] for buildResponsePrompt
             lastMessage: lastMessageText, // Use string for buildResponsePrompt
@@ -637,6 +1236,10 @@ export class ResponseModal {
             if (chunk.done) {
                 await this.finalizeSession(session, context);
             }
+            // Response structure completeness (Requirement 8.1, 8.2, 8.3)
+            // - executedSteps: single step executed in this response
+            // - stoppedReason: 'needs_input' for single-step execution (waiting for user input)
+            // - session.currentStep: reflects the executed step
             yield {
                 delta: chunk.delta,
                 accumulated: chunk.accumulated,
@@ -644,6 +1247,8 @@ export class ResponseModal {
                 session,
                 toolCalls,
                 isRouteComplete: false,
+                executedSteps: chunk.done ? [{ id: nextStep.id, routeId: selectedRoute.id }] : undefined,
+                stoppedReason: chunk.done ? 'needs_input' : undefined,
                 metadata: chunk.metadata,
                 structured: chunk.structured,
             };
@@ -662,7 +1267,7 @@ export class ResponseModal {
             const historyEvents = historyToEvents(history);
             // Execute initial dynamic tool calls
             if (toolCalls && toolCalls.length > 0) {
-                logger.debug(`[ResponseModal] Executing ${toolCalls.length} dynamic tool calls`);
+                logger.debug(`[ResponseModal] Executing ${toolCalls.length} dynamic tool calls:`, toolCalls.map(tc => tc.toolName));
                 for (const toolCall of toolCalls) {
                     const tool = this.findAvailableTool(toolCall.toolName, selectedRoute);
                     if (!tool) {
@@ -732,7 +1337,7 @@ export class ResponseModal {
             let finalMessage;
             while (hasToolCalls && toolLoopCount < MAX_TOOL_LOOPS) {
                 toolLoopCount++;
-                logger.debug(`[ResponseModal] Starting tool loop ${toolLoopCount}/${MAX_TOOL_LOOPS}`);
+                logger.debug(`[ResponseModal] Starting tool loop ${toolLoopCount}/${MAX_TOOL_LOOPS} with ${toolCalls?.length || 0} tool calls`);
                 // Create tool result events with proper Event format structure
                 const toolResultEvents = [];
                 for (const toolCall of toolCalls || []) {
@@ -761,12 +1366,19 @@ export class ResponseModal {
                 // Create updated history with tool results (combine Event arrays)
                 const updatedHistoryEvents = [...historyEvents, ...toolResultEvents];
                 // Make follow-up AI call to see if more tools are needed
+                // After first iteration, don't provide tools to force a text response
                 const agentOptions = this.agent.getAgentOptions();
+                const shouldProvideTools = toolLoopCount === 1;
+                logger.debug(`[ResponseModal] Making follow-up AI call (loop ${toolLoopCount}):`, {
+                    providingTools: shouldProvideTools,
+                    toolsCount: shouldProvideTools ? availableTools.length : 0,
+                    addingTextInstruction: toolLoopCount > 1,
+                });
                 const followUpResult = await agentOptions.provider.generateMessage({
-                    prompt: responsePrompt,
+                    prompt: responsePrompt + (toolLoopCount > 1 ? "\n\nProvide a text response to the user based on the tool results." : ""),
                     history: updatedHistoryEvents, // Use Event[] for AI provider
                     context,
-                    tools: availableTools,
+                    tools: shouldProvideTools ? availableTools : [], // Only provide tools on first iteration
                     parameters: responseSchema ? {
                         jsonSchema: responseSchema,
                         schemaName: "tool_followup",
@@ -776,6 +1388,13 @@ export class ResponseModal {
                 // Check if follow-up call has more tool calls
                 const followUpToolCalls = followUpResult.structured?.toolCalls;
                 hasToolCalls = followUpToolCalls && followUpToolCalls.length > 0;
+                logger.debug(`[ResponseModal] Follow-up AI response (loop ${toolLoopCount}):`, {
+                    hasMessage: !!followUpResult.message,
+                    messageLength: followUpResult.message?.length || 0,
+                    hasToolCalls,
+                    toolCallsCount: followUpToolCalls?.length || 0,
+                    toolNames: followUpToolCalls?.map(tc => tc.toolName) || [],
+                });
                 if (hasToolCalls) {
                     logger.debug(`[ResponseModal] Follow-up call produced ${followUpToolCalls.length} additional tool calls`);
                     // Execute the follow-up tool calls
@@ -849,6 +1468,12 @@ export class ResponseModal {
             if (toolLoopCount >= MAX_TOOL_LOOPS) {
                 logger.warn(`[ResponseModal] Tool loop limit reached (${MAX_TOOL_LOOPS}), stopping`);
             }
+            logger.debug(`[ResponseModal] Tool loop completed:`, {
+                totalIterations: toolLoopCount,
+                hasFinalMessage: !!finalMessage,
+                finalMessageLength: finalMessage?.length || 0,
+                finalToolCallsCount: toolCalls?.length || 0,
+            });
             return {
                 session,
                 finalToolCalls: toolCalls,
@@ -870,14 +1495,29 @@ export class ResponseModal {
             const { result, selectedRoute, nextStep, session } = params;
             let updatedSession = session;
             // Extract collected data from final response (only for route-based interactions)
-            if (selectedRoute && result.structured && nextStep?.collect) {
+            if (selectedRoute && result.structured) {
                 try {
                     const collectedData = {};
                     // AgentStructuredResponse extends Record<string, unknown>, so we can safely access properties
                     const structuredData = result.structured;
-                    for (const field of nextStep.collect) {
+                    // Collect ALL route fields (required + optional) from structured response
+                    const allRouteFields = new Set();
+                    // Add route required fields
+                    if (selectedRoute.requiredFields) {
+                        selectedRoute.requiredFields.forEach(field => allRouteFields.add(String(field)));
+                    }
+                    // Add route optional fields
+                    if (selectedRoute.optionalFields) {
+                        selectedRoute.optionalFields.forEach(field => allRouteFields.add(String(field)));
+                    }
+                    // Also include current step's collect fields (in case they're not in route fields)
+                    if (nextStep?.collect) {
+                        nextStep.collect.forEach(field => allRouteFields.add(String(field)));
+                    }
+                    // Extract all available fields from structured response
+                    for (const field of allRouteFields) {
                         const fieldKey = String(field);
-                        if (fieldKey in structuredData) {
+                        if (fieldKey in structuredData && structuredData[fieldKey] !== undefined && structuredData[fieldKey] !== null) {
                             collectedData[fieldKey] = structuredData[fieldKey];
                         }
                     }
@@ -940,34 +1580,60 @@ export class ResponseModal {
             requires: endStepSpec.requires,
             prompt: endStepSpec.prompt || "Summarize what was accomplished and confirm completion based on the conversation history and collected data",
         });
-        // Build response schema for completion
-        const responseSchema = this.responseEngine.responseSchemaForRoute(selectedRoute, completionStep, this.agent.getSchema());
-        const templateContext = { context, session, history: historyEvents }; // Use Event[] for template context
-        // Build completion response prompt
+        // Build response schema for completion (message only, no data collection)
+        const completionSchema = {
+            type: "object",
+            properties: {
+                message: {
+                    type: "string",
+                    description: "Completion message confirming what was accomplished",
+                },
+            },
+            required: ["message"],
+            additionalProperties: false,
+        };
+        const templateContext = createTemplateContext({ context, session, history: historyEvents });
+        // Build completion response prompt using ResponseEngine
+        // Filter out conditional guidelines - only include always-active ones
+        const alwaysActiveGuidelines = [
+            ...this.agent.getGuidelines().filter(g => !g.condition),
+            ...selectedRoute.getGuidelines().filter(g => !g.condition),
+        ];
+        let completitionPrompt = "Summarize what was accomplished and confirm completion";
+        if (endStepSpec.prompt) {
+            completitionPrompt = await render(endStepSpec.prompt, templateContext);
+        }
         const completionPrompt = await this.responseEngine.buildResponsePrompt({
             route: selectedRoute,
             currentStep: completionStep,
             rules: selectedRoute.getRules(),
             prohibitions: selectedRoute.getProhibitions(),
-            directives: undefined, // No directives for completion
-            history: historyEvents, // Use Event[] for buildResponsePrompt
-            lastMessage: lastMessageText, // Use string for buildResponsePrompt
+            directives: [
+                `Task completed: ${selectedRoute.title}`,
+                `Collected data: ${JSON.stringify(session.data, null, 2)}`,
+                "Do NOT ask for more information - the task is complete",
+                completitionPrompt,
+            ],
+            history: historyEvents,
+            lastMessage: lastMessageText,
             agentOptions: this.agent.getAgentOptions(),
-            combinedGuidelines: [...this.agent.getGuidelines(), ...selectedRoute.getGuidelines()],
+            combinedGuidelines: alwaysActiveGuidelines, // Only non-conditional guidelines
             combinedTerms: this.mergeTerms(this.agent.getTerms(), selectedRoute.getTerms()),
             context,
             session,
-            agentSchema: this.agent.getSchema(),
+            agentSchema: undefined, // No data collection schema for completion
         });
         // Generate completion message using AI provider
         const agentOptions = this.agent.getAgentOptions();
+        logger.debug(`[ResponseModal] Calling AI provider for completion message...`);
         const completionResult = await agentOptions.provider.generateMessage({
             prompt: completionPrompt,
-            history: historyEvents, // Use Event[] for AI provider
+            history: historyEvents,
             context,
             signal,
-            parameters: { jsonSchema: responseSchema, schemaName: "completion_message" },
+            parameters: { jsonSchema: completionSchema, schemaName: "completion_message" },
         });
+        logger.debug(`[ResponseModal] AI provider returned completion result`);
         const message = completionResult.structured?.message || completionResult.message;
         logger.debug(`[ResponseModal] Generated completion message for route: ${selectedRoute.title}`);
         // Check for onComplete transition
@@ -1010,7 +1676,7 @@ export class ResponseModal {
         });
         // Build response schema for completion
         const responseSchema = this.responseEngine.responseSchemaForRoute(selectedRoute, completionStep, this.agent.getSchema());
-        const templateContext = { context, session, history: historyEvents }; // Use Event[] for template context
+        const templateContext = createTemplateContext({ context, session, history: historyEvents }); // Use Event[] for template context
         // Build completion response prompt
         const completionPrompt = await this.responseEngine.buildResponsePrompt({
             route: selectedRoute,
@@ -1068,6 +1734,10 @@ export class ResponseModal {
             if (chunk.done) {
                 await this.finalizeSession(session, context);
             }
+            // Response structure completeness (Requirement 8.1, 8.2, 8.3)
+            // - executedSteps: empty for route completion (no new steps executed)
+            // - stoppedReason: 'route_complete' for completed routes
+            // - session.currentStep: set to END_ROUTE
             yield {
                 delta: chunk.delta,
                 accumulated: chunk.accumulated,
@@ -1075,6 +1745,8 @@ export class ResponseModal {
                 session,
                 toolCalls: undefined,
                 isRouteComplete: true,
+                executedSteps: chunk.done ? [] : undefined,
+                stoppedReason: chunk.done ? 'route_complete' : undefined,
                 metadata: chunk.metadata,
                 structured: chunk.structured,
             };
@@ -1149,6 +1821,10 @@ export class ResponseModal {
             if (chunk.done) {
                 await this.finalizeSession(session, context);
             }
+            // Response structure completeness (Requirement 8.1, 8.2, 8.3)
+            // - executedSteps: empty for fallback (no route/step execution)
+            // - stoppedReason: undefined for fallback (no route context)
+            // - session.currentStep: unchanged (no step progression)
             yield {
                 delta: chunk.delta,
                 accumulated: chunk.accumulated,
@@ -1156,6 +1832,8 @@ export class ResponseModal {
                 session,
                 toolCalls: undefined,
                 isRouteComplete: false,
+                executedSteps: chunk.done ? [] : undefined,
+                stoppedReason: undefined,
                 metadata: chunk.metadata,
                 structured: chunk.structured,
             };