@probelabs/probe 0.6.0-rc226 → 0.6.0-rc227

package/build/agent/ProbeAgent.d.ts

@@ -3,6 +3,26 @@ import { EventEmitter } from 'events';
 import type { RetryOptions } from './RetryManager';
 import type { FallbackOptions } from './FallbackManager';
 
+// ============================================================================
+// Timeout Configuration Constants
+// ============================================================================
+
+/**
+ * Default activity timeout for engine streams (3 minutes / 180000ms).
+ * This is the time allowed between stream chunks before considering the stream stalled.
+ */
+export const ENGINE_ACTIVITY_TIMEOUT_DEFAULT: number;
+
+/**
+ * Minimum allowed activity timeout (5 seconds / 5000ms).
+ */
+export const ENGINE_ACTIVITY_TIMEOUT_MIN: number;
+
+/**
+ * Maximum allowed activity timeout (10 minutes / 600000ms).
+ */
+export const ENGINE_ACTIVITY_TIMEOUT_MAX: number;
+
 /**
  * Configuration options for creating a ProbeAgent instance
  */
@@ -80,6 +100,10 @@ export interface ProbeAgentOptions {
   completionPrompt?: string;
   /** Enable task management system for tracking multi-step progress */
   enableTasks?: boolean;
+  /** Timeout in ms for AI requests (default: 120000 or REQUEST_TIMEOUT env var). Used to abort hung requests. */
+  requestTimeout?: number;
+  /** Maximum timeout in ms for the entire operation including all retries and fallbacks (default: 300000 or MAX_OPERATION_TIMEOUT env var). This is the absolute maximum time for streamTextWithRetryAndFallback. */
+  maxOperationTimeout?: number;
 }
 
 /**

package/build/agent/ProbeAgent.js

@@ -4,6 +4,29 @@
 import dotenv from 'dotenv';
 dotenv.config();
 
+// ============================================================================
+// Timeout Configuration Constants
+// ============================================================================
+
+/**
+ * Default activity timeout for engine streams (3 minutes).
+ * This is the time allowed between stream chunks before considering the stream stalled.
+ * Conservative default to handle extended thinking models that may not stream during thinking.
+ */
+export const ENGINE_ACTIVITY_TIMEOUT_DEFAULT = 180000;
+
+/**
+ * Minimum allowed activity timeout (5 seconds).
+ * Prevents unreasonably short timeouts that could cause premature failures.
+ */
+export const ENGINE_ACTIVITY_TIMEOUT_MIN = 5000;
+
+/**
+ * Maximum allowed activity timeout (10 minutes).
+ * Prevents excessively long waits for stalled streams.
+ */
+export const ENGINE_ACTIVITY_TIMEOUT_MAX = 600000;
+
 import { createAnthropic } from '@ai-sdk/anthropic';
 import { createOpenAI } from '@ai-sdk/openai';
 import { createGoogleGenerativeAI } from '@ai-sdk/google';
@@ -189,6 +212,8 @@ export class ProbeAgent {
    * @param {number} [options.fallback.maxTotalAttempts=10] - Maximum total attempts across all providers
    * @param {string} [options.completionPrompt] - Custom prompt to run after attempt_completion for validation/review (runs before mermaid/JSON validation)
    * @param {number} [options.maxOutputTokens] - Maximum tokens for tool output before truncation (default: 20000, can also be set via PROBE_MAX_OUTPUT_TOKENS env var)
+   * @param {number} [options.requestTimeout] - Timeout in ms for AI requests (default: 120000 or REQUEST_TIMEOUT env var). Used to abort hung requests.
+   * @param {number} [options.maxOperationTimeout] - Maximum timeout in ms for the entire operation including all retries and fallbacks (default: 300000 or MAX_OPERATION_TIMEOUT env var). This is the absolute maximum time for streamTextWithRetryAndFallback.
    */
   constructor(options = {}) {
     // Basic configuration
@@ -330,6 +355,41 @@ export class ProbeAgent {
     // Each ProbeAgent instance has its own limits, not shared globally
     this.delegationManager = new DelegationManager();
 
+    // Request timeout configuration (default 2 minutes)
+    // Validates env var to prevent NaN or unreasonable values
+    this.requestTimeout = options.requestTimeout ?? (() => {
+      if (process.env.REQUEST_TIMEOUT) {
+        const parsed = parseInt(process.env.REQUEST_TIMEOUT, 10);
+        // Validate: must be positive number between 1s and 1 hour
+        if (isNaN(parsed) || parsed < 1000 || parsed > 3600000) {
+          return 120000; // Default 2 minutes
+        }
+        return parsed;
+      }
+      return 120000;
+    })();
+    if (this.debug) {
+      console.log(`[DEBUG] Request timeout: ${this.requestTimeout}ms`);
+    }
+
+    // Maximum operation timeout for entire streamTextWithRetryAndFallback operation (default 5 minutes)
+    // This is the absolute maximum time including all retries and fallbacks
+    // Validates env var to prevent NaN or unreasonable values
+    this.maxOperationTimeout = options.maxOperationTimeout ?? (() => {
+      if (process.env.MAX_OPERATION_TIMEOUT) {
+        const parsed = parseInt(process.env.MAX_OPERATION_TIMEOUT, 10);
+        // Validate: must be positive number between 1s and 2 hours
+        if (isNaN(parsed) || parsed < 1000 || parsed > 7200000) {
+          return 300000; // Default 5 minutes
+        }
+        return parsed;
+      }
+      return 300000;
+    })();
+    if (this.debug) {
+      console.log(`[DEBUG] Max operation timeout: ${this.maxOperationTimeout}ms`);
+    }
+
     // Retry configuration
     this.retryConfig = options.retry || {};
     this.retryManager = null; // Will be initialized lazily when needed
@@ -1105,120 +1165,128 @@ export class ProbeAgent {
   }
 
   /**
-   * Execute streamText with retry and fallback support
-   * @param {Object} options - streamText options
-   * @returns {Promise<Object>} - streamText result
+   * Create a streamText-compatible result from an engine stream with timeout handling
+   * @param {AsyncGenerator} engineStream - The engine's query result
+   * @param {AbortSignal} abortSignal - Signal for aborting the operation
+   * @param {number} requestTimeout - Per-request timeout in ms
+   * @param {Object} timeoutState - Object with timeoutId property (mutable for cleanup)
+   * @returns {Object} - streamText-compatible result with textStream
    * @private
    */
-  async streamTextWithRetryAndFallback(options) {
-    // Check if we should use Claude Code engine
-    if (this.clientApiProvider === 'claude-code' || process.env.USE_CLAUDE_CODE === 'true') {
+  _createEngineTextStreamResult(engineStream, abortSignal, requestTimeout, timeoutState) {
+    // Activity timeout for engine stream - validates env var against defined bounds
+    const activityTimeout = (() => {
+      const parsed = parseInt(process.env.ENGINE_ACTIVITY_TIMEOUT, 10);
+      return isNaN(parsed) || parsed < ENGINE_ACTIVITY_TIMEOUT_MIN || parsed > ENGINE_ACTIVITY_TIMEOUT_MAX
+        ? ENGINE_ACTIVITY_TIMEOUT_DEFAULT
+        : parsed;
+    })();
+    const startTime = Date.now();
+
+    // Create a text stream that extracts text from engine messages with timeout
+    // The generator clears the operation timeout when done to handle the case
+    // where the stream is returned immediately but consumed later
+    async function* createTextStream() {
+      let lastActivity = Date.now();
+
       try {
-        const engine = await this.getEngine();
-        if (engine && engine.query) {
-          // Convert Vercel AI SDK format to engine format
-          // Extract the ORIGINAL user message as the main prompt (skip any warning messages)
-          // Look for user messages that are NOT the warning message
-          const userMessages = options.messages.filter(m =>
-            m.role === 'user' &&
-            !m.content.includes('WARNING: You have reached the maximum tool iterations limit')
-          );
-          const lastUserMessage = userMessages[userMessages.length - 1];
-          const prompt = lastUserMessage ? lastUserMessage.content : '';
-
-          // Pass system message and other options
-          const engineOptions = {
-            maxTokens: options.maxTokens,
-            temperature: options.temperature,
-            messages: options.messages,
-            systemPrompt: options.messages.find(m => m.role === 'system')?.content
-          };
-
-          // Get the engine's query result (async generator)
-          const engineStream = engine.query(prompt, engineOptions);
-
-          // Create a text stream that extracts text from engine messages
-          async function* createTextStream() {
-            for await (const message of engineStream) {
-              if (message.type === 'text' && message.content) {
-                yield message.content;
-              } else if (typeof message === 'string') {
-                // If engine returns plain strings, pass them through
-                yield message;
-              }
-              // Ignore other message types for the text stream
-            }
+        for await (const message of engineStream) {
+          // Check for abort signal
+          if (abortSignal.aborted) {
+            const abortError = new Error('Operation aborted');
+            abortError.name = 'AbortError';
+            throw abortError;
           }
 
-          // Wrap the engine result to match streamText interface
-          return {
-            textStream: createTextStream(),
-            usage: Promise.resolve({}), // Engine should handle its own usage tracking
-            // Add other streamText-compatible properties as needed
-          };
-        }
-      } catch (error) {
-        if (this.debug) {
-          console.log(`[DEBUG] Failed to use Claude Code engine, falling back to Vercel:`, error.message);
-        }
-        // Fall through to use Vercel engine as fallback
-      }
-    }
+          const now = Date.now();
 
-    // Check if we should use Codex engine
-    if (this.clientApiProvider === 'codex' || process.env.USE_CODEX === 'true') {
-      try {
-        const engine = await this.getEngine();
-        if (engine && engine.query) {
-          // Convert Vercel AI SDK format to engine format
-          // Extract the ORIGINAL user message as the main prompt (skip any warning messages)
-          // Look for user messages that are NOT the warning message
-          const userMessages = options.messages.filter(m =>
-            m.role === 'user' &&
-            !m.content.includes('WARNING: You have reached the maximum tool iterations limit')
-          );
-          const lastUserMessage = userMessages[userMessages.length - 1];
-          const prompt = lastUserMessage ? lastUserMessage.content : '';
-
-          // Pass system message and other options
-          const engineOptions = {
-            maxTokens: options.maxTokens,
-            temperature: options.temperature,
-            messages: options.messages,
-            systemPrompt: options.messages.find(m => m.role === 'system')?.content
-          };
-
-          // Get the engine's query result (async generator)
-          const engineStream = engine.query(prompt, engineOptions);
-
-          // Create a text stream that extracts text from engine messages
-          async function* createTextStream() {
-            for await (const message of engineStream) {
-              if (message.type === 'text' && message.content) {
-                yield message.content;
-              } else if (typeof message === 'string') {
-                // If engine returns plain strings, pass them through
-                yield message;
-              }
-              // Ignore other message types for the text stream
-            }
+          // Check for activity timeout (no data received for too long)
+          if (now - lastActivity > activityTimeout) {
+            throw new Error(`Engine stream timeout - no activity for ${activityTimeout}ms`);
           }
 
-          // Wrap the engine result to match streamText interface
-          return {
-            textStream: createTextStream(),
-            usage: Promise.resolve({}), // Engine should handle its own usage tracking
-            // Add other streamText-compatible properties as needed
-          };
+          // Check for overall request timeout
+          if (requestTimeout > 0 && now - startTime > requestTimeout) {
+            throw new Error(`Engine stream timeout - request exceeded ${requestTimeout}ms`);
+          }
+
+          lastActivity = now;
+
+          if (message.type === 'text' && message.content) {
+            yield message.content;
+          } else if (typeof message === 'string') {
+            // If engine returns plain strings, pass them through
+            yield message;
+          }
+          // Ignore other message types for the text stream
         }
-      } catch (error) {
-        if (this.debug) {
-          console.log(`[DEBUG] Failed to use Codex engine, falling back to Vercel:`, error.message);
+      } finally {
+        // Clear operation timeout when stream completes (success or error)
+        // This is done here because for engine paths, the stream is returned
+        // immediately but consumed later by the caller
+        if (timeoutState.timeoutId) {
+          clearTimeout(timeoutState.timeoutId);
+          timeoutState.timeoutId = null;
         }
-        // Fall through to use Vercel engine as fallback
       }
     }
 
+    // Wrap the engine result to match streamText interface
+    // Note: maxOperationTimeout cleanup is handled by the generator's finally block
+    // since the stream is consumed after this function returns.
+    return {
+      textStream: createTextStream(),
+      usage: Promise.resolve({}), // Engine should handle its own usage tracking
+      // Add other streamText-compatible properties as needed
+    };
+  }
+
+  /**
+   * Try to use an engine (claude-code or codex) for streaming
+   * @param {Object} options - streamText options
+   * @param {AbortController} controller - Abort controller for the operation
+   * @param {Object} timeoutState - Mutable timeout state for cleanup
+   * @returns {Promise<Object|null>} - Stream result or null if engine unavailable
+   * @private
+   */
+  async _tryEngineStreamPath(options, controller, timeoutState) {
+    const engine = await this.getEngine();
+    if (!engine || !engine.query) {
+      return null;
+    }
+
+    // Extract the ORIGINAL user message as the main prompt (skip any warning messages)
+    const userMessages = options.messages.filter(m =>
+      m.role === 'user' &&
+      !m.content.includes('WARNING: You have reached the maximum tool iterations limit')
+    );
+    const lastUserMessage = userMessages[userMessages.length - 1];
+    const prompt = lastUserMessage ? lastUserMessage.content : '';
+
+    // Pass system message and other options including abort signal
+    const engineOptions = {
+      maxTokens: options.maxTokens,
+      temperature: options.temperature,
+      messages: options.messages,
+      systemPrompt: options.messages.find(m => m.role === 'system')?.content,
+      abortSignal: controller.signal
+    };
+
+    // Get the engine's query result and wrap with timeout handling
+    const engineStream = engine.query(prompt, engineOptions);
+    return this._createEngineTextStreamResult(
+      engineStream, controller.signal, this.requestTimeout, timeoutState
+    );
+  }
+
+  /**
+   * Execute streamText with Vercel AI SDK using retry/fallback logic
+   * @param {Object} options - streamText options
+   * @param {AbortController} controller - Abort controller for the operation
+   * @returns {Promise<Object>} - Stream result
+   * @private
+   */
+  async _executeWithVercelProvider(options, controller) {
     // Initialize retry manager if not already created
     if (!this.retryManager) {
       this.retryManager = new RetryManager({
@@ -1234,10 +1302,11 @@ export class ProbeAgent {
     // If no fallback manager, just use retry with current provider
     if (!this.fallbackManager) {
       return await this.retryManager.executeWithRetry(
-        () => streamText(options),
+        () => streamText({ ...options, abortSignal: controller.signal }),
         {
           provider: this.apiType,
-          model: this.model
+          model: this.model,
+          signal: controller.signal
         }
       );
     }
@@ -1245,13 +1314,12 @@ export class ProbeAgent {
     // Use fallback manager with retry for each provider
     return await this.fallbackManager.executeWithFallback(
       async (provider, model, config) => {
-        // Create options with the fallback provider
         const fallbackOptions = {
           ...options,
-          model: provider(model)
+          model: provider(model),
+          abortSignal: controller.signal
         };
 
-        // Create a retry manager for this specific provider
         const providerRetryManager = new RetryManager({
           maxRetries: config.maxRetries ?? this.retryConfig.maxRetries ?? 3,
           initialDelay: this.retryConfig.initialDelay ?? 1000,
@@ -1261,18 +1329,70 @@ export class ProbeAgent {
           debug: this.debug
         });
 
-        // Execute with retry for this provider
         return await providerRetryManager.executeWithRetry(
           () => streamText(fallbackOptions),
           {
             provider: config.provider,
-            model: model
+            model: model,
+            signal: controller.signal
           }
         );
       }
     );
   }
 
+  /**
+   * Execute streamText with retry and fallback support
+   * @param {Object} options - streamText options
+   * @returns {Promise<Object>} - streamText result
+   * @private
+   */
+  async streamTextWithRetryAndFallback(options) {
+    // Create AbortController for overall operation timeout
+    const controller = new AbortController();
+    const timeoutState = { timeoutId: null };
+
+    // Set up overall operation timeout (default 5 minutes)
+    if (this.maxOperationTimeout && this.maxOperationTimeout > 0) {
+      timeoutState.timeoutId = setTimeout(() => {
+        controller.abort();
+        if (this.debug) {
+          console.log(`[DEBUG] Operation timed out after ${this.maxOperationTimeout}ms (max operation timeout)`);
+        }
+      }, this.maxOperationTimeout);
+    }
+
+    try {
+      // Try engine paths (claude-code or codex)
+      const useClaudeCode = this.clientApiProvider === 'claude-code' || process.env.USE_CLAUDE_CODE === 'true';
+      const useCodex = this.clientApiProvider === 'codex' || process.env.USE_CODEX === 'true';
+
+      if (useClaudeCode || useCodex) {
+        try {
+          const result = await this._tryEngineStreamPath(options, controller, timeoutState);
+          if (result) {
+            return result;
+          }
+        } catch (error) {
+          if (this.debug) {
+            const engineType = useClaudeCode ? 'Claude Code' : 'Codex';
+            console.log(`[DEBUG] Failed to use ${engineType} engine, falling back to Vercel:`, error.message);
+          }
+          // Fall through to Vercel provider
+        }
+      }
+
+      // Use Vercel AI SDK with retry/fallback
+      return await this._executeWithVercelProvider(options, controller);
+    } finally {
+      // Clean up timeout (for non-engine paths; engine paths clean up in the generator)
+      if (timeoutState.timeoutId) {
+        clearTimeout(timeoutState.timeoutId);
+        timeoutState.timeoutId = null;
+      }
+    }
+  }
+
   /**
    * Initialize Anthropic model
    */

package/build/agent/engines/enhanced-claude-code.js

@@ -15,7 +15,7 @@ import { Session } from '../shared/Session.js';
  * Enhanced Claude Code Engine
  */
 export async function createEnhancedClaudeCLIEngine(options = {}) {
-  const { agent, systemPrompt, customPrompt, debug, sessionId, allowedTools } = options;
+  const { agent, systemPrompt, customPrompt, debug, sessionId, allowedTools, timeout = 120000 } = options;
 
   // Create or reuse session
   const session = new Session(
@@ -154,6 +154,37 @@ export async function createEnhancedClaudeCLIEngine(options = {}) {
         stdio: ['ignore', 'pipe', 'pipe']  // Ignore stdin since echo handles it
       });
 
+      // Subprocess timeout handling
+      let killed = false;
+      let timeoutHandle;
+      let sigkillHandle;
+
+      if (timeout > 0) {
+        timeoutHandle = setTimeout(() => {
+          if (!killed) {
+            killed = true;
+            processEnded = true;
+            proc.kill('SIGTERM');
+
+            if (debug) {
+              console.log(`[DEBUG] Process timed out after ${timeout}ms, sending SIGTERM`);
+            }
+
+            // Force kill after 5 seconds if still running
+            sigkillHandle = setTimeout(() => {
+              if (proc.exitCode === null) {
+                proc.kill('SIGKILL');
+                if (debug) {
+                  console.log('[DEBUG] Process did not exit, sending SIGKILL');
+                }
+              }
+            }, 5000);
+
+            emitter.emit('error', new Error(`Claude CLI process timed out after ${timeout}ms`));
+          }
+        }, timeout);
+      }
+
       // Handle stdout
       proc.stdout.on('data', (data) => {
         buffer += data.toString();
@@ -179,11 +210,25 @@ export async function createEnhancedClaudeCLIEngine(options = {}) {
 
       // Handle process end
       proc.on('close', (code) => {
+        // Clear the timeouts to prevent memory leaks
+        if (timeoutHandle) {
+          clearTimeout(timeoutHandle);
+        }
+        if (sigkillHandle) {
+          clearTimeout(sigkillHandle);
+        }
+
         processEnded = true;
         if (code !== 0 && debug) {
           console.log(`[DEBUG] Process exited with code ${code}`);
         }
 
+        // If killed by timeout, the error was already emitted
+        if (killed) {
+          emitter.emit('end');
+          return;
+        }
+
         // Process any remaining buffer
         if (buffer.trim()) {
           processJsonBuffer(buffer, emitter, session, debug, toolCollector);
@@ -205,6 +250,14 @@ export async function createEnhancedClaudeCLIEngine(options = {}) {
       });
 
       proc.on('error', (error) => {
+        // Clear the timeouts to prevent memory leaks
+        if (timeoutHandle) {
+          clearTimeout(timeoutHandle);
+        }
+        if (sigkillHandle) {
+          clearTimeout(sigkillHandle);
+        }
+        processEnded = true;
         emitter.emit('error', error);
       });
 
@@ -259,8 +312,24 @@ export async function createEnhancedClaudeCLIEngine(options = {}) {
               content: `\n🔧 Using ${msg.name}: ${JSON.stringify(msg.input)}\n`
             };
 
-            // Execute tool
-            const result = await executeProbleTool(agent, msg.name, msg.input);
+            // Execute tool with timeout to prevent indefinite blocking
+            const toolTimeout = 30000; // 30 seconds
+            let toolTimeoutId;
+            const timeoutPromise = new Promise((_, reject) => {
+              toolTimeoutId = setTimeout(() => reject(new Error(`Tool ${msg.name} timed out after ${toolTimeout}ms`)), toolTimeout);
+            });
+            let result;
+            try {
+              result = await Promise.race([
+                executeProbleTool(agent, msg.name, msg.input),
+                timeoutPromise
+              ]);
+            } catch (error) {
+              result = `Tool error: ${error.message}`;
+            } finally {
+              // Always clear timeout to prevent memory leaks
+              clearTimeout(toolTimeoutId);
+            }
             yield { type: 'text', content: `${result}\n` };
           } else if (msg.type === 'toolBatch') {
             // Pass through the tool batch for ProbeAgent to emit