agent-state-machine 2.1.9 → 2.2.1

package/lib/remote/client.js

@@ -29,9 +29,13 @@ export function generateSessionToken() {
 }
 
 /**
- * Make an HTTP/HTTPS request
+ * Make an HTTP/HTTPS request with timeout
+ * @param {string} url - Request URL
+ * @param {object} options - Request options
+ * @param {object|null} body - Request body
+ * @param {number} timeoutMs - Request timeout in milliseconds
  */
-function makeRequest(url, options, body = null) {
+function makeRequest(url, options, body = null, timeoutMs = 60000) {
   return new Promise((resolve, reject) => {
     const parsedUrl = new URL(url);
     const client = parsedUrl.protocol === 'https:' ? https : http;
@@ -60,6 +64,12 @@ function makeRequest(url, options, body = null) {
       });
     });
 
+    // Timeout prevents hanging on sleep/wake cycles
+    req.setTimeout(timeoutMs, () => {
+      req.destroy();
+      reject(new Error('Request timeout'));
+    });
+
     req.on('error', reject);
 
     if (body) {
@@ -222,28 +232,47 @@ export class RemoteClient {
 
   /**
    * Poll for interaction responses
+   * Uses 35s timeout to stay under Vercel's 50s limit with buffer
    */
   async poll() {
+    let consecutiveErrors = 0;
+
     while (this.polling && this.connected) {
       try {
+        // Request 30s poll from server, with 35s client timeout
         const url = `${this.serverUrl}/api/ws/cli?token=${this.sessionToken}&timeout=30000`;
-        const response = await makeRequest(url, { method: 'GET' });
+        const response = await makeRequest(url, { method: 'GET' }, null, 35000);
+
+        consecutiveErrors = 0; // Reset on success
 
         if (response.status === 200 && response.data) {
           const { type, slug, targetKey, response: interactionResponse } = response.data;
 
           if (type === 'interaction_response' && this.onInteractionResponse) {
+            // Confirm receipt BEFORE processing - removes from Redis pending queue
+            // This ensures we don't lose the interaction if processing fails
+            try {
+              const confirmUrl = `${this.serverUrl}/api/ws/cli?token=${this.sessionToken}`;
+              await makeRequest(confirmUrl, { method: 'DELETE' }, null, 10000);
+            } catch (err) {
+              // Non-fatal - interaction will be re-delivered on next poll
+              console.error(`${C.dim}Remote: Failed to confirm receipt: ${err.message}${C.reset}`);
+            }
+
             this.onInteractionResponse(slug, targetKey, interactionResponse);
           }
         }
 
-        // If 204 (no content), just continue polling
-        // Small delay between polls
-        await new Promise(resolve => setTimeout(resolve, 500));
+        // If 204 (no content), just continue polling immediately
+        // Small delay only on success to prevent tight loop
+        await new Promise(resolve => setTimeout(resolve, 100));
 
       } catch (err) {
-        // Connection error - wait and retry
-        await new Promise(resolve => setTimeout(resolve, 5000));
+        consecutiveErrors++;
+
+        // Exponential backoff: 1s, 2s, 4s, max 10s
+        const backoff = Math.min(1000 * Math.pow(2, consecutiveErrors - 1), 10000);
+        await new Promise(resolve => setTimeout(resolve, backoff));
       }
     }
   }

package/lib/runtime/agent.js

@@ -12,6 +12,7 @@ import { createRequire } from 'module';
 import { pathToFileURL } from 'url';
 import { getCurrentRuntime } from './runtime.js';
 import { formatInteractionPrompt } from './interaction.js';
+import { withChangeTracking } from './track-changes.js';
 
 const require = createRequire(import.meta.url);
 
@@ -164,25 +165,37 @@ async function executeJSAgent(runtime, agentPath, name, params, options = {}) {
     _config: {
       models: runtime.workflowConfig.models,
       apiKeys: runtime.workflowConfig.apiKeys,
-      workflowDir: runtime.workflowDir
+      workflowDir: runtime.workflowDir,
+      projectRoot: runtime.workflowConfig.projectRoot
     }
   };
 
-  let result = await handler(context);
-  let interactionDepth = 0;
-
-  // Handle interaction response from JS agent (support multiple rounds)
-  while (result && result._interaction) {
-    const interactionResponse = await handleInteraction(runtime, result._interaction, name);
-    const resumedContext = { ...context, userResponse: interactionResponse };
-    await logAgentStart(runtime, name);
-    result = await handler(resumedContext);
-    interactionDepth += 1;
-    if (interactionDepth > 5) {
-      throw new Error(`Agent ${name} exceeded maximum interaction depth`);
+  // Execute handler with optional file change tracking
+  const executeHandler = async () => {
+    let result = await handler(context);
+    let interactionDepth = 0;
+
+    // Handle interaction response from JS agent (support multiple rounds)
+    while (result && result._interaction) {
+      const interactionResponse = await handleInteraction(runtime, result._interaction, name);
+      const resumedContext = { ...context, userResponse: interactionResponse };
+      await logAgentStart(runtime, name);
+      result = await handler(resumedContext);
+      interactionDepth += 1;
+      if (interactionDepth > 5) {
+        throw new Error(`Agent ${name} exceeded maximum interaction depth`);
+      }
     }
-  }
 
+    return result;
+  };
+
+  let result;
+  if (runtime.workflowConfig.fileTracking !== false) {
+    result = await withChangeTracking(runtime, name, executeHandler);
+  } else {
+    result = await executeHandler();
+  }
 
   // Clean internal properties from result
   if (result && typeof result === 'object') {
@@ -191,6 +204,7 @@ async function executeJSAgent(runtime, agentPath, name, params, options = {}) {
     delete cleanResult._config;
     delete cleanResult._loop;
     delete cleanResult._interaction;
+    delete cleanResult._files;
 
     // If agent returned a context-like object, only return non-internal keys
     const meaningfulKeys = Object.keys(cleanResult).filter((k) => !k.startsWith('_'));
@@ -235,127 +249,141 @@ async function executeMDAgent(runtime, agentPath, name, params, options = {}) {
     ? runtime.loadSteeringFiles(steeringNames)
     : runtime.steering;
 
-  let response = null;
-  let output = null;
-  let interactionDepth = 0;
-  let currentParams = params;
-
-  while (true) {
-    // Build context - only spread params, NOT memory (explicit context passing)
-    const context = {
-      ...currentParams,
-      _steering: steeringContext,
-      _config: {
-        models: runtime.workflowConfig.models,
-        apiKeys: runtime.workflowConfig.apiKeys,
-        workflowDir: runtime.workflowDir
-      }
-    };
-
-    // Interpolate variables in prompt
-    const interpolatedPrompt = interpolatePrompt(prompt, context);
+  // Build base config (used for all iterations)
+  const baseConfig = {
+    models: runtime.workflowConfig.models,
+    apiKeys: runtime.workflowConfig.apiKeys,
+    workflowDir: runtime.workflowDir,
+    projectRoot: runtime.workflowConfig.projectRoot
+  };
 
-    const model = config.model || 'fast';
+  // Execute the MD agent core logic
+  const executeMDAgentCore = async () => {
+    let response = null;
+    let output = null;
+    let interactionDepth = 0;
+    let currentParams = params;
+
+    while (true) {
+      // Build context - only spread params, NOT memory (explicit context passing)
+      const context = {
+        ...currentParams,
+        _steering: steeringContext,
+        _config: baseConfig
+      };
+
+      // Interpolate variables in prompt
+      const interpolatedPrompt = interpolatePrompt(prompt, context);
+
+      const model = config.model || 'fast';
+
+      const fullPrompt = buildPrompt(context, {
+        model,
+        prompt: interpolatedPrompt,
+        includeContext: config.includeContext !== 'false',
+        responseType: config.response
+      });
 
-    const fullPrompt = buildPrompt(context, {
-      model,
-      prompt: interpolatedPrompt,
-      includeContext: config.includeContext !== 'false',
-      responseType: config.response
-    });
+      await logAgentStart(runtime, name, fullPrompt);
 
-    await logAgentStart(runtime, name, fullPrompt);
+      console.log(`    Using model: ${model}`);
 
-    console.log(`    Using model: ${model}`);
+      response = await llm(context, {
+        model: model,
+        prompt: interpolatedPrompt,
+        includeContext: config.includeContext !== 'false',
+        responseType: config.response
+      });
 
-    response = await llm(context, {
-      model: model,
-      prompt: interpolatedPrompt,
-      includeContext: config.includeContext !== 'false',
-      responseType: config.response
-    });
+      // Parse output based on format
+      output = response.text;
+      if (config.format === 'json') {
+        try {
+          output = parseJSON(response.text);
+        } catch {
+          console.warn(`    Warning: Failed to parse JSON output`);
+        }
+      }
 
-    // Parse output based on format
-    output = response.text;
-    if (config.format === 'json') {
-      try {
-        output = parseJSON(response.text);
-      } catch {
-        console.warn(`    Warning: Failed to parse JSON output`);
+      if (output && typeof output === 'object' && output._interaction) {
+        const interactionResponse = await handleInteraction(runtime, output._interaction, name);
+        currentParams = { ...params, userResponse: interactionResponse };
+        interactionDepth += 1;
+        if (interactionDepth > 5) {
+          throw new Error(`Agent ${name} exceeded maximum interaction depth`);
+        }
+        continue;
       }
+
+      break;
     }
 
-    if (output && typeof output === 'object' && output._interaction) {
-      const interactionResponse = await handleInteraction(runtime, output._interaction, name);
-      currentParams = { ...params, userResponse: interactionResponse };
-      interactionDepth += 1;
-      if (interactionDepth > 5) {
-        throw new Error(`Agent ${name} exceeded maximum interaction depth`);
+    // Check for interaction request
+    const parsedInteraction = parseInteractionRequest(response.text);
+    const structuredInteraction =
+      config.autoInteract !== 'false' && parsedInteraction.isInteraction;
+
+    // Check if agent returned an 'interact' object in its JSON response
+    const hasInteractKey = output && typeof output === 'object' && output.interact;
+
+    // Explicit interaction mode (format: interaction OR interaction: true)
+    // But only trigger if agent actually wants to interact (has interact key or parsed interaction)
+    const explicitInteraction =
+      config.format === 'interaction' ||
+      ((config.interaction === 'true' || (typeof config.interaction === 'string' && config.interaction.length > 0)) &&
+        (hasInteractKey || structuredInteraction));
+
+    if (explicitInteraction || structuredInteraction) {
+      // Use interact object if present, otherwise fall back to parsed/raw
+      const interactionData = hasInteractKey ? output.interact : (structuredInteraction ? parsedInteraction : null);
+
+      const slugRaw =
+        interactionData?.slug ||
+        (typeof config.interaction === 'string' && config.interaction !== 'true'
+          ? config.interaction
+          : null) ||
+        config.interactionSlug ||
+        config.interactionKey ||
+        name;
+
+      const slug = sanitizeSlug(slugRaw);
+      const targetKey = config.interactionKey || outputKey || slug;
+
+      // Build interaction object with full metadata
+      const interactionObj = hasInteractKey ? {
+        ...output.interact,
+        slug,
+        targetKey
+      } : {
+        slug,
+        targetKey,
+        content: structuredInteraction ? parsedInteraction.question : response.text
+      };
+
+      const userResponse = await handleInteraction(runtime, interactionObj, name);
+
+      // Return the user's response as the agent result
+      if (outputKey) {
+        return { [outputKey]: userResponse, _debug_prompt: response.fullPrompt };
       }
-      continue;
-    }
 
-    break;
-  }
+      return userResponse;
+    }
 
-  // Check for interaction request
-  const parsedInteraction = parseInteractionRequest(response.text);
-  const structuredInteraction =
-    config.autoInteract !== 'false' && parsedInteraction.isInteraction;
-
-  // Check if agent returned an 'interact' object in its JSON response
-  const hasInteractKey = output && typeof output === 'object' && output.interact;
-
-  // Explicit interaction mode (format: interaction OR interaction: true)
-  // But only trigger if agent actually wants to interact (has interact key or parsed interaction)
-  const explicitInteraction =
-    config.format === 'interaction' ||
-    ((config.interaction === 'true' || (typeof config.interaction === 'string' && config.interaction.length > 0)) &&
-      (hasInteractKey || structuredInteraction));
-
-  if (explicitInteraction || structuredInteraction) {
-    // Use interact object if present, otherwise fall back to parsed/raw
-    const interactionData = hasInteractKey ? output.interact : (structuredInteraction ? parsedInteraction : null);
-    
-    const slugRaw =
-      interactionData?.slug ||
-      (typeof config.interaction === 'string' && config.interaction !== 'true'
-        ? config.interaction
-        : null) ||
-      config.interactionSlug ||
-      config.interactionKey ||
-      name;
-
-    const slug = sanitizeSlug(slugRaw);
-    const targetKey = config.interactionKey || outputKey || slug;
-    
-    // Build interaction object with full metadata
-    const interactionObj = hasInteractKey ? {
-      ...output.interact,
-      slug,
-      targetKey
-    } : {
-      slug,
-      targetKey,
-      content: structuredInteraction ? parsedInteraction.question : response.text
-    };
-
-    const userResponse = await handleInteraction(runtime, interactionObj, name);
-
-    // Return the user's response as the agent result
+    // Return result object
     if (outputKey) {
-      return { [outputKey]: userResponse, _debug_prompt: response.fullPrompt };
+      return { [outputKey]: output, _debug_prompt: response.fullPrompt };
     }
 
-    return userResponse;
-  }
+    return output;
+  };
 
-  // Return result object
-  if (outputKey) {
-    return { [outputKey]: output, _debug_prompt: response.fullPrompt };
+  // Execute with optional file change tracking
+  if (runtime.workflowConfig.fileTracking !== false) {
+    return withChangeTracking(runtime, name, executeMDAgentCore);
+  } else {
+    return executeMDAgentCore();
   }
-
-  return output;
 }
 
 /**

package/lib/runtime/runtime.js

@@ -12,6 +12,7 @@ import readline from 'readline';
 import { pathToFileURL } from 'url';
 import { createMemoryProxy } from './memory.js';
 import { RemoteClient } from '../remote/client.js';
+import { DEFAULT_IGNORE } from '../file-tree.js';
 
 // Global runtime reference for agent() and memory access
 // stored on globalThis to ensure singleton access across different module instances (CLI vs local)
@@ -74,7 +75,12 @@ export class WorkflowRuntime {
     this.workflowConfig = {
       models: {},
       apiKeys: {},
-      description: ''
+      description: '',
+      // File tracking defaults
+      projectRoot: path.resolve(workflowDir, '../..'),
+      fileTracking: true,
+      fileTrackingIgnore: DEFAULT_IGNORE,
+      fileTrackingKeepDeleted: false
     };
 
     // Load steering
@@ -317,7 +323,12 @@ export class WorkflowRuntime {
       this.workflowConfig = {
         models: cfg.models || {},
         apiKeys: cfg.apiKeys || {},
-        description: cfg.description || ''
+        description: cfg.description || '',
+        // File tracking configuration
+        projectRoot: cfg.projectRoot || path.resolve(this.workflowDir, '../..'),
+        fileTracking: cfg.fileTracking ?? true,
+        fileTrackingIgnore: cfg.fileTrackingIgnore || DEFAULT_IGNORE,
+        fileTrackingKeepDeleted: cfg.fileTrackingKeepDeleted ?? false
       };
 
       // Import workflow module

package/lib/runtime/track-changes.js

@@ -0,0 +1,252 @@
+/**
+ * File: /lib/runtime/track-changes.js
+ *
+ * Wraps agent execution with file change tracking.
+ * Captures baseline before agent runs, detects changes after,
+ * and updates memory.fileTree with the results.
+ */
+
+import path from 'path';
+import {
+  captureBaseline,
+  detectChanges,
+  normalizePath,
+  extractExportsFromFile,
+  DEFAULT_IGNORE
+} from '../file-tree.js';
+
+/**
+ * Wrap an async function with file change tracking.
+ * Captures baseline before execution, detects changes after,
+ * and updates the runtime's fileTree.
+ *
+ * @param {Object} runtime - The workflow runtime instance
+ * @param {string} agentName - Name of the agent (for attribution)
+ * @param {Function} fn - Async function to execute
+ * @returns {Promise<any>} - Result of the function
+ */
+export async function withChangeTracking(runtime, agentName, fn) {
+  const projectRoot = runtime.workflowConfig.projectRoot;
+  const ignorePatterns = runtime.workflowConfig.fileTrackingIgnore || DEFAULT_IGNORE;
+
+  // Capture baseline before agent runs
+  const baseline = await captureBaseline(projectRoot, ignorePatterns);
+
+  // Run the agent
+  const result = await fn();
+
+  // Detect changes made during agent execution
+  const changes = await detectChanges(projectRoot, baseline, ignorePatterns);
+
+  // Update fileTree with detected changes
+  applyChangesToFileTree(runtime, changes, agentName);
+
+  // Merge _files annotations if present (preserves existing data unless explicitly overwritten)
+  if (result && typeof result === 'object' && Array.isArray(result._files)) {
+    mergeAnnotations(runtime, result._files);
+  }
+
+  return result;
+}
+
+/**
+ * Apply detected file changes to the runtime's fileTree.
+ */
+function applyChangesToFileTree(runtime, changes, agentName) {
+  const now = new Date().toISOString();
+  const projectRoot = runtime.workflowConfig.projectRoot;
+
+  // Initialize fileTree if needed
+  if (!runtime._rawMemory.fileTree) {
+    runtime._rawMemory.fileTree = {};
+  }
+
+  // Handle created files
+  for (const filePath of changes.created) {
+    try {
+      const normalized = normalizePath(filePath, projectRoot);
+      runtime._rawMemory.fileTree[normalized] = {
+        path: normalized,
+        status: 'created',
+        createdBy: agentName,
+        lastModifiedBy: agentName,
+        createdAt: now,
+        updatedAt: now
+      };
+    } catch (e) {
+      // Skip files with invalid paths
+      console.warn(`[file-tree] Skipping invalid path: ${filePath} - ${e.message}`);
+    }
+  }
+
+  // Handle modified files
+  for (const filePath of changes.modified) {
+    try {
+      const normalized = normalizePath(filePath, projectRoot);
+      const existing = runtime._rawMemory.fileTree[normalized] || {};
+      runtime._rawMemory.fileTree[normalized] = {
+        ...existing,
+        path: normalized,
+        status: 'modified',
+        lastModifiedBy: agentName,
+        updatedAt: now,
+        createdAt: existing.createdAt || now,
+        createdBy: existing.createdBy || agentName
+      };
+    } catch (e) {
+      console.warn(`[file-tree] Skipping invalid path: ${filePath} - ${e.message}`);
+    }
+  }
+
+  // Handle renamed files (MVP: from/to pairs)
+  for (const { from, to } of changes.renamed) {
+    try {
+      const normalizedFrom = normalizePath(from, projectRoot);
+      const normalizedTo = normalizePath(to, projectRoot);
+      const existing = runtime._rawMemory.fileTree[normalizedFrom] || {};
+      delete runtime._rawMemory.fileTree[normalizedFrom];
+      runtime._rawMemory.fileTree[normalizedTo] = {
+        ...existing,
+        path: normalizedTo,
+        status: 'renamed',
+        renamedFrom: normalizedFrom,
+        lastModifiedBy: agentName,
+        updatedAt: now
+      };
+    } catch (e) {
+      console.warn(`[file-tree] Skipping invalid rename: ${from} -> ${to} - ${e.message}`);
+    }
+  }
+
+  // Handle deleted files
+  for (const filePath of changes.deleted) {
+    try {
+      const normalized = normalizePath(filePath, projectRoot);
+      if (runtime.workflowConfig.fileTrackingKeepDeleted) {
+        runtime._rawMemory.fileTree[normalized] = {
+          ...runtime._rawMemory.fileTree[normalized],
+          status: 'deleted',
+          deletedBy: agentName,
+          deletedAt: now
+        };
+      } else {
+        delete runtime._rawMemory.fileTree[normalized];
+      }
+    } catch (e) {
+      console.warn(`[file-tree] Skipping invalid path: ${filePath} - ${e.message}`);
+    }
+  }
+
+  // Trigger persistence
+  runtime.memory.fileTree = runtime._rawMemory.fileTree;
+}
+
+/**
+ * Merge _files annotations from agent result into fileTree.
+ * Only annotates files that were actually detected by change tracking.
+ * Preserves existing values unless explicitly overwritten.
+ */
+function mergeAnnotations(runtime, files) {
+  const projectRoot = runtime.workflowConfig.projectRoot;
+
+  for (const file of files) {
+    if (!file.path) continue;
+
+    try {
+      const normalizedPath = normalizePath(file.path, projectRoot);
+      const existing = runtime._rawMemory.fileTree?.[normalizedPath];
+
+      if (!existing) continue; // Only annotate files that were actually detected
+
+      // Preserve existing values unless _files explicitly provides new ones
+      if (file.caption !== undefined) existing.caption = file.caption;
+      if (file.exports !== undefined) existing.exports = file.exports;
+      if (file.metadata !== undefined) {
+        existing.metadata = { ...existing.metadata, ...file.metadata };
+      }
+
+      // Trigger best-effort export extraction if requested
+      if (file.extractExports) {
+        const absolutePath = path.resolve(projectRoot, normalizedPath);
+        const exports = extractExportsFromFile(absolutePath);
+        if (exports) existing.exports = exports;
+      }
+    } catch (e) {
+      console.warn(`[file-tree] Skipping invalid annotation path: ${file.path} - ${e.message}`);
+    }
+  }
+
+  // Trigger persistence
+  runtime.memory.fileTree = runtime._rawMemory.fileTree;
+}
+
+/**
+ * Manually track a file in the fileTree.
+ * Used for files created outside of agent execution.
+ */
+export function trackFile(runtime, relativePath, options = {}) {
+  const projectRoot = runtime.workflowConfig.projectRoot;
+  const normalized = normalizePath(relativePath, projectRoot);
+  const now = new Date().toISOString();
+  const agentName = options.agentName || 'manual';
+
+  // Initialize fileTree if needed
+  if (!runtime._rawMemory.fileTree) {
+    runtime._rawMemory.fileTree = {};
+  }
+
+  const existing = runtime._rawMemory.fileTree[normalized];
+
+  const entry = {
+    path: normalized,
+    status: existing?.status || 'created',
+    createdBy: existing?.createdBy || agentName,
+    lastModifiedBy: agentName,
+    createdAt: existing?.createdAt || now,
+    updatedAt: now
+  };
+
+  // Apply options
+  if (options.caption !== undefined) entry.caption = options.caption;
+  if (options.exports !== undefined) entry.exports = options.exports;
+  if (options.metadata !== undefined) {
+    entry.metadata = { ...existing?.metadata, ...options.metadata };
+  }
+
+  // Extract exports if requested
+  if (options.extractExports) {
+    const absolutePath = path.resolve(projectRoot, normalized);
+    const exports = extractExportsFromFile(absolutePath);
+    if (exports) entry.exports = exports;
+  }
+
+  runtime._rawMemory.fileTree[normalized] = entry;
+
+  // Trigger persistence
+  runtime.memory.fileTree = runtime._rawMemory.fileTree;
+
+  return entry;
+}
+
+/**
+ * Get the current fileTree.
+ */
+export function getFileTree(runtime) {
+  return runtime._rawMemory.fileTree || {};
+}
+
+/**
+ * Remove a file from tracking.
+ */
+export function untrackFile(runtime, relativePath) {
+  const projectRoot = runtime.workflowConfig.projectRoot;
+  const normalized = normalizePath(relativePath, projectRoot);
+
+  if (runtime._rawMemory.fileTree?.[normalized]) {
+    delete runtime._rawMemory.fileTree[normalized];
+    runtime.memory.fileTree = runtime._rawMemory.fileTree;
+    return true;
+  }
+
+  return false;
+}