mstro-app 0.1.58 → 0.3.0

package/server/cli/headless/index.ts

@@ -26,14 +26,20 @@ export {
 } from './prompt-utils.js';
 // Main runner class
 export { HeadlessRunner } from './runner.js';
+// Tool watchdog
+export { ToolWatchdog } from './tool-watchdog.js';
 // Types
 export type {
+  ExecutionCheckpoint,
   ExecutionResult,
   HeadlessConfig,
   ImageAttachment,
-  ResolvedHeadlessConfig, 
+  PendingToolMap,
+  ResolvedHeadlessConfig,
   SessionResult,
   SessionState,
+  ToolDurationTracker,
+  ToolTimeoutProfile,
   ToolUseAccumulator,
   ToolUseEvent
 } from './types.js';

package/server/cli/headless/output-utils.test.ts

@@ -0,0 +1,225 @@
+import { describe, expect, it } from 'vitest';
+import {
+  detectErrorInStderr,
+  estimateTokensFromOutput,
+  extractCleanOutput,
+  extractModifiedFiles,
+} from './output-utils.js';
+
+// ========== extractCleanOutput ==========
+
+describe('extractCleanOutput', () => {
+  it('filters out JSON lines with "type" field', () => {
+    const input = [
+      '{"type": "system", "data": "init"}',
+      'Hello world',
+      '{"type": "assistant", "text": "hi"}',
+      'Some output',
+    ].join('\n');
+    expect(extractCleanOutput(input)).toBe('Hello world\nSome output');
+  });
+
+  it('strips ANSI color codes', () => {
+    const input = '\x1b[32mgreen text\x1b[0m and \x1b[1;31mred bold\x1b[0m';
+    expect(extractCleanOutput(input)).toBe('green text and red bold');
+  });
+
+  it('normalizes CRLF to LF', () => {
+    const input = 'line1\r\nline2\r\nline3';
+    expect(extractCleanOutput(input)).toBe('line1\nline2\nline3');
+  });
+
+  it('trims whitespace', () => {
+    const input = '  \n  Hello  \n  ';
+    expect(extractCleanOutput(input)).toBe('Hello');
+  });
+
+  it('filters empty lines', () => {
+    const input = 'line1\n\n\nline2';
+    expect(extractCleanOutput(input)).toBe('line1\nline2');
+  });
+
+  it('returns empty string for all-JSON input', () => {
+    const input = '{"type": "system"}\n{"type": "result"}';
+    expect(extractCleanOutput(input)).toBe('');
+  });
+
+  it('handles combined ANSI + JSON + CRLF', () => {
+    const input = '{"type": "system"}\r\n\x1b[33mwarning\x1b[0m\r\n{"type": "result"}';
+    expect(extractCleanOutput(input)).toBe('warning');
+  });
+});
+
+// ========== estimateTokensFromOutput ==========
+
+describe('estimateTokensFromOutput', () => {
+  it('estimates tokens as length / 4', () => {
+    expect(estimateTokensFromOutput('12345678')).toBe(2);
+    expect(estimateTokensFromOutput('1234')).toBe(1);
+  });
+
+  it('floors the result', () => {
+    expect(estimateTokensFromOutput('12345')).toBe(1); // 5/4 = 1.25 → 1
+    expect(estimateTokensFromOutput('123')).toBe(0);   // 3/4 = 0.75 → 0
+  });
+
+  it('returns 0 for empty string', () => {
+    expect(estimateTokensFromOutput('')).toBe(0);
+  });
+});
+
+// ========== extractModifiedFiles ==========
+
+describe('extractModifiedFiles', () => {
+  it('extracts files from "wrote" pattern', () => {
+    const output = 'wrote file "src/index.ts" successfully';
+    expect(extractModifiedFiles(output)).toContain('src/index.ts');
+  });
+
+  it('extracts files from "modified" pattern', () => {
+    const output = 'modified utils.js in place';
+    expect(extractModifiedFiles(output)).toContain('utils.js');
+  });
+
+  it('extracts files from "created" pattern', () => {
+    const output = "created file 'new-file.tsx'";
+    expect(extractModifiedFiles(output)).toContain('new-file.tsx');
+  });
+
+  it('extracts files from "edited" pattern', () => {
+    const output = 'edited config.json';
+    expect(extractModifiedFiles(output)).toContain('config.json');
+  });
+
+  it('deduplicates files', () => {
+    const output = 'wrote src/index.ts\nmodified src/index.ts';
+    const files = extractModifiedFiles(output);
+    expect(files.filter(f => f === 'src/index.ts')).toHaveLength(1);
+  });
+
+  it('returns empty array when no files found', () => {
+    expect(extractModifiedFiles('no files mentioned here')).toEqual([]);
+  });
+
+  it('extracts multiple different files', () => {
+    const output = 'wrote src/a.ts\ncreated src/b.ts\nedited src/c.ts';
+    const files = extractModifiedFiles(output);
+    expect(files).toContain('src/a.ts');
+    expect(files).toContain('src/b.ts');
+    expect(files).toContain('src/c.ts');
+  });
+});
+
+// ========== detectErrorInStderr ==========
+
+describe('detectErrorInStderr', () => {
+  it('detects auth errors', () => {
+    const result = detectErrorInStderr('Error: not logged in to Claude');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('AUTH_REQUIRED');
+  });
+
+  it('detects session expired', () => {
+    const result = detectErrorInStderr('Your session has expired, please re-authenticate');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('AUTH_REQUIRED');
+  });
+
+  it('detects account not found', () => {
+    const result = detectErrorInStderr('account not found for this user');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('ACCOUNT_NOT_FOUND');
+  });
+
+  it('detects API key errors', () => {
+    const result = detectErrorInStderr('invalid api key provided');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('API_KEY_INVALID');
+  });
+
+  it('detects quota exceeded', () => {
+    const result = detectErrorInStderr('quota exceeded for your subscription');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('QUOTA_EXCEEDED');
+  });
+
+  it('detects billing issues', () => {
+    const result = detectErrorInStderr('payment required to continue');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('QUOTA_EXCEEDED');
+  });
+
+  it('detects rate limiting', () => {
+    const result = detectErrorInStderr('rate limit exceeded, retry after 30s');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('RATE_LIMITED');
+  });
+
+  it('detects 429 status', () => {
+    const result = detectErrorInStderr('HTTP 429 too many requests');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('RATE_LIMITED');
+  });
+
+  it('detects network errors', () => {
+    const result = detectErrorInStderr('ECONNREFUSED 127.0.0.1:443');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('NETWORK_ERROR');
+  });
+
+  it('detects DNS failures', () => {
+    const result = detectErrorInStderr('ENOTFOUND api.anthropic.com');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('NETWORK_ERROR');
+  });
+
+  it('detects SSL errors', () => {
+    const result = detectErrorInStderr('CERT_HAS_EXPIRED for api.example.com');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('SSL_ERROR');
+  });
+
+  it('detects service unavailable', () => {
+    const result = detectErrorInStderr('service unavailable, try again later');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('SERVICE_UNAVAILABLE');
+  });
+
+  it('detects 503 status', () => {
+    const result = detectErrorInStderr('HTTP 503 from upstream');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('SERVICE_UNAVAILABLE');
+  });
+
+  it('detects internal errors', () => {
+    const result = detectErrorInStderr('internal server error occurred');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('INTERNAL_ERROR');
+  });
+
+  it('detects context too long', () => {
+    const result = detectErrorInStderr('context too long, exceeds 200k tokens');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('CONTEXT_TOO_LONG');
+  });
+
+  it('detects session not found', () => {
+    const result = detectErrorInStderr('session not found, please create a new one');
+    expect(result).not.toBeNull();
+    expect(result!.errorCode).toBe('SESSION_NOT_FOUND');
+  });
+
+  it('returns null for non-matching stderr', () => {
+    expect(detectErrorInStderr('Processing file...')).toBeNull();
+    expect(detectErrorInStderr('Warning: deprecated API usage')).toBeNull();
+    expect(detectErrorInStderr('')).toBeNull();
+  });
+
+  it('returns user-friendly messages', () => {
+    const result = detectErrorInStderr('not logged in');
+    expect(result).not.toBeNull();
+    expect(result!.message).toContain('authentication');
+    // Should not expose raw error
+    expect(result!.message).not.toContain('not logged in');
+  });
+});

package/server/cli/headless/prompt-utils.ts

@@ -7,16 +7,19 @@
  * Utilities for enriching prompts with context from previous conversation.
  */
 
+import { assessApproval } from './stall-assessor.js';
 import type { ImageAttachment, PromptContext } from './types.js';
 
 /**
- * Enrich prompt with context from previous conversation
+ * Enrich prompt with context from previous conversation.
+ * Async because ambiguous short prompts are classified by Haiku.
  */
-export function enrichPromptWithContext(prompt: string, context: PromptContext): string {
+export async function enrichPromptWithContext(prompt: string, context: PromptContext): Promise<string> {
   let enriched = prompt;
 
-  // Detect if this is a continuation/approval prompt
-  const isApprovalOrContinuation = isApprovalPrompt(prompt);
+  // Detect if this is a continuation/approval prompt.
+  // Fast regex path for obvious approvals, Haiku for ambiguous short prompts.
+  const isApprovalOrContinuation = await detectApproval(prompt);
 
   // Add accumulated knowledge from previous prompts
   if (context.accumulatedKnowledge) {
@@ -36,9 +39,38 @@ export function enrichPromptWithContext(prompt: string, context: PromptContext):
 }
 
 /**
- * Detect if a prompt is an approval or continuation
+ * Detect if a prompt is an approval or continuation.
+ * Layer 1: Regex fast path for obvious approvals (free, sync).
+ * Layer 2: Haiku assessment for ambiguous short prompts (<100 chars).
+ */
+async function detectApproval(prompt: string): Promise<boolean> {
+  // Layer 1: fast regex path
+  if (isApprovalPromptFast(prompt)) return true;
+
+  // Layer 2: Haiku for short ambiguous prompts.
+  // Long prompts (>100 chars) are almost certainly new tasks, not approvals.
+  if (prompt.trim().length <= 100) {
+    try {
+      const claudeCmd = process.env.CLAUDE_COMMAND || 'claude';
+      const verdict = await assessApproval(prompt, claudeCmd, false);
+      return verdict.isApproval;
+    } catch {
+      // Haiku failed — fall through to false
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Fast regex-based approval detection (sync, no API call).
+ * Catches obvious affirmatives. Ambiguous cases fall through to Haiku.
  */
 export function isApprovalPrompt(prompt: string): boolean {
+  return isApprovalPromptFast(prompt);
+}
+
+function isApprovalPromptFast(prompt: string): boolean {
   const lower = prompt.toLowerCase().trim();
 
   // Short affirmative responses

package/server/cli/headless/runner.ts

@@ -20,7 +20,7 @@ import type {
 } from './types.js';
 
 // Re-export types for backward compatibility
-export type { HeadlessConfig, ImageAttachment, SessionResult, SessionState, ToolUseEvent } from './types.js';
+export type { ExecutionCheckpoint, HeadlessConfig, ImageAttachment, SessionResult, SessionState, ToolTimeoutProfile, ToolUseEvent } from './types.js';
 
 export class HeadlessRunner {
   private config: ResolvedHeadlessConfig;
@@ -33,9 +33,9 @@ export class HeadlessRunner {
       maxSessions: config.maxSessions || 50,
       maxRetries: config.maxRetries || 3,
       claudeCommand: config.claudeCommand || process.env.CLAUDE_COMMAND || 'claude',
-      verbose: config.verbose || false,
-      noColor: config.noColor || false,
-      improvisationMode: config.improvisationMode || false,
+      verbose: !!config.verbose,
+      noColor: !!config.noColor,
+      improvisationMode: !!config.improvisationMode,
       movementNumber: config.movementNumber ?? 0,
       outputCallback: config.outputCallback,
       thinkingCallback: config.thinkingCallback,
@@ -51,6 +51,11 @@ export class HeadlessRunner {
       stallMaxExtensions: config.stallMaxExtensions ?? 3,
       stallHardCapMs: config.stallHardCapMs ?? 3_600_000,
       model: config.model,
+      toolTimeoutProfiles: config.toolTimeoutProfiles,
+      enableToolWatchdog: config.enableToolWatchdog !== false,
+      maxAutoRetries: config.maxAutoRetries ?? 2,
+      onToolTimeout: config.onToolTimeout,
+      sandboxed: config.sandboxed,
     };
   }
 
@@ -78,22 +83,60 @@ export class HeadlessRunner {
     const sessionId = `direct-${Date.now()}`;
 
     const enrichedPrompt = context
-      ? enrichPromptWithContext(userPrompt, context)
+      ? await enrichPromptWithContext(userPrompt, context)
       : userPrompt;
 
     const result = await this.executePromptCommand(enrichedPrompt, 'main', 1);
 
     if (result.exitCode !== 0) {
+      // Signal exits (128+) with meaningful output are successful completions —
+      // Claude finished its work but the process was killed by signal (e.g., stall watchdog SIGTERM)
+      const isSignalExit = result.exitCode >= 128;
+      const hasOutput = !!(result.assistantResponse || (result.toolUseHistory && result.toolUseHistory.length > 0));
+
+      if (isSignalExit && hasOutput) {
+        const tokens = estimateTokensFromOutput(result.output);
+        return {
+          completed: true,
+          needsHandoff: false,
+          totalTokens: tokens,
+          sessionId,
+          signalName: result.signalName,
+          assistantResponse: result.assistantResponse,
+          thinkingOutput: result.thinkingOutput,
+          toolUseHistory: result.toolUseHistory,
+          claudeSessionId: result.claudeSessionId,
+          nativeTimeoutCount: result.nativeTimeoutCount,
+          postTimeoutOutput: result.postTimeoutOutput,
+          resumeBufferedOutput: result.resumeBufferedOutput,
+        };
+      }
+
+      // Build meaningful error: prefer stderr, fall back to non-JSON stdout lines
+      let errorMessage = result.error;
+      if (!errorMessage && result.output) {
+        const plainLines = result.output.split('\n')
+          .filter(l => l.trim() && !l.trim().startsWith('{'))
+          .join('\n')
+          .trim();
+        if (plainLines) {
+          errorMessage = plainLines.slice(0, 500);
+        }
+      }
       return {
         completed: false,
         needsHandoff: false,
         totalTokens: 0,
         sessionId,
-        error: result.error || 'Execution failed',
+        error: errorMessage || `Claude exited with code ${result.exitCode}`,
+        signalName: result.signalName,
         assistantResponse: result.assistantResponse,
         thinkingOutput: result.thinkingOutput,
         toolUseHistory: result.toolUseHistory,
-        claudeSessionId: result.claudeSessionId
+        claudeSessionId: result.claudeSessionId,
+        nativeTimeoutCount: result.nativeTimeoutCount,
+        postTimeoutOutput: result.postTimeoutOutput,
+        resumeBufferedOutput: result.resumeBufferedOutput,
       };
     }
 
@@ -104,10 +147,14 @@ export class HeadlessRunner {
       needsHandoff: false,
       totalTokens: tokens,
       sessionId,
+      signalName: result.signalName,
       assistantResponse: result.assistantResponse,
       thinkingOutput: result.thinkingOutput,
       toolUseHistory: result.toolUseHistory,
-      claudeSessionId: result.claudeSessionId
+      claudeSessionId: result.claudeSessionId,
+      nativeTimeoutCount: result.nativeTimeoutCount,
+      postTimeoutOutput: result.postTimeoutOutput,
+      resumeBufferedOutput: result.resumeBufferedOutput,
     };
   }
 

package/server/cli/headless/stall-assessor.test.ts

@@ -0,0 +1,165 @@
+import { describe, expect, it } from 'vitest';
+import type { StallContext } from './stall-assessor.js';
+
+// quickHeuristic, parseAssessmentResponse, and parseVerdictResponse are not exported.
+// We test them via assessStall (which calls quickHeuristic first) and by testing
+// the parsing functions indirectly. Since quickHeuristic is the critical logic
+// and assessStall calls it before Haiku, we can test the heuristic paths by
+// providing contexts that match known patterns.
+//
+// To avoid spawning Haiku (which requires `claude` CLI), we only test contexts
+// that trigger the heuristic fast-path (return non-null from quickHeuristic).
+
+import { assessStall } from './stall-assessor.js';
+
+function makeContext(overrides: Partial<StallContext> = {}): StallContext {
+  return {
+    originalPrompt: 'Fix the bug in auth.ts',
+    silenceMs: 120_000,
+    pendingToolCount: 0,
+    totalToolCalls: 5,
+    elapsedTotalMs: 300_000,
+    ...overrides,
+  };
+}
+
+describe('assessStall - quickHeuristic paths', () => {
+  it('extends when tokens are still flowing (tokenSilenceMs < 60s)', async () => {
+    const ctx = makeContext({ tokenSilenceMs: 30_000 });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.extensionMs).toBe(10 * 60_000);
+    expect(verdict.reason).toContain('Tokens still flowing');
+  });
+
+  it('extends when tokenSilenceMs is 0', async () => {
+    const ctx = makeContext({ tokenSilenceMs: 0 });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('Tokens still flowing');
+  });
+
+  it('does not use token heuristic when tokenSilenceMs >= 60s', async () => {
+    const ctx = makeContext({
+      tokenSilenceMs: 60_000,
+      pendingToolCount: 3, // will trigger parallel tools heuristic
+    });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    // Should NOT hit the token heuristic, should hit the 3+ parallel tools one
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('parallel tool calls');
+  });
+
+  it('defers to watchdog when active and tools are pending', async () => {
+    const ctx = makeContext({ pendingToolCount: 1, lastToolName: 'Bash' });
+    const verdict = await assessStall(ctx, 'claude', false, true);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.extensionMs).toBe(15 * 60_000);
+    expect(verdict.reason).toContain('Watchdog active');
+  });
+
+  it('defers to watchdog and lists pending tool names', async () => {
+    const ctx = makeContext({
+      pendingToolCount: 2,
+      pendingToolNames: new Set(['WebFetch', 'Bash']),
+    });
+    const verdict = await assessStall(ctx, 'claude', false, true);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('WebFetch');
+    expect(verdict.reason).toContain('Bash');
+  });
+
+  it('extends for Task subagent via pendingToolNames', async () => {
+    const ctx = makeContext({
+      pendingToolCount: 1,
+      pendingToolNames: new Set(['Task']),
+    });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('Task subagent');
+  });
+
+  it('extends for Task subagent via lastToolName fallback', async () => {
+    const ctx = makeContext({
+      pendingToolCount: 1,
+      lastToolName: 'Task',
+    });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('Task subagent');
+  });
+
+  it('scales Task extension with pending count', async () => {
+    const ctx1 = makeContext({
+      pendingToolCount: 1,
+      pendingToolNames: new Set(['Task']),
+    });
+    const ctx3 = makeContext({
+      pendingToolCount: 3,
+      pendingToolNames: new Set(['Task']),
+    });
+    const v1 = await assessStall(ctx1, 'claude', false, false);
+    const v3 = await assessStall(ctx3, 'claude', false, false);
+    // More pending = more extension, capped at 30 min
+    expect(v3.extensionMs).toBeGreaterThanOrEqual(v1.extensionMs);
+    expect(v3.extensionMs).toBeLessThanOrEqual(30 * 60_000);
+  });
+
+  it('extends for 3+ parallel tool calls', async () => {
+    const ctx = makeContext({ pendingToolCount: 3 });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.extensionMs).toBe(15 * 60_000);
+    expect(verdict.reason).toContain('parallel tool calls');
+  });
+
+  it('extends for 5 parallel tool calls', async () => {
+    const ctx = makeContext({ pendingToolCount: 5 });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('5 parallel tool calls');
+  });
+
+  it('extends for WebSearch without watchdog', async () => {
+    const ctx = makeContext({ lastToolName: 'WebSearch', pendingToolCount: 1 });
+    // pendingToolCount < 3, not Task, not watchdog active, but WebSearch
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.extensionMs).toBe(5 * 60_000);
+    expect(verdict.reason).toContain('WebSearch');
+  });
+
+  it('extends for WebFetch without watchdog', async () => {
+    const ctx = makeContext({ lastToolName: 'WebFetch', pendingToolCount: 1 });
+    const verdict = await assessStall(ctx, 'claude', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.extensionMs).toBe(5 * 60_000);
+    expect(verdict.reason).toContain('WebFetch');
+  });
+
+  it('does NOT extend for WebSearch when watchdog is active', async () => {
+    // When watchdog is active and tools are pending, the watchdog deferral
+    // takes priority over the WebSearch heuristic
+    const ctx = makeContext({
+      lastToolName: 'WebSearch',
+      pendingToolCount: 1,
+    });
+    const verdict = await assessStall(ctx, 'claude', false, true);
+    // Should defer to watchdog, not WebSearch heuristic
+    expect(verdict.action).toBe('extend');
+    expect(verdict.reason).toContain('Watchdog active');
+  });
+
+  it('falls back to extend when Haiku assessment fails', async () => {
+    // Context that doesn't match any heuristic → triggers Haiku →
+    // Haiku fails (no `claude` binary) → cautious extend
+    const ctx = makeContext({
+      pendingToolCount: 1,
+      lastToolName: 'Edit',
+    });
+    const verdict = await assessStall(ctx, 'nonexistent-claude-binary', false, false);
+    expect(verdict.action).toBe('extend');
+    expect(verdict.extensionMs).toBe(10 * 60_000);
+    expect(verdict.reason).toContain('unavailable');
+  });
+});