mstro-app 0.1.53 → 0.1.56

package/server/cli/headless/claude-invoker.ts

@@ -11,9 +11,10 @@ import { type ChildProcess, spawn } from 'node:child_process';
 import { generateMcpConfig } from './mcp-config.js';
 import { detectErrorInStderr, } from './output-utils.js';
 import { buildMultimodalMessage } from './prompt-utils.js';
+import { assessStall, type StallContext } from './stall-assessor.js';
 import type {
   ExecutionResult,
-  ResolvedHeadlessConfig, 
+  ResolvedHeadlessConfig,
   ToolUseAccumulator,
 } from './types.js';
 
@@ -22,6 +23,85 @@ export interface ClaudeInvokerOptions {
   runningProcesses: Map<number, ChildProcess>;
 }
 
+// ========== Stall Detection Helpers ==========
+
+/** Summarize a tool's input for stall assessment context */
+function summarizeToolInput(input: Record<string, unknown>): string | undefined {
+  try {
+    if (input.description) {
+      return String(input.description).slice(0, 200);
+    }
+    if (input.prompt) {
+      return String(input.prompt).slice(0, 200);
+    }
+    if (input.command) {
+      return String(input.command).slice(0, 200);
+    }
+    if (input.pattern) {
+      return `pattern: ${String(input.pattern).slice(0, 100)}`;
+    }
+    return JSON.stringify(input).slice(0, 200);
+  } catch {
+    return undefined;
+  }
+}
+
+/** Terminate a stalled process: SIGTERM then SIGKILL after 5s */
+function terminateStallProcess(
+  claudeProcess: ChildProcess,
+  interval: ReturnType<typeof setInterval>,
+  config: ResolvedHeadlessConfig,
+  message: string,
+): void {
+  clearInterval(interval);
+  config.outputCallback?.(message);
+  claudeProcess.kill('SIGTERM');
+  setTimeout(() => {
+    if (!claudeProcess.killed) {
+      claudeProcess.kill('SIGKILL');
+    }
+  }, 5000);
+}
+
+interface StallAssessmentParams {
+  stallCtx: StallContext;
+  config: ResolvedHeadlessConfig;
+  now: number;
+  extensionsGranted: number;
+  maxExtensions: number;
+}
+
+/** Run stall assessment and return updated state if extended, null otherwise */
+async function runStallAssessment(
+  params: StallAssessmentParams,
+): Promise<{ extensionsGranted: number; currentKillDeadline: number } | null> {
+  const { stallCtx, config, now, extensionsGranted, maxExtensions } = params;
+  try {
+    const verdict = await assessStall(stallCtx, config.claudeCommand, config.verbose);
+    if (verdict.action === 'extend') {
+      const newExtensions = extensionsGranted + 1;
+      config.outputCallback?.(
+        `\n[[MSTRO_STALL_EXTENDED]] Assessment: process likely working. ${verdict.reason}. Extension ${newExtensions}/${maxExtensions}.\n`
+      );
+      if (config.verbose) {
+        console.log(`[STALL] Extended by ${Math.round(verdict.extensionMs / 60_000)} min: ${verdict.reason}`);
+      }
+      return { extensionsGranted: newExtensions, currentKillDeadline: now + verdict.extensionMs };
+    }
+    config.outputCallback?.(
+      `\n[[MSTRO_STALL_CONFIRMED]] Assessment: process likely stalled. ${verdict.reason}.\n`
+    );
+    if (config.verbose) {
+      console.log(`[STALL] Assessment says stalled: ${verdict.reason}`);
+    }
+  } catch (err) {
+    if (config.verbose) {
+      console.log(`[STALL] Assessment error: ${err}`);
+    }
+  }
+  return null;
+}
+
 // ========== Stream Event Handlers ==========
 
 interface StreamHandlerContext {
@@ -271,6 +351,10 @@ function buildClaudeArgs(
 ): string[] {
   const args = ['--print'];
 
+  if (config.model && config.model !== 'default') {
+    args.push('--model', config.model);
+  }
+
   if (useStreamJson) {
     args.push('--output-format', 'stream-json', '--include-partial-messages', '--verbose');
   }
@@ -288,6 +372,11 @@ function buildClaudeArgs(
   if (mcpConfigPath) {
     args.push('--mcp-config', mcpConfigPath);
     args.push('--permission-prompt-tool', 'mcp__mstro-bouncer__approval_prompt');
+  } else {
+    // Bouncer unavailable: use acceptEdits so file operations work without stdin prompts.
+    // Bash still requires approval — Claude Code will skip tools it can't get permission for,
+    // which is better than hanging on a stdin prompt that can never be answered.
+    args.push('--permission-mode', 'acceptEdits');
   }
 
   if (!hasImageAttachments) {
@@ -316,6 +405,13 @@ export async function executeClaudeCommand(
   const hasImageAttachments = config.imageAttachments && config.imageAttachments.length > 0;
   const useStreamJson = hasImageAttachments || config.thinkingCallback || config.outputCallback || config.toolUseCallback;
   const mcpConfigPath = generateMcpConfig(config.workingDir, config.verbose);
+
+  if (!mcpConfigPath && config.outputCallback) {
+    config.outputCallback(
+      '\n[[MSTRO_ERROR:BOUNCER_UNAVAILABLE]] Security bouncer not available. Running with limited permissions — file edits allowed, but shell commands may be restricted.\n'
+    );
+  }
+
   const args = buildClaudeArgs(config, prompt, !!hasImageAttachments, !!useStreamJson, mcpConfigPath);
 
   if (config.verbose) {
@@ -358,7 +454,41 @@ export async function executeClaudeCommand(
     toolInputBuffers: new Map(),
   };
 
+  // Stall detection state
+  let lastActivityTime = Date.now();
+  let stallWarningEmitted = false;
+  let assessmentInProgress = false;
+  let extensionsGranted = 0;
+  let currentKillDeadline = Date.now() + (config.stallKillMs ?? 1_800_000);
+
+  // Tool activity tracking for stall assessment context
+  let lastToolName: string | undefined;
+  let lastToolInputSummary: string | undefined;
+  let pendingToolCount = 0;
+  let totalToolCalls = 0;
+
+  // Wrap the existing tool handlers to track activity
+  const origToolUseCallback = config.toolUseCallback;
+  config.toolUseCallback = (event) => {
+    if (event.type === 'tool_start' && event.toolName) {
+      lastToolName = event.toolName;
+      pendingToolCount++;
+      totalToolCalls++;
+    } else if (event.type === 'tool_complete' && event.completeInput) {
+      lastToolInputSummary = summarizeToolInput(event.completeInput);
+    } else if (event.type === 'tool_result') {
+      pendingToolCount = Math.max(0, pendingToolCount - 1);
+    }
+    origToolUseCallback?.(event);
+  };
+
   claudeProcess.stdout!.on('data', (data) => {
+    lastActivityTime = Date.now();
+    stallWarningEmitted = false;
+    // Push kill deadline forward on any activity
+    const killMs = config.stallKillMs ?? 1_800_000;
+    currentKillDeadline = Date.now() + killMs;
+
     if (!firstStdoutReceived) {
       firstStdoutReceived = true;
       if (config.verbose) {
@@ -389,8 +519,68 @@ export async function executeClaudeCommand(
     }
   });
 
+  // Stall detection with intelligent assessment
+  const stallWarningMs = config.stallWarningMs ?? 300_000;
+  const stallHardCapMs = config.stallHardCapMs ?? 3_600_000;
+  const maxExtensions = config.stallMaxExtensions ?? 3;
+  const stallAssessEnabled = config.stallAssessEnabled !== false;
+
+  const stallCheckInterval = setInterval(async () => {
+    const now = Date.now();
+    const silenceMs = now - lastActivityTime;
+    const totalElapsed = now - perfStart;
+
+    // Hard cap: absolute wall-clock limit regardless of extensions
+    if (totalElapsed >= stallHardCapMs) {
+      terminateStallProcess(claudeProcess, stallCheckInterval, config,
+        `\n[[MSTRO_ERROR:EXECUTION_STALLED]] Hard time limit reached (${Math.round(stallHardCapMs / 60000)} min total). Terminating process.\n`
+      );
+      return;
+    }
+
+    // Kill deadline reached
+    if (now >= currentKillDeadline) {
+      terminateStallProcess(claudeProcess, stallCheckInterval, config,
+        `\n[[MSTRO_ERROR:EXECUTION_STALLED]] No output for ${Math.round(silenceMs / 60_000)} minutes. Terminating process.\n`
+      );
+      return;
+    }
+
+    // Warning + assessment trigger
+    if (silenceMs < stallWarningMs || stallWarningEmitted) return;
+
+    stallWarningEmitted = true;
+    const killIn = Math.round((currentKillDeadline - now) / 60_000);
+    config.outputCallback?.(
+      `\n[[MSTRO_ERROR:EXECUTION_STALLED]] No output for ${Math.round(silenceMs / 60_000)} minutes. Will terminate in ${killIn} minutes if no activity.\n`
+    );
+
+    // Run stall assessment if enabled and we haven't exhausted extensions
+    if (!stallAssessEnabled || assessmentInProgress || extensionsGranted >= maxExtensions) return;
+
+    assessmentInProgress = true;
+    const stallCtx: StallContext = {
+      originalPrompt: prompt,
+      silenceMs,
+      lastToolName,
+      lastToolInputSummary,
+      pendingToolCount,
+      totalToolCalls,
+      elapsedTotalMs: totalElapsed,
+    };
+
+    const result = await runStallAssessment({ stallCtx, config, now, extensionsGranted, maxExtensions });
+    if (result) {
+      extensionsGranted = result.extensionsGranted;
+      currentKillDeadline = result.currentKillDeadline;
+      stallWarningEmitted = false; // Allow re-warning after extension
+    }
+    assessmentInProgress = false;
+  }, 10_000);
+
   return new Promise((resolve, reject) => {
     claudeProcess.on('close', (code) => {
+      clearInterval(stallCheckInterval);
       if (claudeProcess.pid) {
         runningProcesses.delete(claudeProcess.pid);
       }
@@ -406,6 +596,7 @@ export async function executeClaudeCommand(
     });
 
     claudeProcess.on('error', (error: NodeJS.ErrnoException) => {
+      clearInterval(stallCheckInterval);
       if (claudeProcess.pid) {
         runningProcesses.delete(claudeProcess.pid);
       }

package/server/cli/headless/runner.ts

@@ -44,7 +44,13 @@ export class HeadlessRunner {
       claudeSessionId: config.claudeSessionId,
       directPrompt: config.directPrompt || '',
       promptContext: config.promptContext || { accumulatedKnowledge: '', filesModified: [] },
-      imageAttachments: config.imageAttachments
+      imageAttachments: config.imageAttachments,
+      stallWarningMs: config.stallWarningMs ?? 300_000,
+      stallKillMs: config.stallKillMs ?? 1_800_000,
+      stallAssessEnabled: config.stallAssessEnabled !== false,
+      stallMaxExtensions: config.stallMaxExtensions ?? 3,
+      stallHardCapMs: config.stallHardCapMs ?? 3_600_000,
+      model: config.model,
     };
   }
 

package/server/cli/headless/stall-assessor.ts

@@ -0,0 +1,245 @@
+// Copyright (c) 2025-present Mstro, Inc. All rights reserved.
+// Licensed under the MIT License. See LICENSE file for details.
+
+/**
+ * Stall Assessor
+ *
+ * Intelligently determines whether a silent Claude Code process is
+ * legitimately working or genuinely stalled. Uses a two-layer approach:
+ *
+ * 1. Fast heuristic: known long-running patterns (Task subagents, parallel
+ *    tool calls) get an automatic extension without any API call.
+ *
+ * 2. Haiku assessment: for ambiguous cases, spawns a quick Claude Haiku
+ *    call to evaluate the situation and recommend an extension (or kill).
+ */
+
+import { type ChildProcess, spawn } from 'node:child_process';
+
+export interface StallContext {
+  /** The original user prompt being executed */
+  originalPrompt: string;
+  /** How long the process has been silent (ms) */
+  silenceMs: number;
+  /** Name of the last tool that started executing */
+  lastToolName?: string;
+  /** Summarized input of the last tool call */
+  lastToolInputSummary?: string;
+  /** Number of tool calls started but not yet returned */
+  pendingToolCount: number;
+  /** Total tool calls made so far this session */
+  totalToolCalls: number;
+  /** Total wall-clock time since process started (ms) */
+  elapsedTotalMs: number;
+}
+
+export interface StallVerdict {
+  /** Whether to extend the deadline or proceed with kill */
+  action: 'extend' | 'kill';
+  /** Additional time to grant (ms), only meaningful when action is 'extend' */
+  extensionMs: number;
+  /** Human-readable reason for the verdict */
+  reason: string;
+}
+
+/**
+ * Fast heuristic for known long-running patterns.
+ * Returns a verdict immediately if the pattern is recognized, null otherwise.
+ */
+function quickHeuristic(ctx: StallContext): StallVerdict | null {
+  // Task/subagent launches are known to produce long silence periods.
+  // The parent Claude process emits nothing while waiting for subagent results.
+  if (ctx.lastToolName === 'Task' && ctx.pendingToolCount > 0) {
+    const extensionMin = Math.min(30, 10 + ctx.pendingToolCount * 5);
+    return {
+      action: 'extend',
+      extensionMs: extensionMin * 60_000,
+      reason: `${ctx.pendingToolCount} Task subagent(s) still executing — extending ${extensionMin} min`,
+    };
+  }
+
+  // Multiple parallel tool calls (e.g., parallel Bash, parallel Read/Grep)
+  if (ctx.pendingToolCount >= 3) {
+    return {
+      action: 'extend',
+      extensionMs: 15 * 60_000,
+      reason: `${ctx.pendingToolCount} parallel tool calls in progress — extending 15 min`,
+    };
+  }
+
+  // WebSearch/WebFetch can be slow depending on the site
+  if (
+    ctx.lastToolName === 'WebSearch' ||
+    ctx.lastToolName === 'WebFetch'
+  ) {
+    return {
+      action: 'extend',
+      extensionMs: 5 * 60_000,
+      reason: `${ctx.lastToolName} in progress — extending 5 min`,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Main assessment entry point. Tries the fast heuristic first,
+ * falls back to a Haiku model call for ambiguous cases.
+ */
+export async function assessStall(
+  ctx: StallContext,
+  claudeCommand: string,
+  verbose: boolean,
+): Promise<StallVerdict> {
+  // Layer 1: fast heuristic
+  const quick = quickHeuristic(ctx);
+  if (quick) {
+    if (verbose) {
+      console.log(`[STALL-ASSESS] Heuristic verdict: ${quick.reason}`);
+    }
+    return quick;
+  }
+
+  // Layer 2: Haiku assessment
+  try {
+    if (verbose) {
+      console.log('[STALL-ASSESS] Running Haiku assessment...');
+    }
+    return await runHaikuAssessment(ctx, claudeCommand, verbose);
+  } catch (err) {
+    if (verbose) {
+      console.log(`[STALL-ASSESS] Haiku assessment failed: ${err}`);
+    }
+    // If Haiku fails (timeout, auth issue, etc.), extend cautiously
+    return {
+      action: 'extend',
+      extensionMs: 10 * 60_000,
+      reason: 'Stall assessment unavailable — extending 10 min as precaution',
+    };
+  }
+}
+
+function buildAssessmentPrompt(ctx: StallContext): string {
+  const silenceMin = Math.round(ctx.silenceMs / 60_000);
+  const totalMin = Math.round(ctx.elapsedTotalMs / 60_000);
+
+  // Truncate prompt to avoid huge payloads
+  const promptPreview = ctx.originalPrompt.length > 500
+    ? `${ctx.originalPrompt.slice(0, 500)}...`
+    : ctx.originalPrompt;
+
+  return [
+    'You are a process health monitor. A Claude Code subprocess has been silent (no stdout) and you must determine if it is working or stalled.',
+    '',
+    `Silent for: ${silenceMin} minutes`,
+    `Total runtime: ${totalMin} minutes`,
+    `Last tool before silence: ${ctx.lastToolName || 'none'}`,
+    ctx.lastToolInputSummary ? `Last tool input: ${ctx.lastToolInputSummary}` : '',
+    `Pending tool calls: ${ctx.pendingToolCount}`,
+    `Total tool calls this session: ${ctx.totalToolCalls}`,
+    `Task being executed: ${promptPreview}`,
+    '',
+    'Respond in EXACTLY this format (3 lines, no extra text):',
+    'VERDICT: WORKING or STALLED',
+    'MINUTES: <number 5-30, only if WORKING, how many more minutes to allow>',
+    'REASON: <brief one-line explanation>',
+  ].filter(Boolean).join('\n');
+}
+
+function parseAssessmentResponse(output: string): StallVerdict {
+  const lines = output.trim().split('\n');
+  let verdict = 'STALLED';
+  let minutes = 10;
+  let reason = 'Assessment inconclusive';
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith('VERDICT:')) {
+      verdict = trimmed.slice('VERDICT:'.length).trim().toUpperCase();
+    } else if (trimmed.startsWith('MINUTES:')) {
+      const parsed = parseInt(trimmed.slice('MINUTES:'.length).trim(), 10);
+      if (!Number.isNaN(parsed) && parsed >= 1 && parsed <= 60) {
+        minutes = parsed;
+      }
+    } else if (trimmed.startsWith('REASON:')) {
+      reason = trimmed.slice('REASON:'.length).trim();
+    }
+  }
+
+  if (verdict.includes('WORKING')) {
+    return {
+      action: 'extend',
+      extensionMs: minutes * 60_000,
+      reason,
+    };
+  }
+
+  return {
+    action: 'kill',
+    extensionMs: 0,
+    reason,
+  };
+}
+
+const HAIKU_TIMEOUT_MS = 30_000;
+
+function runHaikuAssessment(
+  ctx: StallContext,
+  claudeCommand: string,
+  verbose: boolean,
+): Promise<StallVerdict> {
+  const prompt = buildAssessmentPrompt(ctx);
+
+  return new Promise((resolve, reject) => {
+    let stdout = '';
+    let settled = false;
+
+    const proc: ChildProcess = spawn(
+      claudeCommand,
+      ['--print', '--model', 'haiku', prompt],
+      { stdio: ['ignore', 'pipe', 'pipe'] },
+    );
+
+    const timer = setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        proc.kill('SIGTERM');
+        reject(new Error('Haiku assessment timed out'));
+      }
+    }, HAIKU_TIMEOUT_MS);
+
+    proc.stdout!.on('data', (data) => {
+      stdout += data.toString();
+    });
+
+    proc.stderr!.on('data', (data) => {
+      if (verbose) {
+        console.log(`[STALL-ASSESS] haiku stderr: ${data.toString().trim()}`);
+      }
+    });
+
+    proc.on('close', (code) => {
+      clearTimeout(timer);
+      if (settled) return;
+      settled = true;
+
+      if (code !== 0 || !stdout.trim()) {
+        reject(new Error(`Haiku exited with code ${code}, output: ${stdout.trim()}`));
+        return;
+      }
+
+      if (verbose) {
+        console.log(`[STALL-ASSESS] Haiku response: ${stdout.trim()}`);
+      }
+
+      resolve(parseAssessmentResponse(stdout));
+    });
+
+    proc.on('error', (err) => {
+      clearTimeout(timer);
+      if (settled) return;
+      settled = true;
+      reject(err);
+    });
+  });
+}

package/server/cli/headless/types.ts

@@ -51,6 +51,13 @@ export interface HeadlessConfig {
   continueSession?: boolean;
   claudeSessionId?: string;
   imageAttachments?: ImageAttachment[];
+  stallWarningMs?: number;   // No stdout before warning (default: 300000 = 5 min)
+  stallKillMs?: number;      // No stdout before kill (default: 1800000 = 30 min)
+  stallAssessEnabled?: boolean; // Use Haiku to assess stalls (default: true)
+  stallMaxExtensions?: number;  // Max number of Haiku-granted extensions (default: 3)
+  stallHardCapMs?: number;     // Absolute wall-clock kill cap (default: 3600000 = 60 min)
+  /** Claude model for main execution (e.g., 'opus', 'sonnet'). 'default' = no --model flag. */
+  model?: string;
 }
 
 export interface SessionState {
@@ -108,11 +115,12 @@ export interface ExecutionResult {
 }
 
 /** Resolved config with all defaults applied */
-export type ResolvedHeadlessConfig = Omit<Required<HeadlessConfig>, 'outputCallback' | 'thinkingCallback' | 'toolUseCallback' | 'continueSession' | 'claudeSessionId' | 'imageAttachments'> & {
+export type ResolvedHeadlessConfig = Omit<Required<HeadlessConfig>, 'outputCallback' | 'thinkingCallback' | 'toolUseCallback' | 'continueSession' | 'claudeSessionId' | 'imageAttachments' | 'model'> & {
   outputCallback?: (text: string) => void;
   thinkingCallback?: (text: string) => void;
   toolUseCallback?: (event: ToolUseEvent) => void;
   continueSession?: boolean;
   claudeSessionId?: string;
   imageAttachments?: ImageAttachment[];
+  model?: string;
 };