crewly 1.12.0 → 1.13.0

package/frontend/dist/index.html

@@ -7,7 +7,7 @@
     <meta name="color-scheme" content="dark" />
     <!-- Nunito font is self-hosted via @fontsource/nunito (imported in main.tsx) -->
     <title>Crewly AI Studio</title>
-    <script type="module" crossorigin src="/assets/index-890d3f9d.js"></script>
+    <script type="module" crossorigin src="/assets/index-aa737fd3.js"></script>
     <link rel="stylesheet" href="/assets/index-8205ea5e.css">
   </head>
   <body class="bg-background-dark font-display text-text-primary-dark">

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crewly",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "type": "module",
   "description": "Multi-agent orchestration platform for AI coding teams — coordinates Claude Code, Gemini CLI, and Codex agents with a real-time web dashboard",
   "workspaces": [

package/packages/crewly-agent/src/cli.ts

@@ -10,15 +10,19 @@ import type {
 
 type ParentMessage =
   | { type: 'init'; config: CrewlyAgentConfig }
-  | { type: 'run'; message: string; conversationId?: string; metadata?: Record<string, string> }
+  | { type: 'run'; runId?: string; message: string; conversationId?: string; metadata?: Record<string, string> }
   | { type: 'abort' }
   | { type: 'get-state' }
   | { type: 'shutdown' };
 
 type WorkerMessage =
   | { type: 'ready' }
-  | { type: 'result'; data: AgentRunResult }
-  | { type: 'error'; error: string; code?: string }
+  // `runId` is echoed straight back from the `run` that produced it so the
+  // parent can match a reply to its request. Without it the parent has to
+  // guess, and guessing wrong books one conversation's answer against
+  // another. Optional so an older parent that sends no id still works.
+  | { type: 'result'; runId?: string; data: AgentRunResult }
+  | { type: 'error'; runId?: string; error: string; code?: string }
   | { type: 'log'; level: 'debug' | 'info' | 'warn' | 'error'; message: string }
   | { type: 'stream'; event: 'text'; data: { chunk: string } }
   | { type: 'stream'; event: 'toolStart'; data: { toolName: string; args: Record<string, unknown> } }
@@ -71,9 +75,10 @@ async function handleMessage(message: ParentMessage): Promise<void> {
         runner = null;
       }
       break;
-    case 'run':
+    case 'run': {
+      const { runId } = message;
       if (!runner || !runner.isInitialized()) {
-        send({ type: 'error', error: 'Worker not initialized', code: 'NOT_INITIALIZED' });
+        send({ type: 'error', runId, error: 'Worker not initialized', code: 'NOT_INITIALIZED' });
         return;
       }
       currentAbort = new AbortController();
@@ -88,16 +93,18 @@ async function handleMessage(message: ParentMessage): Promise<void> {
           },
         );
         currentAbort = null;
-        send({ type: 'result', data: result });
+        send({ type: 'result', runId, data: result });
       } catch (error) {
         currentAbort = null;
         send({
           type: 'error',
+          runId,
           error: error instanceof Error ? error.message : String(error),
           code: 'RUN_FAILED',
         });
       }
       break;
+    }
     case 'abort':
       if (currentAbort) {
         currentAbort.abort();
@@ -145,6 +152,9 @@ rl.on('line', (line) => {
   handleMessage(message).catch((error) => {
     send({
       type: 'error',
+      // Carry the correlation id here too — an unhandled failure must settle
+      // the run it belongs to, not whichever run happens to be oldest.
+      runId: message.type === 'run' ? message.runId : undefined,
       error: error instanceof Error ? error.message : String(error),
       code: 'UNHANDLED',
     });

package/packages/crewly-agent/src/runtime/agent-runner.service.test.ts

@@ -1,8 +1,8 @@
-import { describe, it, expect, beforeEach, vi, type Mocked, type MockInstance } from 'vitest';
-import { AgentRunnerService, ToolCallLoopDetector } from './agent-runner.service.js';
+import { describe, it, expect, beforeEach, afterEach, vi, type Mocked, type MockInstance } from 'vitest';
+import { AgentRunnerService, AgentRunTimeoutError, ToolCallLoopDetector } from './agent-runner.service.js';
 import { ModelManager } from './model-manager.js';
 import { CrewlyApiClient } from './api-client.js';
-import type { CrewlyAgentConfig, SecurityPolicy, AuditEntry } from './types.js';
+import { CREWLY_AGENT_DEFAULTS, type CrewlyAgentConfig, type SecurityPolicy, type AuditEntry } from './types.js';
 
 describe('AgentRunnerService', () => {
   let runner: AgentRunnerService;
@@ -2352,4 +2352,138 @@ describe('B4 — DeepSeek tool_choice passthrough regression', () => {
       expect(runner.getConversationCount()).toBe(1);
     });
   });
+
+});
+
+/**
+ * Wall-clock run budget — the 假死 (silent-catatonia) guard.
+ *
+ * Regression cover for the production incident where a stalled model
+ * connection hung `await streamResult` forever. `MESSAGE_TIMEOUT_MS` and
+ * `MODEL_TIMEOUT_MS` existed as documented constants but nothing ever read
+ * them, so the orchestrator went silent for 12 days without a single log
+ * line. These tests assert the budget is actually ENFORCED, not merely
+ * declared.
+ */
+describe('AgentRunnerService run wall-clock budget', () => {
+  let runner: AgentRunnerService;
+  let mockGenerateText: vi.Mock<any>;
+  let mockModelManager: any;
+  let mockApiClient: any;
+
+  const baseConfig: CrewlyAgentConfig = {
+    model: { provider: 'anthropic', modelId: 'claude-sonnet-4-20250514', temperature: 0.3, maxTokens: 8192 },
+    maxSteps: 10,
+    sessionName: 'budget-session',
+    apiBaseUrl: 'http://localhost:8787',
+    systemPrompt: 'You are a test agent.',
+    maxHistoryMessages: 20,
+    compactionThreshold: 0.8,
+  };
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+    vi.useFakeTimers();
+
+    mockGenerateText = vi.fn<any>();
+    mockModelManager = {
+      getModel: vi.fn<any>().mockResolvedValue({ provider: 'mock', modelId: 'test-model' }),
+      getAvailableProviders: vi.fn<any>(),
+      clearCache: vi.fn<any>(),
+      consumeDeepseekReasoning: vi.fn<any>().mockResolvedValue(null),
+    };
+    mockApiClient = { get: vi.fn<any>(), post: vi.fn<any>(), delete: vi.fn<any>() };
+
+    runner = new AgentRunnerService(baseConfig, mockModelManager, mockApiClient);
+    runner._generateTextFn = mockGenerateText;
+    await runner.initialize();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+    it('rejects with AgentRunTimeoutError when the model never responds', async () => {
+      // A model call that never settles — exactly the stalled-socket shape.
+      mockGenerateText.mockImplementation(() => new Promise(() => {}));
+
+      const runPromise = runner.run('will stall');
+      const assertion = expect(runPromise).rejects.toBeInstanceOf(AgentRunTimeoutError);
+
+      await vi.advanceTimersByTimeAsync(CREWLY_AGENT_DEFAULTS.MESSAGE_TIMEOUT_MS + 1);
+
+      await assertion;
+    });
+
+    it('does not wedge the queue — later messages still process after a timeout', async () => {
+      // THE regression: a hung run left `processing` true forever, so every
+      // subsequent message queued behind it and the agent went catatonic.
+      mockGenerateText.mockImplementationOnce(() => new Promise(() => {}));
+
+      const stalled = runner.run('will stall');
+      const stalledAssertion = expect(stalled).rejects.toBeInstanceOf(AgentRunTimeoutError);
+      await vi.advanceTimersByTimeAsync(CREWLY_AGENT_DEFAULTS.MESSAGE_TIMEOUT_MS + 1);
+      await stalledAssertion;
+
+      expect(runner.isProcessing()).toBe(false);
+
+      mockGenerateText.mockResolvedValueOnce({
+        text: 'recovered',
+        toolCalls: [],
+        steps: [],
+        usage: { inputTokens: 1, outputTokens: 1 },
+        finishReason: 'stop',
+      });
+
+      await expect(runner.run('after the stall')).resolves.toMatchObject({ text: 'recovered' });
+    });
+
+    it('does not retry a budget expiry (it spans the retries already)', async () => {
+      mockGenerateText.mockImplementation(() => new Promise(() => {}));
+
+      const runPromise = runner.run('will stall');
+      const assertion = expect(runPromise).rejects.toBeInstanceOf(AgentRunTimeoutError);
+      await vi.advanceTimersByTimeAsync(CREWLY_AGENT_DEFAULTS.MESSAGE_TIMEOUT_MS + 1);
+      await assertion;
+
+      // One attempt only — a retried timeout would multiply the stall.
+      expect(mockGenerateText).toHaveBeenCalledTimes(1);
+    });
+
+    it('honors the per-model budget override for reasoning models', async () => {
+      const reasoningRunner = new AgentRunnerService(
+        { ...baseConfig, model: { ...baseConfig.model, modelId: 'deepseek-reasoner' } },
+        mockModelManager,
+        mockApiClient,
+      );
+      reasoningRunner._generateTextFn = mockGenerateText;
+      await reasoningRunner.initialize();
+      mockGenerateText.mockImplementation(() => new Promise(() => {}));
+
+      const runPromise = reasoningRunner.run('slow chain-of-thought');
+      const settled = vi.fn();
+      runPromise.then(settled, settled);
+
+      // Still alive at the default budget — the override must win.
+      await vi.advanceTimersByTimeAsync(CREWLY_AGENT_DEFAULTS.MESSAGE_TIMEOUT_MS + 1);
+      expect(settled).not.toHaveBeenCalled();
+
+      const assertion = expect(runPromise).rejects.toBeInstanceOf(AgentRunTimeoutError);
+      await vi.advanceTimersByTimeAsync(
+        CREWLY_AGENT_DEFAULTS.MODEL_TIMEOUT_MS['deepseek-reasoner'],
+      );
+      await assertion;
+    });
+
+    it('leaves a run that completes in time untouched', async () => {
+      mockGenerateText.mockResolvedValueOnce({
+        text: 'fast enough',
+        toolCalls: [],
+        steps: [],
+        usage: { inputTokens: 1, outputTokens: 1 },
+        finishReason: 'stop',
+      });
+
+      await expect(runner.run('quick')).resolves.toMatchObject({ text: 'fast enough' });
+    });
 });

package/packages/crewly-agent/src/runtime/agent-runner.service.ts

@@ -158,7 +158,10 @@ export class ToolCallLoopDetector {
       }
       if (this.consecutiveErrors >= this.errorThreshold) {
         this.loopDetected = true;
-        this.loopReason = `Tool "${toolName}" returned errors ${this.consecutiveErrors} consecutive times. Last result: ${String(result).slice(0, 200)}`;
+        // Serialize objects properly — `String(obj)` yields "[object Object]"
+        // and loses the actual error. Mirror isErrorResult()'s serialization.
+        const resultStr = typeof result === 'string' ? result : JSON.stringify(result);
+        this.loopReason = `Tool "${toolName}" returned errors ${this.consecutiveErrors} consecutive times. Last result: ${resultStr.slice(0, 200)}`;
         return true;
       }
     } else {
@@ -198,6 +201,40 @@ export class ToolCallLoopDetector {
  * const result = await runner.run('Check all team statuses');
  * ```
  */
+/**
+ * Thrown when a single agent run exceeds its wall-clock budget.
+ *
+ * Distinct class (rather than a message-sniffed "timeout" string) because
+ * {@link AgentRunnerService.isRecoverableError} treats anything mentioning
+ * "timeout" as retryable. A hard-budget expiry is deliberately NOT retryable:
+ * the budget covers the whole run including retries, so re-entering the retry
+ * loop would multiply the very stall the budget exists to bound.
+ *
+ * @example
+ * ```typescript
+ * try { await runner.run(msg); }
+ * catch (e) { if (e instanceof AgentRunTimeoutError) recycleAgent(); }
+ * ```
+ */
+export class AgentRunTimeoutError extends Error {
+  /** Wall-clock budget that was exceeded, in milliseconds. */
+  readonly timeoutMs: number;
+
+  /**
+   * @param timeoutMs - The budget that was exceeded, in milliseconds
+   * @param modelId - Model the run was using, for operator diagnosis
+   */
+  constructor(timeoutMs: number, modelId: string) {
+    super(
+      `Agent run exceeded its ${timeoutMs}ms wall-clock budget (model: ${modelId}) `
+      + `and was hard-aborted. The model connection stalled without delivering a `
+      + `response — see CREWLY_AGENT_MESSAGE_TIMEOUT_MS to tune the budget.`,
+    );
+    this.name = 'AgentRunTimeoutError';
+    this.timeoutMs = timeoutMs;
+  }
+}
+
 /** Function type for generateText — used for dependency injection in tests */
 type GenerateTextFn = (opts: Record<string, unknown>) => Promise<Record<string, unknown>>;
 
@@ -824,19 +861,73 @@ export class AgentRunnerService {
       externalAbortSignal.addEventListener('abort', () => runAbort.abort(), { once: true });
     }
 
+    // Wall-clock budget for the ENTIRE run (all retries included).
+    //
+    // Without this, a model connection that opens but then goes silent — no
+    // bytes, no FIN, no error — hangs `await streamResult` forever: undici
+    // applies no idle deadline to a streaming body. That stalled promise wedges
+    // processQueue permanently (`this.processing` stays true), so every later
+    // message queues behind it and the agent goes silently catatonic: the
+    // process is alive and heartbeating, but no run ever completes and no error
+    // is ever raised. Bounding the run converts that invisible hang into a
+    // normal rejection the caller can log, surface, and recover from.
+    const timeoutMs = this.resolveRunTimeoutMs();
+    let hardTimer: ReturnType<typeof setTimeout> | undefined;
+    let softTimer: ReturnType<typeof setTimeout> | undefined;
+
     try {
       // If a test override is set, use generateText path (backward compatible)
-      if (this._generateTextFn) {
-        return await this.executeRunWithGenerateText(tools, runAbort.signal);
-      }
+      const runPromise = this._generateTextFn
+        ? this.executeRunWithGenerateText(tools, runAbort.signal)
+        // Production path: streamText for real-time feedback
+        : this.executeRunWithStreamText(tools, runAbort.signal);
+
+      // The loser of the race stays pending. Attach a no-op catch so its
+      // eventual abort-rejection is never an unhandled rejection (which
+      // crashes the agent process under Node's default policy).
+      runPromise.catch(() => { /* settled by the race below */ });
+
+      const timeoutPromise = new Promise<never>((_, reject) => {
+        softTimer = setTimeout(() => {
+          console.warn(
+            `[AgentRunner] Run exceeded soft warning threshold `
+            + `(${CREWLY_AGENT_DEFAULTS.MESSAGE_SOFT_WARNING_MS}ms) — still waiting on the model.`,
+          );
+        }, CREWLY_AGENT_DEFAULTS.MESSAGE_SOFT_WARNING_MS);
+
+        hardTimer = setTimeout(() => {
+          // Abort first so the in-flight HTTP request and any tool work are
+          // torn down, then reject — a well-behaved provider unwinds on the
+          // signal, and a wedged one no longer holds the queue hostage.
+          runAbort.abort();
+          reject(new AgentRunTimeoutError(timeoutMs, this.config.model.modelId));
+        }, timeoutMs);
+      });
 
-      // Production path: streamText for real-time feedback
-      return await this.executeRunWithStreamText(tools, runAbort.signal);
+      return await Promise.race([runPromise, timeoutPromise]);
     } finally {
+      if (hardTimer) clearTimeout(hardTimer);
+      if (softTimer) clearTimeout(softTimer);
       this.currentRunAbort = null;
     }
   }
 
+  /**
+   * Resolve the wall-clock budget for one run.
+   *
+   * Per-model overrides win over the global default: reasoning models emit
+   * chain-of-thought tokens far slower than plain content, so a budget tuned
+   * for a chat model would kill them mid-thought.
+   *
+   * @returns Budget in milliseconds for the currently configured model
+   */
+  private resolveRunTimeoutMs(): number {
+    return (
+      CREWLY_AGENT_DEFAULTS.MODEL_TIMEOUT_MS[this.config.model.modelId]
+      ?? CREWLY_AGENT_DEFAULTS.MESSAGE_TIMEOUT_MS
+    );
+  }
+
   /**
    * Check if an error is recoverable and eligible for automatic retry.
    *
@@ -850,6 +941,9 @@ export class AgentRunnerService {
    */
   private isRecoverableError(error: unknown): boolean {
     if (!(error instanceof Error)) return false;
+    // Hard-budget expiry is terminal by construction: the budget spans the
+    // whole run, so retrying would just re-enter the stall it bounded.
+    if (error instanceof AgentRunTimeoutError) return false;
     const msg = error.message.toLowerCase();
     const statusMatch = msg.match(/\b(429|5\d{2})\b/);
     if (statusMatch) return true;

package/packages/crewly-agent/src/runtime/tool-registry.test.ts

@@ -2186,6 +2186,25 @@ describe('Tool Registry', () => {
       expect(validateFn('rm -rf ./dist')).toBeNull();
       expect(validateFn('rm -rf node_modules')).toBeNull();
     });
+
+    it('should signal the block is permanent so the model stops retrying', () => {
+      // A blocked command is a policy decision, not a transient failure. When
+      // the message read like a retryable error the model retried variations
+      // until the loop detector aborted the whole run, so the wording carries
+      // real behavioural weight and is worth pinning.
+      const reason = validateFn('kill -9 $$');
+
+      expect(reason).not.toBeNull();
+      expect(reason).toMatch(/permanently blocked/i);
+      expect(reason).toMatch(/NOT a transient error/i);
+      expect(reason).toMatch(/do NOT retry/i);
+    });
+
+    it('should still identify which pattern caused the block', () => {
+      // The matched pattern stays in the message: without it the model cannot
+      // tell which part of a compound command was rejected.
+      expect(validateFn('shutdown -h now')).toMatch(/shutdown/);
+    });
   });
 
   describe('bash_exec tool', () => {

package/packages/crewly-agent/src/runtime/tool-registry.ts

@@ -79,7 +79,10 @@ export const APPROVAL_REQUIRED_BASH_PATTERNS: Array<{ pattern: RegExp; label: st
 export function validateBashCommand(command: string): string | null {
   for (const pattern of BLOCKED_COMMAND_PATTERNS) {
     if (pattern.test(command)) {
-      return `Command blocked by security policy: matches forbidden pattern ${pattern}`;
+      // Phrased to signal the model this is a PERMANENT policy block, not a
+      // transient failure — otherwise it retries variations until the loop
+      // detector aborts the whole run.
+      return `Command permanently blocked by security policy (matched ${pattern}). This is NOT a transient error — do NOT retry this or a variation. Achieve the goal a different way (use a Crewly API/tool, or skip this step).`;
     }
   }
   return null;

package/packages/crewly-agent/src/runtime/types.ts

@@ -540,6 +540,15 @@ export const CREWLY_AGENT_DEFAULTS = {
   /** Soft warning threshold in milliseconds — logs a warning but does not kill the request.
    *  Override via CREWLY_AGENT_MESSAGE_SOFT_WARNING_MS env var. */
   MESSAGE_SOFT_WARNING_MS: Number(process.env.CREWLY_AGENT_MESSAGE_SOFT_WARNING_MS) || 240_000,
+  /** Grace period added on top of the agent's own run budget before the PARENT
+   *  gives up waiting on the child's IPC reply.
+   *
+   *  The child enforces the run budget itself and reports a clean error, so in
+   *  normal operation the parent timer never fires. It exists for the case the
+   *  child can no longer report at all — blocked event loop, broken stdout pipe,
+   *  SIGSTOP. The grace keeps the parent from racing the child and mislabelling
+   *  a normal in-budget timeout as a wedged runtime. */
+  IPC_RUN_TIMEOUT_GRACE_MS: 30_000,
   /** Cooldown period in milliseconds after rate limit retries are exhausted */
   RATE_LIMIT_COOLDOWN_MS: 300_000,
   /** Maximum number of automatic retries for recoverable errors (429, 5xx, network) */