bosun 0.35.0 → 0.35.2

package/agent-pool.mjs

@@ -45,6 +45,11 @@ import { loadConfig } from "./config.mjs";
 import { resolveRepoRoot, resolveAgentRepoRoot } from "./repo-root.mjs";
 import { resolveCodexProfileRuntime } from "./codex-model-profiles.mjs";
 import { getGitHubToken } from "./github-auth-manager.mjs";
+import {
+  isTransientStreamError,
+  streamRetryDelay,
+  MAX_STREAM_RETRIES,
+} from "./stream-resilience.mjs";
 
 // Lazy-load MCP registry to avoid circular dependencies.
 // Cached at module scope per AGENTS.md hard rules.
@@ -1053,6 +1058,24 @@ async function launchCodexThread(prompt, cwd, timeoutMs, extra = {}) {
         threadId: null,
       };
     }
+    // ── Transient stream / network error ─ retry without resetting thread state ─
+    if (isTransientStreamError(err)) {
+      const retryAttempt = (extra._streamRetryAttempt || 0) + 1;
+      if (retryAttempt < MAX_STREAM_RETRIES) {
+        const delay = streamRetryDelay(retryAttempt - 1);
+        console.warn(
+          `${TAG} codex transient stream error (attempt ${retryAttempt}/${MAX_STREAM_RETRIES}): ${err.message || err} — retrying in ${Math.round(delay)}ms`,
+        );
+        await new Promise((r) => setTimeout(r, delay));
+        return launchCodexThread(prompt, cwd, timeoutMs, {
+          ...extra,
+          _streamRetryAttempt: retryAttempt,
+        });
+      }
+      console.error(
+        `${TAG} codex stream disconnection not resolved after ${MAX_STREAM_RETRIES} attempts`,
+      );
+    }
     return {
       success: false,
       output: "",
@@ -1472,6 +1495,24 @@ async function launchCopilotThread(prompt, cwd, timeoutMs, extra = {}) {
         threadId: resumeThreadId,
       };
     }
+    // ── Transient stream / network error ─ retry with a fresh Copilot session ─
+    if (!isTimeout && !isIdleWaitTimeout && isTransientStreamError(err)) {
+      const retryAttempt = (extra._streamRetryAttempt || 0) + 1;
+      if (retryAttempt < MAX_STREAM_RETRIES) {
+        const delay = streamRetryDelay(retryAttempt - 1);
+        console.warn(
+          `${TAG} copilot transient stream error (attempt ${retryAttempt}/${MAX_STREAM_RETRIES}): ${errMsg} — retrying in ${Math.round(delay)}ms`,
+        );
+        await new Promise((r) => setTimeout(r, delay));
+        return launchCopilotThread(prompt, cwd, timeoutMs, {
+          ...extra,
+          _streamRetryAttempt: retryAttempt,
+        });
+      }
+      console.error(
+        `${TAG} copilot stream disconnection not resolved after ${MAX_STREAM_RETRIES} attempts`,
+      );
+    }
     return {
       success: false,
       output: "",
@@ -1846,6 +1887,24 @@ async function launchClaudeThread(prompt, cwd, timeoutMs, extra = {}) {
         threadId: resumeThreadId,
       };
     }
+    // ── Transient stream / network error ─ retry with a fresh Claude query ──
+    if (isTransientStreamError(err)) {
+      const retryAttempt = (extra._streamRetryAttempt || 0) + 1;
+      if (retryAttempt < MAX_STREAM_RETRIES) {
+        const delay = streamRetryDelay(retryAttempt - 1);
+        console.warn(
+          `${TAG} claude transient stream error (attempt ${retryAttempt}/${MAX_STREAM_RETRIES}): ${err.message || err} — retrying in ${Math.round(delay)}ms`,
+        );
+        await new Promise((r) => setTimeout(r, delay));
+        return launchClaudeThread(prompt, cwd, timeoutMs, {
+          ...extra,
+          _streamRetryAttempt: retryAttempt,
+        });
+      }
+      console.error(
+        `${TAG} claude stream disconnection not resolved after ${MAX_STREAM_RETRIES} attempts`,
+      );
+    }
     return {
       success: false,
       output: "",

package/claude-shell.mjs

@@ -11,6 +11,11 @@ import { resolve } from "node:path";
 import { homedir } from "node:os";
 import { fileURLToPath } from "node:url";
 import { resolveRepoRoot } from "./repo-root.mjs";
+import {
+  isTransientStreamError,
+  streamRetryDelay,
+  MAX_STREAM_RETRIES,
+} from "./stream-resilience.mjs";
 
 const __dirname = resolve(fileURLToPath(new URL(".", import.meta.url)));
 
@@ -471,7 +476,7 @@ export async function execClaudePrompt(userMessage, options = {}) {
     abortController = null,
   } = options;
 
-  if (activeTurn) {
+  if (activeTurn && !options._holdActiveTurn) {
     return {
       finalResponse:
         "⏳ Agent is still executing a previous task. Please wait.",
@@ -489,7 +494,9 @@ export async function execClaudePrompt(userMessage, options = {}) {
     };
   }
 
-  activeTurn = true;
+  if (!options._holdActiveTurn) activeTurn = true;
+  /** Sentinel: true while a retry call is pending so finally skips cleanup. */
+  let _retryPending = false;
   toolUseById.clear();
 
   const transport = resolveClaudeTransport();
@@ -643,6 +650,33 @@ export async function execClaudePrompt(userMessage, options = {}) {
           : `⏱️ Agent timed out after ${timeoutMs / 1000}s`;
       return { finalResponse: msg, items: [], usage: null };
     }
+    // ── Transient stream retry ──────────────────────────────────────────────────
+    // Network/stream blips are safe to retry without resetting session state.
+    // _retryPending keeps the finally block from releasing activeTurn early.
+    if (isTransientStreamError(err)) {
+      const retryAttempt = (options._streamRetryAttempt || 0) + 1;
+      if (retryAttempt < MAX_STREAM_RETRIES) {
+        if (activeQueue) activeQueue.close();
+        activeQueue = null;
+        activeQuery = null;
+        toolUseById.clear();
+        const delay = streamRetryDelay(retryAttempt - 1);
+        console.warn(
+          `[claude-shell] transient stream error (attempt ${retryAttempt}/${MAX_STREAM_RETRIES}): ${err.message || err} — retrying in ${Math.round(delay)}ms`,
+        );
+        _retryPending = true; // prevent outer finally from releasing activeTurn
+        await new Promise((r) => setTimeout(r, delay));
+        // _retryPending stays true through the return so outer finally skips cleanup
+        return execClaudePrompt(userMessage, {
+          ...options,
+          _streamRetryAttempt: retryAttempt,
+          _holdActiveTurn: true, // skip activeTurn guard in recursive call
+        });
+      }
+      console.error(
+        `[claude-shell] stream disconnection not resolved after ${MAX_STREAM_RETRIES} attempts`,
+      );
+    }
     const message = err?.message || String(err || "unknown error");
     return {
       finalResponse: `❌ Claude agent failed: ${message}`,
@@ -650,10 +684,14 @@ export async function execClaudePrompt(userMessage, options = {}) {
       usage: null,
     };
   } finally {
-    if (activeQueue) activeQueue.close();
-    activeQueue = null;
-    activeQuery = null;
-    activeTurn = false;
+    // Only the outermost invocation (or the final retry) cleans up.
+    // When _retryPending is true, the recursive retry call owns the activeTurn lock.
+    if (!_retryPending) {
+      if (activeQueue) activeQueue.close();
+      activeQueue = null;
+      activeQuery = null;
+      activeTurn = false;
+    }
   }
 }
 

package/codex-shell.mjs

@@ -18,12 +18,18 @@ import { fileURLToPath } from "node:url";
 import { resolveAgentSdkConfig } from "./agent-sdk.mjs";
 import { resolveRepoRoot } from "./repo-root.mjs";
 import { resolveCodexProfileRuntime } from "./codex-model-profiles.mjs";
+import {
+  isTransientStreamError,
+  streamRetryDelay,
+  MAX_STREAM_RETRIES,
+} from "./stream-resilience.mjs";
 
 const __dirname = resolve(fileURLToPath(new URL(".", import.meta.url)));
 
 // ── Configuration ────────────────────────────────────────────────────────────
 
 const DEFAULT_TIMEOUT_MS = 60 * 60 * 1000; // 60 min for agentic tasks (matches Azure stream timeout)
+// MAX_STREAM_RETRIES, isTransientStreamError, streamRetryDelay ← imported from ./stream-resilience.mjs
 const STATE_FILE = resolve(__dirname, "logs", "codex-shell-state.json");
 const SESSIONS_DIR = resolve(__dirname, "logs", "sessions");
 const MAX_PERSISTENT_TURNS = 50;
@@ -554,30 +560,32 @@ export async function execCodexPrompt(userMessage, options = {}) {
     }
     // else: persistent && same session && under limit → reuse activeThread
 
-    for (let attempt = 0; attempt < 2; attempt += 1) {
-      const thread = await getThread();
+    // Build the user prompt with optional status context (built once, reused across retries)
+    let prompt = userMessage;
+    if (statusData) {
+      const statusSnippet = JSON.stringify(statusData, null, 2).slice(0, 2000);
+      prompt = `[Orchestrator Status]\n\`\`\`json\n${statusSnippet}\n\`\`\`\n\n# YOUR TASK — EXECUTE NOW\n\n${userMessage}\n\n---\nDo NOT respond with "Ready" or ask what to do. EXECUTE this task. Read files, run commands, produce detailed output.`;
+    } else {
+      prompt = `${userMessage}\n\n\n# YOUR TASK — EXECUTE NOW\n\n\n---\nDo NOT respond with "Ready" or ask what to do. EXECUTE this task. Read files, run commands, produce detailed output & complete the user's request E2E.`;
+    }
+    // Sanitize & size-guard once — prevents invalid_request_error from oversized
+    // bodies (BytePositionInLine > 80 000) or unescaped control characters.
+    const safePrompt = sanitizeAndTruncatePrompt(prompt);
 
-      // Build the user prompt with optional status context
-      let prompt = userMessage;
-      if (statusData) {
-        const statusSnippet = JSON.stringify(statusData, null, 2).slice(
-          0,
-          2000,
-        );
-        prompt = `[Orchestrator Status]\n\`\`\`json\n${statusSnippet}\n\`\`\`\n\n# YOUR TASK — EXECUTE NOW\n\n${userMessage}\n\n---\nDo NOT respond with "Ready" or ask what to do. EXECUTE this task. Read files, run commands, produce detailed output.`;
-      } else {
-        prompt = `${userMessage}\n\n\n# YOUR TASK — EXECUTE NOW\n\n\n---\nDo NOT respond with "Ready" or ask what to do. EXECUTE this task. Read files, run commands, produce detailed output & complete the user's request E2E.`;
-      }
+    let threadResetDone = false;
 
-      // Set up timeout
+    for (let attempt = 0; attempt < MAX_STREAM_RETRIES; attempt += 1) {
+      const thread = await getThread();
+
+      // Each attempt gets a fresh AbortController tied to the same timeout budget.
+      // We intentionally do NOT share the same controller across retries: if the
+      // first attempt times out the signal is already aborted and the retry would
+      // immediately fail.  The total wall-clock budget is still bounded by the
+      // outer timeoutMs passed in.
       const controller = abortController || new AbortController();
       const timer = setTimeout(() => controller.abort("timeout"), timeoutMs);
 
       try {
-        // Sanitize & size-guard before sending — prevents invalid_request_error
-        // from oversized bodies (BytePositionInLine > 80 000) or control chars.
-        const safePrompt = sanitizeAndTruncatePrompt(prompt);
-
         // Use runStreamed for real-time event streaming
         const streamedTurn = await thread.runStreamed(safePrompt, {
           signal: controller.signal,
@@ -585,6 +593,7 @@ export async function execCodexPrompt(userMessage, options = {}) {
 
         let finalResponse = "";
         const allItems = [];
+        let turnFailedErr = null;
 
         // Process events from the async generator
         for await (const event of streamedTurn.events) {
@@ -594,6 +603,13 @@ export async function execCodexPrompt(userMessage, options = {}) {
             await saveState();
           }
 
+          // turn.failed is emitted by the SDK when the server signals response.failed.
+          // Convert it into a retriable error so the retry loop can back off & retry.
+          if (event.type === "turn.failed") {
+            const detail = event.error?.message || "response.failed";
+            turnFailedErr = new Error(`stream disconnected before completion: ${detail}`);
+          }
+
           // Format and emit event
           if (onEvent) {
             const formatted = formatEvent(event);
@@ -628,6 +644,10 @@ export async function execCodexPrompt(userMessage, options = {}) {
           }
         }
 
+        // If a turn.failed event was seen during the stream, treat it as a
+        // transient stream error so the retry loop handles it correctly.
+        if (turnFailedErr) throw turnFailedErr;
+
         clearTimeout(timer);
 
         return {
@@ -638,6 +658,7 @@ export async function execCodexPrompt(userMessage, options = {}) {
         };
       } catch (err) {
         clearTimeout(timer);
+
         if (err.name === "AbortError") {
           const reason = controller.signal.reason;
           const msg =
@@ -646,18 +667,44 @@ export async function execCodexPrompt(userMessage, options = {}) {
               : `⏱️ Agent timed out after ${timeoutMs / 1000}s`;
           return { finalResponse: msg, items: [], usage: null };
         }
-        if (attempt === 0 && isRecoverableThreadError(err)) {
+
+        // ── Thread corruption errors: reset thread & retry once ──────────────
+        if (!threadResetDone && isRecoverableThreadError(err)) {
           console.warn(
             `[codex-shell] recoverable thread error: ${err.message || err} — resetting thread`,
           );
           await resetThread();
-          continue;
+          threadResetDone = true;
+          continue; // retry without counting against stream-retry budget
         }
+
+        // ── Transient stream/network errors: backoff & retry ─────────────────
+        if (isTransientStreamError(err)) {
+          const attemptsLeft = MAX_STREAM_RETRIES - 1 - attempt;
+          if (attemptsLeft > 0) {
+            const delay = streamRetryDelay(attempt);
+            console.warn(
+              `[codex-shell] transient stream error (attempt ${attempt + 1}/${MAX_STREAM_RETRIES}): ${err.message || err} — retrying in ${Math.round(delay)}ms`,
+            );
+            await new Promise((r) => setTimeout(r, delay));
+            continue;
+          }
+          // Exhausted all retries
+          console.error(
+            `[codex-shell] stream disconnection not resolved after ${MAX_STREAM_RETRIES} attempts — giving up`,
+          );
+          return {
+            finalResponse: `❌ Stream disconnected after ${MAX_STREAM_RETRIES} retries: ${err.message}`,
+            items: [],
+            usage: null,
+          };
+        }
+
         throw err;
       }
     }
     return {
-      finalResponse: "❌ Agent failed after retry.",
+      finalResponse: "❌ Agent failed after all retry attempts.",
       items: [],
       usage: null,
     };

package/copilot-shell.mjs

@@ -14,6 +14,11 @@ import { fileURLToPath } from "node:url";
 import { execSync } from "node:child_process";
 import { resolveRepoRoot } from "./repo-root.mjs";
 import { getGitHubToken } from "./github-auth-manager.mjs";
+import {
+  isTransientStreamError,
+  streamRetryDelay,
+  MAX_STREAM_RETRIES,
+} from "./stream-resilience.mjs";
 
 const __dirname = resolve(fileURLToPath(new URL(".", import.meta.url)));
 
@@ -710,7 +715,7 @@ export async function execCopilotPrompt(userMessage, options = {}) {
     persistent = false,
   } = options;
 
-  if (activeTurn) {
+  if (activeTurn && !options._holdActiveTurn) {
     return {
       finalResponse:
         "⏳ Agent is still executing a previous task. Please wait.",
@@ -719,7 +724,9 @@ export async function execCopilotPrompt(userMessage, options = {}) {
     };
   }
 
-  activeTurn = true;
+  if (!options._holdActiveTurn) activeTurn = true;
+  /** Sentinel: true while a retry call is pending so finally skips cleanup. */
+  let _retryPending = false;
 
   if (!persistent) {
     // Task executor path — fresh session each call
@@ -841,22 +848,54 @@ export async function execCopilotPrompt(userMessage, options = {}) {
           : `⏱️ Agent timed out after ${timeoutMs / 1000}s`;
       return { finalResponse: msg, items: [], usage: null };
     }
+    // ── Transient stream retry ──────────────────────────────────────────────────
+    // Reset session and re-run; _retryPending holds the activeTurn lock.
+    if (isTransientStreamError(err)) {
+      const retryAttempt = (options._streamRetryAttempt || 0) + 1;
+      if (retryAttempt < MAX_STREAM_RETRIES) {
+        if (typeof unsubscribe === "function") try { unsubscribe(); } catch { /* best effort */ }
+        unsubscribe = null;
+        activeSession = null; // force getSession() to create a fresh session
+        const delay = streamRetryDelay(retryAttempt - 1);
+        console.warn(
+          `[copilot-shell] transient stream error (attempt ${retryAttempt}/${MAX_STREAM_RETRIES}): ${err.message || err} — retrying in ${Math.round(delay)}ms`,
+        );
+        _retryPending = true; // prevent outer finally from releasing activeTurn
+        await new Promise((r) => setTimeout(r, delay));
+        return execCopilotPrompt(userMessage, {
+          ...options,
+          _streamRetryAttempt: retryAttempt,
+          _holdActiveTurn: true, // skip activeTurn guard in recursive call
+        });
+      }
+      console.error(
+        `[copilot-shell] stream disconnection not resolved after ${MAX_STREAM_RETRIES} attempts`,
+      );
+      return {
+        finalResponse: `❌ Stream disconnected after ${MAX_STREAM_RETRIES} retries: ${err.message}`,
+        items: [],
+        usage: null,
+      };
+    }
     throw err;
   } finally {
-    if (typeof unsubscribe === "function") {
-      try {
-        unsubscribe();
-      } catch {
-        /* best effort */
-      }
-    } else if (typeof session.off === "function") {
-      try {
-        session.off(handleEvent);
-      } catch {
-        /* best effort */
+    // Only the outermost invocation (or the final retry) cleans up.
+    if (!_retryPending) {
+      if (typeof unsubscribe === "function") {
+        try {
+          unsubscribe();
+        } catch {
+          /* best effort */
+        }
+      } else if (typeof session.off === "function") {
+        try {
+          session.off(handleEvent);
+        } catch {
+          /* best effort */
+        }
       }
+      activeTurn = false;
     }
-    activeTurn = false;
   }
 }
 

package/maintenance.mjs

@@ -1019,6 +1019,55 @@ export function syncLocalTrackingBranches(repoRoot, branches) {
 
       if (behind === 0 && ahead === 0) continue; // Already in sync
 
+      // 'main' must always mirror upstream — NEVER push local commits.
+      // Hard-reset instead of rebase-push to prevent workspace drift.
+      if (branch === "main" && ahead > 0) {
+        if (branch === currentBranch) {
+          const reset = spawnSync("git", ["reset", "--hard", remoteRef], {
+            cwd: repoRoot,
+            encoding: "utf8",
+            timeout: 10_000,
+            windowsHide: true,
+          });
+          if (reset.status === 0) {
+            logThrottledBranchSync(
+              `sync:${branch}:hard-reset`,
+              `[maintenance] hard-reset 'main' to ${remoteRef} (was ${ahead}\u2191 ${behind}\u2193) — main must not diverge from upstream`,
+              "warn",
+            );
+            synced++;
+          } else {
+            logThrottledBranchSync(
+              `sync:${branch}:hard-reset-failed`,
+              `[maintenance] hard-reset of 'main' to ${remoteRef} failed`,
+              "error",
+            );
+          }
+        } else {
+          // Not checked out — force-update the ref directly
+          const update = spawnSync(
+            "git",
+            ["update-ref", `refs/heads/${branch}`, `refs/remotes/${remoteRef}`],
+            { cwd: repoRoot, timeout: 5000, windowsHide: true },
+          );
+          if (update.status === 0) {
+            logThrottledBranchSync(
+              `sync:${branch}:force-update-ref`,
+              `[maintenance] force-updated 'main' ref to ${remoteRef} (was ${ahead}\u2191 ${behind}\u2193) — discarded local-only commits`,
+              "warn",
+            );
+            synced++;
+          } else {
+            logThrottledBranchSync(
+              `sync:${branch}:force-update-ref-failed`,
+              `[maintenance] force-update of 'main' ref to ${remoteRef} failed`,
+              "error",
+            );
+          }
+        }
+        continue;
+      }
+
       // Local is ahead of remote but not behind — try a plain push
       if (behind === 0 && ahead > 0) {
         const push = spawnSync(