@commonlyai/cli 0.1.7 → 0.1.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@commonlyai/cli",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "license": "Apache-2.0",
   "description": "The Commonly CLI — connect agents, manage pods, iterate fast",
   "type": "module",

package/src/commands/agent.js

@@ -34,6 +34,11 @@ import { detectSkills, importSkills } from '../lib/skills-import.js';
 import { parseEnvironmentFile, resolveWorkspace } from '../lib/environment.js';
 import { detectBwrap } from '../lib/sandbox/bwrap.js';
 import { detectSeatbelt } from '../lib/sandbox/seatbelt.js';
+import {
+  formatRetryDelay,
+  spawnRetryJitter,
+  spawnRetryPolicy,
+} from '../lib/spawn-retry.js';
 
 // ── Token file I/O — ~/.commonly/tokens/<name>.json (ADR-005) ───────────────
 
@@ -502,6 +507,7 @@ export const performRun = ({
   log = () => {},
   onError,
   setTimeoutImpl = setTimeout,
+  retryJitterRatio,
 }) => {
   const client = createClient({ instance: instanceUrl, token });
   let running = true;
@@ -512,6 +518,8 @@ export const performRun = ({
   // reprovision-all; 5+ wastes rate-limit budget after the real-revoke case.
   let consecutiveAuthErrors = 0;
   const MAX_AUTH_ERRORS = 3;
+  let consecutiveSpawnFailures = 0;
+  const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName);
 
   // Adapters default `ctx.cwd` to this path. Node's child_process.spawn
   // rejects with "spawn <bin> ENOENT" when cwd does not exist — same shape
@@ -693,6 +701,7 @@ export const performRun = ({
 
   const tick = async () => {
     if (!running) return;
+    let nextPollDelayMs = intervalMs;
     try {
       const { events = [] } = await client.get('/api/agents/runtime/events', {
         agentName, instanceId, limit: 10,
@@ -705,15 +714,46 @@ export const performRun = ({
           result = { outcome: 'no_action', reason: 'duplicate-delivery' };
           log(`[${event.type}] duplicate delivery ${event._id} — skipping spawn and re-acking`);
         } else {
+          const eventWillSpawn = Boolean(extractPrompt(event) && (event.podId || podId));
           try {
             result = await processEvent(event);
           } catch (err) {
-            // Spawn failed — do not record or ack, so the kernel can re-deliver
-            // the event after the local runtime recovers (ADR-005).
-            log(`[${event.type}] spawn error: ${err.message}`);
-            onError?.(err);
-            continue;
+            // Do not record or ack: the kernel must retain the event for
+            // at-least-once delivery. Stop this fetched batch immediately —
+            // continuing could launch every one of the 10 returned events
+            // into the same provider outage before the next poll (#782).
+            consecutiveSpawnFailures += 1;
+            const retry = spawnRetryPolicy({
+              error: err,
+              consecutiveFailures: consecutiveSpawnFailures,
+              intervalMs,
+              jitterRatio: spawnJitterRatio,
+            });
+            nextPollDelayMs = retry.delayMs;
+            const retryIn = formatRetryDelay(retry.delayMs);
+            const state = retry.circuitOpen ? 'circuit open' : 'retry scheduled';
+            const wrapped = new Error(
+              `${event.type} processing failed (${retry.failureClass}; `
+              + `${consecutiveSpawnFailures} consecutive) — event ${event._id} remains unacked; `
+              + `${state}, next probe in ${retryIn}: ${err.message}`,
+              { cause: err },
+            );
+            Object.assign(wrapped, {
+              code: 'agent_spawn_retry_scheduled',
+              failureClass: retry.failureClass,
+              consecutiveFailures: consecutiveSpawnFailures,
+              retryAfterMs: retry.delayMs,
+              circuitOpen: retry.circuitOpen,
+              eventId: event._id,
+            });
+            log(`[${event.type}] ${wrapped.message}`);
+            onError?.(wrapped);
+            break;
           }
+          // Only a completed model turn proves the local runtime and delivery
+          // path recovered. A malformed/no-destination event is still acked,
+          // but must not erase the failure streak without exercising either.
+          if (eventWillSpawn) consecutiveSpawnFailures = 0;
           // Record after successful processing but before ack. If the ack
           // fails, the next delivery is skipped and re-acked instead of
           // burning a second model turn for work that already completed.
@@ -739,7 +779,7 @@ export const performRun = ({
       }
       onError?.(err);
     }
-    if (running) setTimeoutImpl(tick, intervalMs);
+    if (running) setTimeoutImpl(tick, nextPollDelayMs);
   };
 
   tick();

package/src/lib/adapters/claude.js

@@ -244,7 +244,16 @@ const runClaude = ({ cmd, args, cwd, env, timeoutMs, spawnImpl = childSpawn }) =
   proc.on('close', (code) => {
     clearTimeout(timer);
     if (timedOut) return reject(new Error(`claude timed out after ${timeoutMs}ms`));
-    if (code !== 0) return reject(new Error(`claude exited with code ${code}: ${stderr.trim()}`));
+    if (code !== 0) {
+      // Report stdout too, not just stderr. In `-p` mode claude writes terminal
+      // conditions (usage limits especially) to stdout and exits non-zero with
+      // stderr empty — 361 consecutive failures on 2026-08-03 carried no reason
+      // at all because of this. It is not only a diagnosability problem: the
+      // circuit breaker classifies from the error message, so a blank message
+      // downgrades a hard quota failure to RUNTIME and its shortest backoff.
+      const detail = [stderr.trim(), stdout.trim()].filter(Boolean).join(' | ');
+      return reject(new Error(`claude exited with code ${code}: ${detail.slice(0, 2000)}`));
+    }
     resolve(stdout);
   });
 });

package/src/lib/spawn-retry.js

@@ -0,0 +1,160 @@
+/**
+ * Retry policy for local wrapper event-processing failures (#782).
+ *
+ * The kernel deliberately re-delivers an event that the wrapper does not
+ * acknowledge. That preserves at-least-once handling, but a flat poll loop
+ * turns a model-provider outage into repeated subprocess launches. Keep the
+ * retry policy here so every adapter gets the same bounded behavior.
+ */
+
+export const SPAWN_FAILURE_CLASS = Object.freeze({
+  QUOTA: 'quota',
+  RATE_LIMIT: 'rate_limit',
+  CONFIGURATION: 'configuration',
+  RUNTIME: 'runtime',
+});
+
+export const SPAWN_CIRCUIT_THRESHOLD = 3;
+// Base ceiling before the stable per-agent 0–20% anti-herd offset.
+export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000;
+export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2;
+
+// `out of credits` is codex's exact wording for an exhausted workspace balance
+// ("Your workspace is out of credits. Ask your workspace owner to refill…").
+// Without it that outage classified as RUNTIME and drew the shortest backoff —
+// observed live on 2026-08-03 before this pattern was added.
+const QUOTA_RE = /(?:quota|usage limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i;
+const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i;
+const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i;
+
+const errorText = (error) => [
+  error?.message,
+  error?.stderr,
+  error?.body?.error,
+  error?.body?.message,
+]
+  .filter(Boolean)
+  .map(String)
+  .join('\n');
+
+export const classifySpawnFailure = (error) => {
+  const text = errorText(error);
+  // Provider APIs commonly report an exhausted account quota as HTTP 429.
+  // Prefer the more specific body/message over the generic status code so a
+  // hard quota failure gets the long cooldown rather than a one-minute probe.
+  if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA;
+  if (error?.status === 429 || RATE_LIMIT_RE.test(text)) {
+    return SPAWN_FAILURE_CLASS.RATE_LIMIT;
+  }
+  if (
+    error?.code === 'ENOENT'
+    || error?.status === 401
+    || error?.status === 403
+    || CONFIGURATION_RE.test(text)
+  ) {
+    return SPAWN_FAILURE_CLASS.CONFIGURATION;
+  }
+  return SPAWN_FAILURE_CLASS.RUNTIME;
+};
+
+// A stable per-agent offset keeps a fleet from probing a recovering provider
+// in lockstep. Stability matters: random jitter makes operator logs and tests
+// harder to reason about, while distinct agent names already provide entropy.
+export const spawnRetryJitter = (agentName) => {
+  let hash = 2166136261;
+  for (const char of String(agentName || 'agent')) {
+    hash ^= char.charCodeAt(0);
+    hash = Math.imul(hash, 16777619);
+  }
+  return ((hash >>> 0) % 2001) / 10000;
+};
+
+const applyJitter = (delayMs, jitterRatio) => {
+  const safeJitter = Number.isFinite(jitterRatio)
+    ? Math.min(SPAWN_RETRY_JITTER_MAX_RATIO, Math.max(0, jitterRatio))
+    : 0;
+  return Math.round(delayMs * (1 + safeJitter));
+};
+
+/**
+ * Return the next probe delay and whether the circuit is open.
+ *
+ * Known non-transient failures open immediately. Unknown runtime failures get
+ * two quick retries, then open the circuit on the third consecutive failure.
+ * Later probes back off exponentially to the same 15-minute base ceiling.
+ */
+export const spawnRetryPolicy = ({
+  error,
+  consecutiveFailures,
+  intervalMs,
+  jitterRatio = 0,
+}) => {
+  const failureClass = classifySpawnFailure(error);
+  const safeIntervalMs = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : 5000;
+  const failureCount = Number.isInteger(consecutiveFailures) && consecutiveFailures > 0
+    ? consecutiveFailures
+    : 1;
+
+  if (
+    failureClass === SPAWN_FAILURE_CLASS.QUOTA
+    || failureClass === SPAWN_FAILURE_CLASS.CONFIGURATION
+  ) {
+    return {
+      failureClass,
+      circuitOpen: true,
+      delayMs: applyJitter(SPAWN_RETRY_MAX_MS, jitterRatio),
+    };
+  }
+
+  if (failureClass === SPAWN_FAILURE_CLASS.RATE_LIMIT) {
+    return {
+      failureClass,
+      circuitOpen: true,
+      delayMs: applyJitter(
+        Math.min(
+          SPAWN_RETRY_MAX_MS,
+          60 * 1000 * (2 ** (failureCount - 1)),
+        ),
+        jitterRatio,
+      ),
+    };
+  }
+
+  if (failureCount < SPAWN_CIRCUIT_THRESHOLD) {
+    return {
+      failureClass,
+      circuitOpen: false,
+      delayMs: applyJitter(
+        Math.min(
+          SPAWN_RETRY_MAX_MS,
+          safeIntervalMs * (2 ** (failureCount - 1)),
+        ),
+        jitterRatio,
+      ),
+    };
+  }
+
+  return {
+    failureClass,
+    circuitOpen: true,
+    delayMs: applyJitter(
+      Math.min(
+        SPAWN_RETRY_MAX_MS,
+        60 * 1000 * (2 ** (failureCount - SPAWN_CIRCUIT_THRESHOLD)),
+      ),
+      jitterRatio,
+    ),
+  };
+};
+
+export const formatRetryDelay = (delayMs) => {
+  if (delayMs >= 60000) {
+    const minutes = delayMs / 60000;
+    return `${Number.isInteger(minutes) ? minutes : minutes.toFixed(1)}m`;
+  }
+  if (delayMs >= 1000) {
+    const seconds = delayMs / 1000;
+    return `${Number.isInteger(seconds) ? seconds : seconds.toFixed(1)}s`;
+  }
+  return `${delayMs}ms`;
+};