@iamoberlin/chorus 2.2.0 → 2.3.0

package/src/scheduler.ts

@@ -8,7 +8,9 @@
 import type { OpenClawPluginService, PluginLogger } from "openclaw/plugin-sdk";
 import type { ChorusConfig } from "./config.js";
 import { CHOIRS, shouldRunChoir, CASCADE_ORDER, type Choir } from "./choirs.js";
+import { deliverChoirOutput } from "./delivery.js";
 import { recordExecution, type ChoirExecution } from "./metrics.js";
+import { recordCost, estimateCost } from "./economics.js";
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
 import { join } from "path";
 import { homedir } from "os";
@@ -22,8 +24,8 @@ interface AgentTurnResult {
 }
 
 // ── Delivery ─────────────────────────────────────────────────
-// Choir agents handle their own delivery via OpenClaw messaging tools.
-// The scheduler's job is execution and scheduling — not routing messages.
+// Delivery is handled via shared channel adapter to keep behavior consistent
+// across scheduler and manual CLI execution paths.
 
 interface ChoirContext {
   choirId: string;
@@ -41,6 +43,29 @@ interface ChoirRunState {
 const CHORUS_DIR = join(homedir(), ".chorus");
 const RUN_STATE_PATH = join(CHORUS_DIR, "run-state.json");
 
+// ── 429 Circuit Breaker ──────────────────────────────────────
+// Exponential backoff on rate limit errors. Skips choir execution
+// when backing off, resets on successful runs.
+interface CircuitBreakerState {
+  backoffUntil: number; // timestamp ms — skip all choirs until this time
+  consecutiveFailures: number;
+}
+
+const MAX_BACKOFF_MS = 16 * 60 * 1000; // 16 minutes cap
+const BASE_BACKOFF_MS = 60 * 1000; // 1 minute base
+
+function isRateLimitError(err: unknown): boolean {
+  const msg = String(err).toLowerCase();
+  return msg.includes("429") || msg.includes("rate limit") || msg.includes("rate_limit") || msg.includes("too many requests");
+}
+
+function computeBackoffMs(failures: number): number {
+  const ms = BASE_BACKOFF_MS * Math.pow(2, Math.min(failures - 1, 4));
+  // Add jitter: ±25%
+  const jitter = ms * 0.25 * (Math.random() * 2 - 1);
+  return Math.min(ms + jitter, MAX_BACKOFF_MS);
+}
+
 // Load persisted run state from disk
 function loadRunState(log: PluginLogger): Map<string, ChoirRunState> {
   const state = new Map<string, ChoirRunState>();
@@ -106,12 +131,15 @@ export function createChoirScheduler(
   // Load persisted state instead of starting fresh
   const runState = loadRunState(log);
 
+  // Circuit breaker state
+  const circuitBreaker: CircuitBreakerState = { backoffUntil: 0, consecutiveFailures: 0 };
+
   // CLI fallback for executing choirs when plugin API is unavailable
   async function executeChoirViaCli(choir: Choir, prompt: string): Promise<string> {
     const result = await new Promise<{ status: number | null; stdout: string; stderr: string }>((resolve) => {
       const child = spawn('openclaw', [
         'agent',
-        '--session-id', `chorus:${choir.id}`,
+        '--session-id', `chorus-${choir.id}`,
         '--message', prompt,
         '--json',
       ], { stdio: ['pipe', 'pipe', 'pipe'] });
@@ -190,7 +218,7 @@ export function createChoirScheduler(
       if (typeof api.runAgentTurn === 'function') {
         try {
           const result: AgentTurnResult = await api.runAgentTurn({
-            sessionLabel: `chorus:${choir.id}`,
+            sessionLabel: `chorus-${choir.id}`,
             message: prompt,
             isolated: true,
             timeoutSeconds: 300,
@@ -208,6 +236,17 @@ export function createChoirScheduler(
             output = result.text;
           }
         } catch (apiErr) {
+          if (isRateLimitError(apiErr)) {
+            circuitBreaker.consecutiveFailures++;
+            const backoffMs = computeBackoffMs(circuitBreaker.consecutiveFailures);
+            circuitBreaker.backoffUntil = Date.now() + backoffMs;
+            log.warn(`[chorus] ⚡ 429 RATE LIMIT on ${choir.name} — backing off ${(backoffMs/1000).toFixed(0)}s (failure #${circuitBreaker.consecutiveFailures})`);
+            execution.durationMs = Date.now() - startTime;
+            execution.success = false;
+            execution.error = "429_rate_limit_backoff";
+            recordExecution(execution);
+            return; // Skip this choir and remaining choirs will be skipped by checkAndRunChoirs
+          }
           log.warn(`[chorus] API runAgentTurn failed for ${choir.name}, falling back to CLI: ${apiErr}`);
           output = await executeChoirViaCli(choir, prompt);
         }
@@ -221,6 +260,21 @@ export function createChoirScheduler(
       execution.outputLength = output.length;
       execution.tokensUsed = estimateTokens(output);
 
+      // Record economics cost (only for runs that produced output)
+      if (execution.success) {
+        const inputTokensEst = Math.ceil(prompt.length / 4);
+        const outputTokensEst = execution.tokensUsed || Math.ceil(output.length / 4);
+        recordCost({
+          choirId: choir.id,
+          timestamp: new Date().toISOString(),
+          inputTokens: inputTokensEst,
+          outputTokens: outputTokensEst,
+          model: "default",
+          inferenceMs: execution.durationMs,
+          costUsd: estimateCost(inputTokensEst, outputTokensEst),
+        });
+      }
+
       // Parse output for metrics (findings, alerts, improvements)
       execution.findings = countFindings(output);
       execution.alerts = countAlerts(output);
@@ -243,8 +297,18 @@ export function createChoirScheduler(
       // Persist state to disk after each run
       saveRunState(runState, log);
 
+      // Reset circuit breaker on success
+      if (circuitBreaker.consecutiveFailures > 0) {
+        log.info(`[chorus] ✅ Circuit breaker reset after ${circuitBreaker.consecutiveFailures} failures`);
+        circuitBreaker.consecutiveFailures = 0;
+        circuitBreaker.backoffUntil = 0;
+      }
+
       log.info(`[chorus] ${choir.emoji} ${choir.name} completed (${(execution.durationMs/1000).toFixed(1)}s)`);
 
+      // Deliver output via shared adapter for consistent routing/format behavior
+      await deliverChoirOutput(api, choir, output, log);
+
       // Log illumination flow
       if (choir.passesTo.length > 0) {
         log.debug(`[chorus] Illumination ready for: ${choir.passesTo.join(", ")}`);
@@ -318,10 +382,20 @@ export function createChoirScheduler(
     return improvements.slice(0, 5); // Cap at 5
   }
 
+  // Track choirs currently executing to prevent duplicate runs
+  const executing = new Set<string>();
+
   // Check and run due choirs
   async function checkAndRunChoirs(): Promise<void> {
     const now = new Date();
 
+    // Circuit breaker: skip entire cycle if backing off from rate limit
+    if (Date.now() < circuitBreaker.backoffUntil) {
+      const remainingSec = ((circuitBreaker.backoffUntil - Date.now()) / 1000).toFixed(0);
+      log.debug(`[chorus] ⚡ Circuit breaker active — ${remainingSec}s remaining`);
+      return;
+    }
+
     // Check choirs in cascade order (important for illumination flow)
     for (const choirId of CASCADE_ORDER) {
       const choir = CHOIRS[choirId];
@@ -332,10 +406,20 @@ export function createChoirScheduler(
         continue;
       }
 
+      // Skip if already executing (prevents duplicate runs for slow choirs)
+      if (executing.has(choirId)) {
+        continue;
+      }
+
       // Check if due based on interval
       const state = runState.get(choirId);
       if (shouldRunChoir(choir, now, state?.lastRun)) {
-        await executeChoir(choir);
+        executing.add(choirId);
+        try {
+          await executeChoir(choir);
+        } finally {
+          executing.delete(choirId);
+        }
       }
     }
   }