@iamoberlin/chorus 2.2.1 → 2.3.0

package/src/scheduler.ts

@@ -8,7 +8,9 @@
 import type { OpenClawPluginService, PluginLogger } from "openclaw/plugin-sdk";
 import type { ChorusConfig } from "./config.js";
 import { CHOIRS, shouldRunChoir, CASCADE_ORDER, type Choir } from "./choirs.js";
+import { deliverChoirOutput } from "./delivery.js";
 import { recordExecution, type ChoirExecution } from "./metrics.js";
+import { recordCost, estimateCost } from "./economics.js";
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
 import { join } from "path";
 import { homedir } from "os";
@@ -22,8 +24,8 @@ interface AgentTurnResult {
 }
 
 // ── Delivery ─────────────────────────────────────────────────
-// Choir agents handle their own delivery via OpenClaw messaging tools.
-// The scheduler's job is execution and scheduling — not routing messages.
+// Delivery is handled via shared channel adapter to keep behavior consistent
+// across scheduler and manual CLI execution paths.
 
 interface ChoirContext {
   choirId: string;
@@ -41,6 +43,29 @@ interface ChoirRunState {
 const CHORUS_DIR = join(homedir(), ".chorus");
 const RUN_STATE_PATH = join(CHORUS_DIR, "run-state.json");
 
+// ── 429 Circuit Breaker ──────────────────────────────────────
+// Exponential backoff on rate limit errors. Skips choir execution
+// when backing off, resets on successful runs.
+interface CircuitBreakerState {
+  backoffUntil: number; // timestamp ms — skip all choirs until this time
+  consecutiveFailures: number;
+}
+
+const MAX_BACKOFF_MS = 16 * 60 * 1000; // 16 minutes cap
+const BASE_BACKOFF_MS = 60 * 1000; // 1 minute base
+
+function isRateLimitError(err: unknown): boolean {
+  const msg = String(err).toLowerCase();
+  return msg.includes("429") || msg.includes("rate limit") || msg.includes("rate_limit") || msg.includes("too many requests");
+}
+
+function computeBackoffMs(failures: number): number {
+  const ms = BASE_BACKOFF_MS * Math.pow(2, Math.min(failures - 1, 4));
+  // Add jitter: ±25%
+  const jitter = ms * 0.25 * (Math.random() * 2 - 1);
+  return Math.min(ms + jitter, MAX_BACKOFF_MS);
+}
+
 // Load persisted run state from disk
 function loadRunState(log: PluginLogger): Map<string, ChoirRunState> {
   const state = new Map<string, ChoirRunState>();
@@ -106,12 +131,15 @@ export function createChoirScheduler(
   // Load persisted state instead of starting fresh
   const runState = loadRunState(log);
 
+  // Circuit breaker state
+  const circuitBreaker: CircuitBreakerState = { backoffUntil: 0, consecutiveFailures: 0 };
+
   // CLI fallback for executing choirs when plugin API is unavailable
   async function executeChoirViaCli(choir: Choir, prompt: string): Promise<string> {
     const result = await new Promise<{ status: number | null; stdout: string; stderr: string }>((resolve) => {
       const child = spawn('openclaw', [
         'agent',
-        '--session-id', `chorus:${choir.id}`,
+        '--session-id', `chorus-${choir.id}`,
         '--message', prompt,
         '--json',
       ], { stdio: ['pipe', 'pipe', 'pipe'] });
@@ -190,7 +218,7 @@ export function createChoirScheduler(
       if (typeof api.runAgentTurn === 'function') {
         try {
           const result: AgentTurnResult = await api.runAgentTurn({
-            sessionLabel: `chorus:${choir.id}`,
+            sessionLabel: `chorus-${choir.id}`,
             message: prompt,
             isolated: true,
             timeoutSeconds: 300,
@@ -208,6 +236,17 @@ export function createChoirScheduler(
             output = result.text;
           }
         } catch (apiErr) {
+          if (isRateLimitError(apiErr)) {
+            circuitBreaker.consecutiveFailures++;
+            const backoffMs = computeBackoffMs(circuitBreaker.consecutiveFailures);
+            circuitBreaker.backoffUntil = Date.now() + backoffMs;
+            log.warn(`[chorus] ⚡ 429 RATE LIMIT on ${choir.name} — backing off ${(backoffMs/1000).toFixed(0)}s (failure #${circuitBreaker.consecutiveFailures})`);
+            execution.durationMs = Date.now() - startTime;
+            execution.success = false;
+            execution.error = "429_rate_limit_backoff";
+            recordExecution(execution);
+            return; // Skip this choir and remaining choirs will be skipped by checkAndRunChoirs
+          }
           log.warn(`[chorus] API runAgentTurn failed for ${choir.name}, falling back to CLI: ${apiErr}`);
           output = await executeChoirViaCli(choir, prompt);
         }
@@ -221,6 +260,21 @@ export function createChoirScheduler(
       execution.outputLength = output.length;
       execution.tokensUsed = estimateTokens(output);
 
+      // Record economics cost (only for runs that produced output)
+      if (execution.success) {
+        const inputTokensEst = Math.ceil(prompt.length / 4);
+        const outputTokensEst = execution.tokensUsed || Math.ceil(output.length / 4);
+        recordCost({
+          choirId: choir.id,
+          timestamp: new Date().toISOString(),
+          inputTokens: inputTokensEst,
+          outputTokens: outputTokensEst,
+          model: "default",
+          inferenceMs: execution.durationMs,
+          costUsd: estimateCost(inputTokensEst, outputTokensEst),
+        });
+      }
+
       // Parse output for metrics (findings, alerts, improvements)
       execution.findings = countFindings(output);
       execution.alerts = countAlerts(output);
@@ -243,69 +297,17 @@ export function createChoirScheduler(
       // Persist state to disk after each run
       saveRunState(runState, log);
 
-      log.info(`[chorus] ${choir.emoji} ${choir.name} completed (${(execution.durationMs/1000).toFixed(1)}s)`);
-
-      // Deliver output to user via OpenClaw messaging if choir is marked for delivery
-      // Reads target from OpenClaw config (channels.*.allowFrom) — no hardcoded PII
-      if (choir.delivers && output && output !== "(no response)" && output !== "HEARTBEAT_OK" && output !== "NO_REPLY") {
-        const channels = api.config?.channels as Record<string, any> | undefined;
-        let target: string | undefined;
-        let channel: string | undefined;
-
-        if (channels) {
-          for (const [ch, cfg] of Object.entries(channels)) {
-            if (cfg?.enabled && cfg?.allowFrom?.[0]) {
-              target = cfg.allowFrom[0];
-              channel = ch;
-              break;
-            }
-          }
-        }
+      // Reset circuit breaker on success
+      if (circuitBreaker.consecutiveFailures > 0) {
+        log.info(`[chorus] ✅ Circuit breaker reset after ${circuitBreaker.consecutiveFailures} failures`);
+        circuitBreaker.consecutiveFailures = 0;
+        circuitBreaker.backoffUntil = 0;
+      }
 
-        if (target) {
-          // Strip markdown for channels that don't support it
-          let deliveryText = output.slice(0, 4000);
-          if (channel === 'imessage') {
-            deliveryText = deliveryText
-              .replace(/\*\*(.+?)\*\*/g, '$1')    // bold
-              .replace(/\*(.+?)\*/g, '$1')          // italic
-              .replace(/__(.+?)__/g, '$1')          // bold alt
-              .replace(/_(.+?)_/g, '$1')            // italic alt
-              .replace(/`(.+?)`/g, '$1')            // inline code
-              .replace(/```[\s\S]*?```/g, '')       // code blocks
-              .replace(/^#{1,6}\s+/gm, '')          // headers
-              .replace(/^\s*[-*+]\s+/gm, '• ')     // bullet lists
-              .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); // links
-          }
+      log.info(`[chorus] ${choir.emoji} ${choir.name} completed (${(execution.durationMs/1000).toFixed(1)}s)`);
 
-          try {
-            const args = [
-              'message', 'send',
-              '--target', target,
-              '--message', deliveryText,
-            ];
-            if (channel) args.push('--channel', channel);
-
-            const deliveryProc = spawn('openclaw', args, { stdio: ['pipe', 'pipe', 'pipe'] });
-
-            deliveryProc.on('close', (code) => {
-              if (code === 0) {
-                log.info(`[chorus] 📨 ${choir.name} output delivered via ${channel || 'default'}`);
-              } else {
-                log.warn(`[chorus] ⚠ ${choir.name} delivery failed (exit ${code})`);
-              }
-            });
-
-            deliveryProc.on('error', (err) => {
-              log.warn(`[chorus] ⚠ ${choir.name} delivery error: ${err.message}`);
-            });
-          } catch (deliveryErr) {
-            log.warn(`[chorus] ⚠ ${choir.name} delivery error: ${deliveryErr}`);
-          }
-        } else {
-          log.warn(`[chorus] ⚠ No delivery target found in OpenClaw config for ${choir.name}`);
-        }
-      }
+      // Deliver output via shared adapter for consistent routing/format behavior
+      await deliverChoirOutput(api, choir, output, log);
 
       // Log illumination flow
       if (choir.passesTo.length > 0) {
@@ -380,10 +382,20 @@ export function createChoirScheduler(
     return improvements.slice(0, 5); // Cap at 5
   }
 
+  // Track choirs currently executing to prevent duplicate runs
+  const executing = new Set<string>();
+
   // Check and run due choirs
   async function checkAndRunChoirs(): Promise<void> {
     const now = new Date();
 
+    // Circuit breaker: skip entire cycle if backing off from rate limit
+    if (Date.now() < circuitBreaker.backoffUntil) {
+      const remainingSec = ((circuitBreaker.backoffUntil - Date.now()) / 1000).toFixed(0);
+      log.debug(`[chorus] ⚡ Circuit breaker active — ${remainingSec}s remaining`);
+      return;
+    }
+
     // Check choirs in cascade order (important for illumination flow)
     for (const choirId of CASCADE_ORDER) {
       const choir = CHOIRS[choirId];
@@ -394,10 +406,20 @@ export function createChoirScheduler(
         continue;
       }
 
+      // Skip if already executing (prevents duplicate runs for slow choirs)
+      if (executing.has(choirId)) {
+        continue;
+      }
+
       // Check if due based on interval
       const state = runState.get(choirId);
       if (shouldRunChoir(choir, now, state?.lastRun)) {
-        await executeChoir(choir);
+        executing.add(choirId);
+        try {
+          await executeChoir(choir);
+        } finally {
+          executing.delete(choirId);
+        }
       }
     }
   }