@iamoberlin/chorus 2.2.1 → 2.4.0

package/src/purpose-research.ts

@@ -8,6 +8,7 @@
 import type { OpenClawPluginService, PluginLogger } from "openclaw/plugin-sdk";
 import { loadPurposes, updatePurpose, type Purpose } from "./purposes.js";
 import { recordExecution, type ChoirExecution } from "./metrics.js";
+import { sortByFairness, type FairnessScore } from "./behavioral-sink.js";
 import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
 import { join } from "path";
 import { homedir } from "os";
@@ -35,6 +36,9 @@ export const DEFAULT_PURPOSE_RESEARCH_CONFIG: PurposeResearchConfig = {
   checkIntervalMs: 60000,
 };
 
+// Guard: purposes currently running research (prevents duplicate triggers)
+const runningPurposes = new Set<string>();
+
 interface DailyRunTracker {
   date: string;
   count: number;
@@ -71,6 +75,36 @@ function countAlerts(output: string): number {
   return 1;
 }
 
+function extractTextFromApiResult(result: any): string {
+  if (!result) return "";
+  const payloadText = (result.payloads || [])
+    .map((p: any) => p?.text || "")
+    .filter((t: string) => Boolean(t))
+    .pop();
+  return payloadText || result.text || result.result?.text || result.response || result.content || "";
+}
+
+function extractTextFromCliStdout(stdout: string): string {
+  if (!stdout) return "";
+
+  // Find the last top-level JSON object to skip any log noise before payload.
+  for (let i = stdout.length - 1; i >= 0; i--) {
+    if (stdout[i] !== "{") continue;
+    try {
+      const parsed = JSON.parse(stdout.slice(i));
+      const payloadText = (parsed.result?.payloads || [])
+        .map((p: any) => p?.text || "")
+        .filter((t: string) => Boolean(t))
+        .pop();
+      return payloadText || parsed.result?.text || parsed.text || parsed.response || parsed.content || "";
+    } catch {
+      // Keep scanning backward for valid JSON.
+    }
+  }
+
+  return "";
+}
+
 const STATE_DIR = join(homedir(), ".chorus");
 const STATE_FILE = join(STATE_DIR, "research-state.json");
 
@@ -160,6 +194,8 @@ export function createPurposeResearchScheduler(
     if (purpose.progress >= 100) return false;
     if (purpose.research?.enabled === false) return false;
     if (!purpose.criteria?.length && !purpose.research?.domains?.length) return false;
+    // Guard: skip if already running (prevents duplicate triggers while agent is executing)
+    if (runningPurposes.has(purpose.id)) return false;
 
     const lastRun = purpose.research?.lastRun ?? 0;
     const frequency = calculateFrequency(purpose);
@@ -214,33 +250,43 @@ PURPOSE RESEARCH: ${purpose.name}
 You are researching for the following purpose:
 ${purpose.description || purpose.name}
 
-Search domains: ${domains}
+YOU HAVE FULL TOOL ACCESS. Use exec to run shell commands, web_search to search the web, web_fetch to read URLs. If criteria below include "run:" commands, EXECUTE THEM — do not just describe what you would do.
 
-Success criteria to inform research:
+Working directory: ${WORKSPACE_PATH}
+
+Criteria (follow these — if they say "run:", actually run the command):
 ${criteria}
 
 Tasks:
-1. Search for recent developments relevant to this purpose
-2. Assess impact on purpose progress or timeline
-3. Flag anything that challenges or validates current assumptions
-4. Note actionable insights
+1. EXECUTE any commands specified in criteria above
+2. Use web_search for at least 2 queries on recent developments
+3. Assess impact on purpose progress or timeline
+4. Surface actionable insights with measurable specifics (metrics, thresholds, dates)
 
 Alert threshold: ${alertThreshold}
 ${alertGuidance[alertThreshold]}
 
 Output format:
-- FINDINGS: Key discoveries (bullet points)
+- EXECUTED: Commands run and their results (summarized)
+- FINDINGS: Key discoveries (bullet points with numbers)
 - IMPACT: How this affects the purpose (progress/timeline/risk)
 - ALERTS: Anything requiring immediate attention (or "none")
 - NEXT: What to research next time
 
-Your output will be saved automatically. Focus on the research content.
+Your output will be saved automatically. Focus on actionable content, not analysis of what you could do.
 
-CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
+CRITICAL: If sending alerts to plain-text channels, use PLAIN TEXT ONLY (no markdown).
 `.trim();
   }
 
-  async function runResearch(purpose: Purpose): Promise<void> {
+  async function runResearch(purpose: Purpose): Promise<boolean> {
+    // Guard: prevent concurrent runs of the same purpose
+    if (runningPurposes.has(purpose.id)) {
+      log.debug(`[purpose-research] ⏭️ "${purpose.name}" already running, skipping`);
+      return false;
+    }
+    runningPurposes.add(purpose.id);
+
     const startTime = Date.now();
     log.info(`[purpose-research] 🔬 Running research for "${purpose.name}"`);
 
@@ -261,12 +307,16 @@ CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
       if (typeof api.runAgentTurn === "function") {
         try {
           result = await api.runAgentTurn({
-            sessionLabel: `chorus:purpose:${purpose.id}`,
+            sessionLabel: `chorus-purpose-${purpose.id}`,
             message: prompt,
             isolated: true,
             timeoutSeconds: config.researchTimeoutMs / 1000,
           });
-          output = result?.response || "";
+          output = extractTextFromApiResult(result);
+          if (!output.trim()) {
+            log.debug(`[purpose-research] API returned empty output, falling back to CLI for "${purpose.name}"`);
+            result = null;
+          }
         } catch (apiErr) {
           log.debug(`[purpose-research] API runAgentTurn failed, falling back to CLI: ${apiErr}`);
           result = null;
@@ -279,7 +329,7 @@ CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
         result = await new Promise<any>((resolve) => {
           const child = spawn("openclaw", [
             "agent",
-            "--session-id", `chorus:purpose:${purpose.id}`,
+            "--session-id", `chorus-purpose-${purpose.id}`,
             "--message", prompt,
             "--json",
           ], { stdio: ['pipe', 'pipe', 'pipe'] });
@@ -296,16 +346,15 @@ CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
         });
 
         if (result.status === 0 && result.stdout) {
-          try {
-            const json = JSON.parse(result.stdout);
-            output = json.result?.payloads?.[0]?.text || json.response || "";
-          } catch {
-            output = result.stdout;
-          }
+          output = extractTextFromCliStdout(result.stdout);
         } else if (result.stderr) {
           log.error(`[purpose-research] CLI error: ${result.stderr}`);
         }
       }
+
+      if (!output.trim()) {
+        throw new Error("No research output produced");
+      }
       execution.durationMs = Date.now() - startTime;
       execution.success = true;
       execution.outputLength = output.length;
@@ -335,24 +384,51 @@ CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
         }
       }
 
+      // Re-read purpose from disk to avoid overwriting fields changed during the run
+      // (e.g., frequency modified externally while agent was executing)
+      const freshPurposes = await loadPurposes();
+      const freshPurpose = freshPurposes.find((p) => p.id === purpose.id);
+      const freshResearch = freshPurpose?.research ?? purpose.research;
+
       await updatePurpose(purpose.id, {
         research: {
-          ...purpose.research,
-          enabled: purpose.research?.enabled ?? true,
+          ...freshResearch,
+          enabled: freshResearch?.enabled ?? true,
           lastRun: Date.now(),
-          runCount: (purpose.research?.runCount ?? 0) + 1,
+          runCount: (freshResearch?.runCount ?? 0) + 1,
         },
       });
+      dailyRuns.count++;
+      persistState();
+      return true;
     } catch (err) {
       execution.durationMs = Date.now() - startTime;
       execution.success = false;
       execution.error = String(err);
       log.error(`[purpose-research] ✗ "${purpose.name}" failed: ${err}`);
-    }
 
-    recordExecution(execution);
-    dailyRuns.count++;
-    persistState();
+      // Update lastRun even on failure to prevent retry storms
+      // (without this, failed runs leave lastRun stale and the scheduler retries immediately)
+      try {
+        const failPurposes = await loadPurposes();
+        const failPurpose = failPurposes.find((p) => p.id === purpose.id);
+        if (failPurpose) {
+          await updatePurpose(purpose.id, {
+            research: {
+              ...(failPurpose.research ?? purpose.research),
+              lastRun: Date.now(),
+            },
+          });
+        }
+      } catch (updateErr) {
+        log.error(`[purpose-research] Failed to update lastRun after error: ${updateErr}`);
+      }
+
+      return false;
+    } finally {
+      runningPurposes.delete(purpose.id);
+      recordExecution(execution);
+    }
   }
 
   async function checkAndRun(): Promise<void> {
@@ -377,21 +453,37 @@ CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
 
     if (duePurposes.length === 0) return;
 
-    duePurposes.sort((a, b) => {
-      const aDeadline = a.deadline
-        ? typeof a.deadline === "string"
-          ? Date.parse(a.deadline)
-          : a.deadline
-        : Infinity;
-      const bDeadline = b.deadline
-        ? typeof b.deadline === "string"
-          ? Date.parse(b.deadline)
-          : b.deadline
-        : Infinity;
-      return aDeadline - bDeadline;
-    });
-
-    const purpose = duePurposes[0];
+    // ── Universe 25 Fairness Sort ─────────────────────────────
+    // Replaces naive deadline sort that caused purpose starvation
+    // (calibration lesson #23). Overdue purposes get exponentially
+    // increasing priority — no purpose can be crowded out.
+    const fairnessSorted = sortByFairness(
+      duePurposes.map(p => ({
+        id: p.id,
+        lastRun: p.research?.lastRun,
+        frequency: calculateFrequency(p),
+        deadline: p.deadline
+          ? typeof p.deadline === "string"
+            ? Date.parse(p.deadline)
+            : p.deadline
+          : undefined,
+      }))
+    );
+
+    const selectedId = fairnessSorted[0]?.id;
+    const purpose = duePurposes.find(p => p.id === selectedId);
+    const fairness = fairnessSorted[0]?.fairness;
+
+    if (!purpose) return;
+
+    if (fairness?.starving) {
+      log.warn(
+        `[purpose-research] 🏚️ STARVATION RECOVERY: "${purpose.name}" — ` +
+        `${fairness.starvationRatio.toFixed(1)}x overdue (expected every ` +
+        `${(fairness.expectedInterval / 3600000).toFixed(1)}h, actual ` +
+        `${(fairness.actualInterval / 3600000).toFixed(1)}h). Boost: ${fairness.priorityBoost.toFixed(1)}x`
+      )
+    }
 
     if (dailyRuns.count < config.dailyRunCap) {
       await runResearch(purpose);
@@ -442,7 +534,10 @@ CRITICAL: If sending alerts via iMessage, use PLAIN TEXT ONLY (no markdown).
       const purposes = await loadPurposes();
       const purpose = purposes.find((p) => p.id === purposeId);
       if (!purpose) throw new Error(`Purpose "${purposeId}" not found`);
-      await runResearch(purpose);
+      const success = await runResearch(purpose);
+      if (!success) {
+        throw new Error(`Research run failed for "${purposeId}"`);
+      }
     },
 
     getStatus: () => {

package/src/scheduler.ts

@@ -8,11 +8,14 @@
 import type { OpenClawPluginService, PluginLogger } from "openclaw/plugin-sdk";
 import type { ChorusConfig } from "./config.js";
 import { CHOIRS, shouldRunChoir, CASCADE_ORDER, type Choir } from "./choirs.js";
+import { deliverChoirOutput } from "./delivery.js";
 import { recordExecution, type ChoirExecution } from "./metrics.js";
+import { recordCost, estimateCost } from "./economics.js";
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
 import { join } from "path";
 import { homedir } from "os";
 import { spawn } from "child_process";
+import { recordAndAssess } from "./behavioral-sink.js";
 
 // Type for the plugin API's runAgentTurn method
 interface AgentTurnResult {
@@ -22,8 +25,8 @@ interface AgentTurnResult {
 }
 
 // ── Delivery ─────────────────────────────────────────────────
-// Choir agents handle their own delivery via OpenClaw messaging tools.
-// The scheduler's job is execution and scheduling — not routing messages.
+// Delivery is handled via shared channel adapter to keep behavior consistent
+// across scheduler and manual CLI execution paths.
 
 interface ChoirContext {
   choirId: string;
@@ -41,6 +44,29 @@ interface ChoirRunState {
 const CHORUS_DIR = join(homedir(), ".chorus");
 const RUN_STATE_PATH = join(CHORUS_DIR, "run-state.json");
 
+// ── 429 Circuit Breaker ──────────────────────────────────────
+// Exponential backoff on rate limit errors. Skips choir execution
+// when backing off, resets on successful runs.
+interface CircuitBreakerState {
+  backoffUntil: number; // timestamp ms — skip all choirs until this time
+  consecutiveFailures: number;
+}
+
+const MAX_BACKOFF_MS = 16 * 60 * 1000; // 16 minutes cap
+const BASE_BACKOFF_MS = 60 * 1000; // 1 minute base
+
+function isRateLimitError(err: unknown): boolean {
+  const msg = String(err).toLowerCase();
+  return msg.includes("429") || msg.includes("rate limit") || msg.includes("rate_limit") || msg.includes("too many requests") || msg.includes("overloaded") || msg.includes("unavailable");
+}
+
+function computeBackoffMs(failures: number): number {
+  const ms = BASE_BACKOFF_MS * Math.pow(2, Math.min(failures - 1, 4));
+  // Add jitter: ±25%
+  const jitter = ms * 0.25 * (Math.random() * 2 - 1);
+  return Math.min(ms + jitter, MAX_BACKOFF_MS);
+}
+
 // Load persisted run state from disk
 function loadRunState(log: PluginLogger): Map<string, ChoirRunState> {
   const state = new Map<string, ChoirRunState>();
@@ -106,12 +132,15 @@ export function createChoirScheduler(
   // Load persisted state instead of starting fresh
   const runState = loadRunState(log);
 
+  // Circuit breaker state
+  const circuitBreaker: CircuitBreakerState = { backoffUntil: 0, consecutiveFailures: 0 };
+
   // CLI fallback for executing choirs when plugin API is unavailable
   async function executeChoirViaCli(choir: Choir, prompt: string): Promise<string> {
     const result = await new Promise<{ status: number | null; stdout: string; stderr: string }>((resolve) => {
       const child = spawn('openclaw', [
         'agent',
-        '--session-id', `chorus:${choir.id}`,
+        '--session-id', `chorus-${choir.id}`,
         '--message', prompt,
         '--json',
       ], { stdio: ['pipe', 'pipe', 'pipe'] });
@@ -190,7 +219,7 @@ export function createChoirScheduler(
       if (typeof api.runAgentTurn === 'function') {
         try {
           const result: AgentTurnResult = await api.runAgentTurn({
-            sessionLabel: `chorus:${choir.id}`,
+            sessionLabel: `chorus-${choir.id}`,
             message: prompt,
             isolated: true,
             timeoutSeconds: 300,
@@ -208,6 +237,17 @@ export function createChoirScheduler(
             output = result.text;
           }
         } catch (apiErr) {
+          if (isRateLimitError(apiErr)) {
+            circuitBreaker.consecutiveFailures++;
+            const backoffMs = computeBackoffMs(circuitBreaker.consecutiveFailures);
+            circuitBreaker.backoffUntil = Date.now() + backoffMs;
+            log.warn(`[chorus] ⚡ 429 RATE LIMIT on ${choir.name} — backing off ${(backoffMs/1000).toFixed(0)}s (failure #${circuitBreaker.consecutiveFailures})`);
+            execution.durationMs = Date.now() - startTime;
+            execution.success = false;
+            execution.error = "429_rate_limit_backoff";
+            recordExecution(execution);
+            return; // Skip this choir and remaining choirs will be skipped by checkAndRunChoirs
+          }
           log.warn(`[chorus] API runAgentTurn failed for ${choir.name}, falling back to CLI: ${apiErr}`);
           output = await executeChoirViaCli(choir, prompt);
         }
@@ -221,17 +261,50 @@ export function createChoirScheduler(
       execution.outputLength = output.length;
       execution.tokensUsed = estimateTokens(output);
 
+      // Record economics cost (only for runs that produced output)
+      if (execution.success) {
+        const inputTokensEst = Math.ceil(prompt.length / 4);
+        const outputTokensEst = execution.tokensUsed || Math.ceil(output.length / 4);
+        recordCost({
+          choirId: choir.id,
+          timestamp: new Date().toISOString(),
+          inputTokens: inputTokensEst,
+          outputTokens: outputTokensEst,
+          model: "default",
+          inferenceMs: execution.durationMs,
+          costUsd: estimateCost(inputTokensEst, outputTokensEst),
+        });
+      }
+
       // Parse output for metrics (findings, alerts, improvements)
       execution.findings = countFindings(output);
       execution.alerts = countAlerts(output);
       execution.improvements = extractImprovements(output, choir.id);
 
-      // Store context for downstream choirs
-      contextStore.set(choir.id, {
-        choirId: choir.id,
-        output: output.slice(0, 2000), // Truncate for context passing
-        timestamp: new Date(),
-      });
+      // ── Behavioral Sink Prevention ──────────────────────────
+      // Assess output quality and gate illumination to prevent
+      // degraded context from cascading downstream (Universe 25).
+      const sinkAssessment = recordAndAssess(choir.id, output);
+
+      if (sinkAssessment.quality.score < 20 && choir.id !== "angels") {
+        log.warn(`[chorus] 🪞 BEAUTIFUL ONE detected: ${choir.name} scored ${sinkAssessment.quality.score}/100 — flags: ${sinkAssessment.quality.flags.join(", ")}`);
+      }
+
+      // Store context for downstream choirs — only if quality passes sink gate
+      if (sinkAssessment.illumination.usable) {
+        const qualityTag = sinkAssessment.illumination.degraded
+          ? `\n[⚠️ CONTEXT QUALITY DEGRADED — score ${sinkAssessment.quality.score}/100]\n`
+          : "";
+        contextStore.set(choir.id, {
+          choirId: choir.id,
+          output: qualityTag + output.slice(0, 2000),
+          timestamp: new Date(),
+        });
+      } else {
+        log.warn(`[chorus] 🚫 SINK GATE: ${choir.name} output blocked from illumination chain (score ${sinkAssessment.quality.score}/100)`);
+        // Don't update context — downstream choirs keep stale but good context
+        // rather than receiving fresh but degraded context
+      }
 
       // Update run state
       runState.set(choir.id, {
@@ -243,69 +316,17 @@ export function createChoirScheduler(
       // Persist state to disk after each run
       saveRunState(runState, log);
 
-      log.info(`[chorus] ${choir.emoji} ${choir.name} completed (${(execution.durationMs/1000).toFixed(1)}s)`);
+      // Reset circuit breaker on success
+      if (circuitBreaker.consecutiveFailures > 0) {
+        log.info(`[chorus] ✅ Circuit breaker reset after ${circuitBreaker.consecutiveFailures} failures`);
+        circuitBreaker.consecutiveFailures = 0;
+        circuitBreaker.backoffUntil = 0;
+      }
 
-      // Deliver output to user via OpenClaw messaging if choir is marked for delivery
-      // Reads target from OpenClaw config (channels.*.allowFrom) — no hardcoded PII
-      if (choir.delivers && output && output !== "(no response)" && output !== "HEARTBEAT_OK" && output !== "NO_REPLY") {
-        const channels = api.config?.channels as Record<string, any> | undefined;
-        let target: string | undefined;
-        let channel: string | undefined;
-
-        if (channels) {
-          for (const [ch, cfg] of Object.entries(channels)) {
-            if (cfg?.enabled && cfg?.allowFrom?.[0]) {
-              target = cfg.allowFrom[0];
-              channel = ch;
-              break;
-            }
-          }
-        }
+      log.info(`[chorus] ${choir.emoji} ${choir.name} completed (${(execution.durationMs/1000).toFixed(1)}s)`);
 
-        if (target) {
-          // Strip markdown for channels that don't support it
-          let deliveryText = output.slice(0, 4000);
-          if (channel === 'imessage') {
-            deliveryText = deliveryText
-              .replace(/\*\*(.+?)\*\*/g, '$1')    // bold
-              .replace(/\*(.+?)\*/g, '$1')          // italic
-              .replace(/__(.+?)__/g, '$1')          // bold alt
-              .replace(/_(.+?)_/g, '$1')            // italic alt
-              .replace(/`(.+?)`/g, '$1')            // inline code
-              .replace(/```[\s\S]*?```/g, '')       // code blocks
-              .replace(/^#{1,6}\s+/gm, '')          // headers
-              .replace(/^\s*[-*+]\s+/gm, '• ')     // bullet lists
-              .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); // links
-          }
-
-          try {
-            const args = [
-              'message', 'send',
-              '--target', target,
-              '--message', deliveryText,
-            ];
-            if (channel) args.push('--channel', channel);
-
-            const deliveryProc = spawn('openclaw', args, { stdio: ['pipe', 'pipe', 'pipe'] });
-
-            deliveryProc.on('close', (code) => {
-              if (code === 0) {
-                log.info(`[chorus] 📨 ${choir.name} output delivered via ${channel || 'default'}`);
-              } else {
-                log.warn(`[chorus] ⚠ ${choir.name} delivery failed (exit ${code})`);
-              }
-            });
-
-            deliveryProc.on('error', (err) => {
-              log.warn(`[chorus] ⚠ ${choir.name} delivery error: ${err.message}`);
-            });
-          } catch (deliveryErr) {
-            log.warn(`[chorus] ⚠ ${choir.name} delivery error: ${deliveryErr}`);
-          }
-        } else {
-          log.warn(`[chorus] ⚠ No delivery target found in OpenClaw config for ${choir.name}`);
-        }
-      }
+      // Deliver output via shared adapter for consistent routing/format behavior
+      await deliverChoirOutput(api, choir, output, log);
 
       // Log illumination flow
       if (choir.passesTo.length > 0) {
@@ -380,10 +401,20 @@ export function createChoirScheduler(
     return improvements.slice(0, 5); // Cap at 5
   }
 
+  // Track choirs currently executing to prevent duplicate runs
+  const executing = new Set<string>();
+
   // Check and run due choirs
   async function checkAndRunChoirs(): Promise<void> {
     const now = new Date();
 
+    // Circuit breaker: skip entire cycle if backing off from rate limit
+    if (Date.now() < circuitBreaker.backoffUntil) {
+      const remainingSec = ((circuitBreaker.backoffUntil - Date.now()) / 1000).toFixed(0);
+      log.debug(`[chorus] ⚡ Circuit breaker active — ${remainingSec}s remaining`);
+      return;
+    }
+
     // Check choirs in cascade order (important for illumination flow)
     for (const choirId of CASCADE_ORDER) {
       const choir = CHOIRS[choirId];
@@ -394,10 +425,20 @@ export function createChoirScheduler(
         continue;
       }
 
+      // Skip if already executing (prevents duplicate runs for slow choirs)
+      if (executing.has(choirId)) {
+        continue;
+      }
+
       // Check if due based on interval
       const state = runState.get(choirId);
       if (shouldRunChoir(choir, now, state?.lastRun)) {
-        await executeChoir(choir);
+        executing.add(choirId);
+        try {
+          await executeChoir(choir);
+        } finally {
+          executing.delete(choirId);
+        }
       }
     }
   }