@adaptic/maestro 1.0.0

package/scripts/daemon/dispatcher.mjs

@@ -0,0 +1,385 @@
+#!/usr/bin/env node
+// Dispatcher — manages concurrent claude --print sessions
+// Spawns child processes, manages queue, enforces concurrency cap
+
+import { spawn } from "child_process";
+import { appendFileSync, mkdirSync, writeFileSync, readFileSync, renameSync } from "fs";
+import { join } from "path";
+import { releaseLock, releaseThreadLock, releaseRequestClaim } from "./session-lock.mjs";
+
+const SOPHIE_AI_DIR = join(new URL(".", import.meta.url).pathname, "../..");
+const CLAUDE_BIN = process.env.CLAUDE_BIN || "/Users/sophie/.local/bin/claude";
+const MAX_CONCURRENT = parseInt(process.env.DAEMON_MAX_CONCURRENT || "10", 10);
+const RESERVED_INBOX_SLOTS = 3;    // Always keep 3 slots free for real-time inbox items
+
+// Timeouts — tiered by model and source
+const SONNET_INBOX_TIMEOUT = 10 * 60 * 1000;          // 10 min (inbox — user expects a reply)
+const SONNET_BACKLOG_TIMEOUT = 30 * 60 * 1000;        // 30 min (backlog tasks need more time)
+const OPUS_INBOX_TIMEOUT = 45 * 60 * 1000;            // 45 min (complex inbox — CEO requests, research)
+const OPUS_BACKLOG_TIMEOUT = 12 * 60 * 60 * 1000;     // 12 hours (ultra-complex backlog — deep work)
+
+// Legacy aliases for compatibility
+const SONNET_TIMEOUT = SONNET_INBOX_TIMEOUT;
+const OPUS_TIMEOUT = OPUS_INBOX_TIMEOUT;
+
+const activeSessions = new Map();  // sessionId -> { process, item, startTime, model, source }
+const priorityQueue = [];          // critical/high items
+const normalQueue = [];            // normal items
+let sessionCounter = 0;
+
+// Backlog dedup: track which items have active sessions to prevent retry storms
+const activeBacklogKeys = new Set();  // backlog item key -> true (while session is running)
+const backlogRetryCount = new Map();  // backlog item key -> number of times dispatched
+const MAX_BACKLOG_RETRIES = 6;        // Max retries before skipping (was 3 — too aggressive)
+
+function logDir() {
+  const dir = join(SOPHIE_AI_DIR, "logs", "daemon");
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function sessionLogDir() {
+  const dir = join(SOPHIE_AI_DIR, "logs", "daemon", "sessions");
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function today() {
+  return new Date().toISOString().split("T")[0];
+}
+
+function logSession(entry) {
+  const path = join(logDir(), `${today()}-sessions.jsonl`);
+  appendFileSync(path, JSON.stringify({ timestamp: new Date().toISOString(), ...entry }) + "\n");
+}
+
+const ACTIVE_PATH = join(SOPHIE_AI_DIR, "state", "sessions", "active.json");
+
+/**
+ * Reset active.json on daemon startup.
+ * Previous daemon instances may have left stale entries (killed sessions
+ * whose close handlers never fired). Since we're starting fresh, no
+ * sessions from prior instances are still running.
+ */
+export function resetActiveSessions() {
+  try {
+    let staleCount = 0;
+    try {
+      const old = JSON.parse(readFileSync(ACTIVE_PATH, "utf-8"));
+      staleCount = Object.keys(old).length;
+    } catch {}
+    writeFileSync(ACTIVE_PATH, "{}");
+    if (staleCount > 0) {
+      console.log(`[dispatcher] Cleared ${staleCount} stale entries from active.json`);
+    }
+  } catch (err) {
+    console.warn(`[dispatcher] Failed to reset active.json: ${err.message}`);
+  }
+}
+
+function writeActiveSession(sessionId, entry) {
+  try {
+    let active = {};
+    try { active = JSON.parse(readFileSync(ACTIVE_PATH, "utf-8")); } catch {}
+    active[sessionId] = {
+      sender: entry.item?.sender || null,
+      channel: entry.item?.channel || entry.item?.channel_id || null,
+      summary: entry.classResult?.summary || "unknown",
+      model: entry.classResult?.model || "sonnet",
+      startTime: Date.now(),
+      source: entry.source,
+    };
+    const tmpPath = ACTIVE_PATH + ".tmp";
+    writeFileSync(tmpPath, JSON.stringify(active, null, 2));
+    renameSync(tmpPath, ACTIVE_PATH);
+  } catch (err) {
+    console.warn(`[dispatcher] Failed to write active.json: ${err.message}`);
+  }
+}
+
+function removeActiveSession(sessionId) {
+  try {
+    let active = {};
+    try { active = JSON.parse(readFileSync(ACTIVE_PATH, "utf-8")); } catch {}
+    delete active[sessionId];
+    const tmpPath = ACTIVE_PATH + ".tmp";
+    writeFileSync(tmpPath, JSON.stringify(active, null, 2));
+    renameSync(tmpPath, ACTIVE_PATH);
+  } catch (err) {
+    console.warn(`[dispatcher] Failed to update active.json: ${err.message}`);
+  }
+}
+
+/**
+ * Count active sessions by source type.
+ */
+function countBySource(source) {
+  let count = 0;
+  for (const [, s] of activeSessions) {
+    if (s.source === source) count++;
+  }
+  return count;
+}
+
+/**
+ * Evict the lowest-priority, longest-running backlog session to make room
+ * for a high-priority inbox item. Returns true if a session was evicted.
+ */
+function evictForPriority() {
+  const priorityRank = { low: 0, normal: 1, high: 2, critical: 3 };
+  let worst = null;
+  let worstId = null;
+
+  for (const [id, s] of activeSessions) {
+    if (s.source !== "backlog") continue;
+    const rank = priorityRank[s.classResult.priority] || 1;
+    if (!worst) {
+      worst = s; worstId = id;
+      continue;
+    }
+    const worstRank = priorityRank[worst.classResult.priority] || 1;
+    // Prefer to evict lower priority; break ties by oldest session
+    if (rank < worstRank || (rank === worstRank && s.startTime < worst.startTime)) {
+      worst = s; worstId = id;
+    }
+  }
+
+  if (worst) {
+    console.log(`[dispatcher] PREEMPT: Evicting backlog session ${worstId} (${worst.classResult.summary}) for priority inbox item`);
+    logSession({ event: "evicted", sessionId: worstId, reason: "priority_preemption", summary: worst.classResult.summary });
+    worst.process.kill("SIGTERM");
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Generate a stable key for a backlog item to track in-flight status.
+ */
+function backlogKey(item) {
+  return item.id || item.title || item.summary || JSON.stringify(item).substring(0, 100);
+}
+
+/**
+ * Check if a backlog item already has an active session or has exceeded retry limit.
+ * Returns { allowed: boolean, reason?: string }
+ */
+export function canDispatchBacklog(item) {
+  const key = backlogKey(item);
+  if (activeBacklogKeys.has(key)) {
+    return { allowed: false, reason: "session_already_active" };
+  }
+  const retries = backlogRetryCount.get(key) || 0;
+  if (retries >= MAX_BACKLOG_RETRIES) {
+    return { allowed: false, reason: "max_retries_exceeded", retries };
+  }
+  return { allowed: true };
+}
+
+/**
+ * Dispatch an item for processing.
+ *
+ * @param {string} prompt
+ * @param {object} item
+ * @param {object} classResult
+ * @param {string} source - "inbox" or "backlog"
+ */
+export function dispatch(prompt, item, classResult, source = "inbox") {
+  const entry = { prompt, item, classResult, source };
+
+  // Backlog items: check dedup and retry limits
+  if (source === "backlog") {
+    const check = canDispatchBacklog(item);
+    if (!check.allowed) {
+      logSession({ event: "skipped", reason: check.reason, summary: classResult.summary, retries: check.retries });
+      return;
+    }
+  }
+
+  // Backlog items respect the reserved slot cap
+  if (source === "backlog") {
+    const backlogCount = countBySource("backlog");
+    const backlogCap = MAX_CONCURRENT - RESERVED_INBOX_SLOTS;
+    if (backlogCount >= backlogCap) {
+      // No room for more backlog — silently skip (don't queue indefinitely)
+      return;
+    }
+  }
+
+  if (activeSessions.size < MAX_CONCURRENT) {
+    spawnSession(entry);
+  } else if (source === "inbox" && (classResult.priority === "critical" || classResult.priority === "high")) {
+    // Priority inbox item but no slots — try to evict a backlog session
+    if (evictForPriority()) {
+      // Session evicted; it will call drainQueue on close. Queue this item at front.
+      priorityQueue.unshift(entry);
+      logSession({ event: "queued_after_eviction", priority: classResult.priority, summary: classResult.summary });
+    } else {
+      // No backlog sessions to evict — queue normally
+      priorityQueue.push(entry);
+      logSession({ event: "queued", priority: classResult.priority, summary: classResult.summary, queue_position: priorityQueue.length });
+    }
+  } else if (classResult.priority === "critical" || classResult.priority === "high") {
+    priorityQueue.push(entry);
+    logSession({ event: "queued", priority: classResult.priority, summary: classResult.summary, queue_position: priorityQueue.length });
+  } else {
+    normalQueue.push(entry);
+    logSession({ event: "queued", priority: classResult.priority, summary: classResult.summary, queue_position: normalQueue.length });
+  }
+}
+
+function spawnSession(entry) {
+  const { prompt, item, classResult, source } = entry;
+  const sessionId = `s-${Date.now()}-${++sessionCounter}`;
+  const model = classResult.model || "sonnet";
+  const timeout = model === "opus"
+    ? (source === "backlog" ? OPUS_BACKLOG_TIMEOUT : OPUS_INBOX_TIMEOUT)
+    : (source === "backlog" ? SONNET_BACKLOG_TIMEOUT : SONNET_INBOX_TIMEOUT);
+
+  // Track backlog items to prevent retry storms
+  if (source === "backlog") {
+    const key = backlogKey(item);
+    activeBacklogKeys.add(key);
+    backlogRetryCount.set(key, (backlogRetryCount.get(key) || 0) + 1);
+  }
+
+  const args = [
+    "--print",
+    "--dangerously-skip-permissions",
+    "--model", model,
+    prompt,
+  ];
+
+  const proc = spawn(CLAUDE_BIN, args, {
+    cwd: SOPHIE_AI_DIR,
+    env: { ...process.env },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  let stdout = "";
+  let stderr = "";
+  const logFile = join(sessionLogDir(), `${sessionId}.log`);
+
+  proc.stdout.on("data", (chunk) => { stdout += chunk.toString(); });
+  proc.stderr.on("data", (chunk) => { stderr += chunk.toString(); });
+
+  const timer = setTimeout(() => {
+    console.error(`[dispatcher] Session ${sessionId} timed out (${timeout / 1000}s), killing`);
+    proc.kill("SIGTERM");
+    setTimeout(() => { if (!proc.killed) proc.kill("SIGKILL"); }, 5000);
+  }, timeout);
+
+  const startTime = Date.now();
+  activeSessions.set(sessionId, { process: proc, item, classResult, startTime, model, source });
+  writeActiveSession(sessionId, entry);
+
+  logSession({
+    event: "spawned",
+    sessionId,
+    model,
+    source,
+    priority: classResult.priority,
+    summary: classResult.summary,
+    active_count: activeSessions.size,
+  });
+
+  console.log(`[dispatcher] Spawned ${sessionId} (${model}) — ${classResult.summary} [${activeSessions.size}/${MAX_CONCURRENT}]`);
+
+  proc.on("close", (code) => {
+    clearTimeout(timer);
+    activeSessions.delete(sessionId);
+    removeActiveSession(sessionId);
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+
+    // Release item lock — MUST use same key order as acquireLock in daemon
+    // (raw_ref first, then id). Previously this used id || raw_ref, creating
+    // a different sanitised filename so the release silently missed the lock.
+    const itemId = item.raw_ref || item.id || item.title;
+    if (itemId) releaseLock(itemId);
+
+    // Release thread lock so new messages in this thread can be processed
+    if (item.thread_id) {
+      const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
+      if (channel) releaseThreadLock(channel, item.thread_id);
+    }
+
+    // Release request claim so the same type of request can be processed again
+    if (classResult && classResult.summary) {
+      releaseRequestClaim({
+        recipient: item.channel_id || item.channel || item.sender || "unknown",
+        subject: classResult.summary || item.subject || "",
+        action_type: classResult.action || "respond",
+      });
+    }
+
+    // Release backlog tracking
+    if (source === "backlog") {
+      const key = backlogKey(item);
+      activeBacklogKeys.delete(key);
+      const retries = backlogRetryCount.get(key) || 0;
+
+      // If session timed out (143=SIGTERM) and hit retry limit, log it
+      if (code === 143 && retries >= MAX_BACKLOG_RETRIES) {
+        console.warn(`[dispatcher] Backlog item "${classResult.summary}" exhausted ${MAX_BACKLOG_RETRIES} retries — will not retry`);
+        logSession({ event: "retries_exhausted", summary: classResult.summary, retries });
+      }
+    }
+
+    // Write session log
+    writeFileSync(logFile, `# Session ${sessionId}\n# Model: ${model}\n# Duration: ${duration}s\n# Exit: ${code}\n\n## STDOUT\n${stdout}\n\n## STDERR\n${stderr}\n`);
+
+    logSession({
+      event: "completed",
+      sessionId,
+      model,
+      source,
+      exit_code: code,
+      duration_s: parseFloat(duration),
+      summary: classResult.summary,
+      active_count: activeSessions.size,
+    });
+
+    console.log(`[dispatcher] Session ${sessionId} done (${duration}s, exit ${code}) [${activeSessions.size}/${MAX_CONCURRENT}]`);
+
+    // Dispatch next queued item
+    drainQueue();
+  });
+
+  proc.on("error", (err) => {
+    clearTimeout(timer);
+    activeSessions.delete(sessionId);
+    console.error(`[dispatcher] Session ${sessionId} error: ${err.message}`);
+    logSession({ event: "error", sessionId, error: err.message });
+    drainQueue();
+  });
+}
+
+function drainQueue() {
+  while (activeSessions.size < MAX_CONCURRENT) {
+    const next = priorityQueue.shift() || normalQueue.shift();
+    if (!next) break;
+    spawnSession(next);
+  }
+}
+
+/** Get current status for health checks */
+export function getStatus() {
+  return {
+    active_sessions: activeSessions.size,
+    max_concurrent: MAX_CONCURRENT,
+    priority_queue_length: priorityQueue.length,
+    normal_queue_length: normalQueue.length,
+    sessions: Array.from(activeSessions.entries()).map(([id, s]) => ({
+      id,
+      model: s.model,
+      priority: s.classResult.priority,
+      summary: s.classResult.summary,
+      duration_s: ((Date.now() - s.startTime) / 1000).toFixed(0),
+    })),
+  };
+}
+
+/** Get available slots count */
+export function availableSlots() {
+  return MAX_CONCURRENT - activeSessions.size;
+}

package/scripts/daemon/health.mjs

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+// Health — periodic health metrics and status logging
+
+import { writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { getStatus } from "./dispatcher.mjs";
+
+const SOPHIE_AI_DIR = join(new URL(".", import.meta.url).pathname, "../..");
+
+let stats = {
+  daemon_start: new Date().toISOString(),
+  polls_total: 0,
+  items_detected: 0,
+  items_classified: 0,
+  sessions_spawned: 0,
+  sessions_completed: 0,
+  sessions_failed: 0,
+  classifications_failed: 0,
+  last_poll: null,
+  last_item: null,
+};
+
+export function recordPoll(itemCount) {
+  stats.polls_total++;
+  stats.items_detected += itemCount;
+  stats.last_poll = new Date().toISOString();
+  if (itemCount > 0) stats.last_item = stats.last_poll;
+}
+
+export function recordClassification(success) {
+  if (success) stats.items_classified++;
+  else stats.classifications_failed++;
+}
+
+export function recordSession(completed, success) {
+  if (!completed) {
+    stats.sessions_spawned++;
+  } else if (success) {
+    stats.sessions_completed++;
+  } else {
+    stats.sessions_failed++;
+  }
+}
+
+export function writeHealthDashboard() {
+  const dispatcher = getStatus();
+  const dashboard = {
+    updated: new Date().toISOString(),
+    uptime_s: Math.floor((Date.now() - new Date(stats.daemon_start).getTime()) / 1000),
+    ...stats,
+    dispatcher,
+  };
+
+  const dir = join(SOPHIE_AI_DIR, "state", "dashboards");
+  mkdirSync(dir, { recursive: true });
+  const path = join(dir, "daemon-health.yaml");
+
+  const yaml = Object.entries(dashboard)
+    .map(([k, v]) => {
+      if (typeof v === "object" && v !== null) {
+        return `${k}:\n${JSON.stringify(v, null, 2).split("\n").map(l => `  ${l}`).join("\n")}`;
+      }
+      return `${k}: ${v}`;
+    })
+    .join("\n");
+
+  writeFileSync(path, yaml + "\n");
+}
+
+export function getStats() {
+  return { ...stats };
+}