@adaptic/maestro 1.0.0

package/scripts/daemon/sophie-daemon.mjs

@@ -0,0 +1,521 @@
+#!/usr/bin/env node
+// =============================================================================
+// Sophie Daemon — Reactive event-driven message processor
+// =============================================================================
+//
+// Persistent Node.js process that:
+//   1. Polls Slack/Gmail/Calendar every 30s
+//   2. Classifies each item via Haiku API (~0.5-1s)
+//   3. Spawns dedicated claude --print sessions per item (up to 10 parallel)
+//   4. Sweeps backlog queues every 2 min for stale items
+//
+// Replaces: poller + inbox-processor + backlog-executor (3-stage pipeline)
+// Target: CEO DM → response in under 2 minutes
+//
+// Run: node scripts/daemon/sophie-daemon.mjs
+// Install: launchd plist with KeepAlive: true
+// =============================================================================
+
+import { config } from "dotenv";
+import { resolve, join } from "path";
+import { readdirSync, readFileSync, renameSync, mkdirSync, appendFileSync } from "fs";
+
+// Load .env before anything else
+const SOPHIE_AI_DIR = resolve(new URL(".", import.meta.url).pathname, "../..");
+config({ path: join(SOPHIE_AI_DIR, ".env") });
+
+import { pollSlack } from "../poller/slack-poller.mjs";
+import { pollGmail } from "../poller/gmail-poller.mjs";
+import { pollCalendar } from "../poller/calendar-poller.mjs";
+import { pollMehranGmail } from "../poller/mehran-gmail-poller.mjs";
+import { isPriorityItem } from "../poller/utils.mjs";
+import { classifyItem, isDirectedAtSophie } from "./classifier.mjs";
+import { dispatch, getStatus, availableSlots, canDispatchBacklog, resetActiveSessions } from "./dispatcher.mjs";
+import { buildPrompt } from "./prompt-builder.mjs";
+import { sendQuickResponse, sendHoldingMessage, isQuickReply } from "./responder.mjs";
+import { recordPoll, recordClassification, recordSession, writeHealthDashboard } from "./health.mjs";
+import { acquireLock, updateLock, scanStaleLocks, acquireThreadLock, claimRequest } from "./session-lock.mjs";
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+const POLL_INTERVAL = parseInt(process.env.DAEMON_POLL_INTERVAL || "60000", 10); // 60s (up from 30s to avoid Slack rate limits)
+const BACKLOG_INTERVAL = parseInt(process.env.DAEMON_BACKLOG_INTERVAL || "120000", 10); // 2 min
+const HEALTH_INTERVAL = 60000; // 1 min
+// Note: dedup is now handled by file-based locks in session-lock.mjs
+
+// ---------------------------------------------------------------------------
+// Logging
+// ---------------------------------------------------------------------------
+
+function logDir() {
+  const dir = join(SOPHIE_AI_DIR, "logs", "daemon");
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function today() {
+  return new Date().toISOString().split("T")[0];
+}
+
+function logEvent(type, entry) {
+  const path = join(logDir(), `${today()}-${type}.jsonl`);
+  appendFileSync(path, JSON.stringify({ timestamp: new Date().toISOString(), ...entry }) + "\n");
+}
+
+// ---------------------------------------------------------------------------
+// Deduplication — now handled by file-based locks in session-lock.mjs
+// ---------------------------------------------------------------------------
+
+// ---------------------------------------------------------------------------
+// Poll Loop
+// ---------------------------------------------------------------------------
+
+async function poll() {
+  const services = [
+    { name: "slack", fn: pollSlack },
+    { name: "gmail", fn: pollGmail },
+    { name: "mehran-gmail", fn: pollMehranGmail },
+    { name: "calendar", fn: pollCalendar },
+  ];
+
+  let totalItems = 0;
+
+  for (const svc of services) {
+    try {
+      const result = await svc.fn();
+      // Dedup by raw_ref first: the same Slack message arrives via
+      // multiple paths (channel scan, thread scan, events-server JSON)
+      // with different item IDs. Using raw_ref as the lock key ensures
+      // a single message is only processed once regardless of path.
+      const seenRefs = new Set();
+      const newItems = result.items.filter((item) => {
+        // Skip Sophie's own messages — defensive check in case intake filters miss them
+        const sender = (item.sender || "").toLowerCase();
+        if (sender === "sophie-nguyen" || sender === "sophie") return false;
+
+        const lockKey = item.raw_ref || item.id || `${svc.name}-${Date.now()}`;
+        if (seenRefs.has(lockKey)) return false;
+        seenRefs.add(lockKey);
+        const lock = acquireLock(lockKey, {
+          sender: item.sender || "unknown",
+          channel: item.channel || item.channel_id || "unknown",
+        });
+        return lock.acquired;
+      });
+      totalItems += newItems.length;
+
+      for (const item of newItems) {
+        await processItem(item, svc.name);
+      }
+
+      if (result.errors.length > 0) {
+        console.warn(`[daemon] ${svc.name} errors:`, result.errors.slice(0, 3));
+      }
+    } catch (err) {
+      console.error(`[daemon] ${svc.name} poll failed:`, err.message);
+    }
+  }
+
+  recordPoll(totalItems);
+  if (totalItems > 0) {
+    console.log(`[daemon] Poll found ${totalItems} new items`);
+  }
+
+}
+
+// ---------------------------------------------------------------------------
+// Process a single item
+// ---------------------------------------------------------------------------
+
+async function processItem(item, service) {
+  const itemId = item.raw_ref || item.id || `${service}-${Date.now()}`;
+
+  try {
+    // Enrich item with channel-type metadata for directed-message detection
+    const channelStr = (item.channel || "").toLowerCase();
+    const channelId = item.channel_id || "";
+    const isDm = channelStr.startsWith("dm/") || channelId.startsWith("D");
+    const sophieInThread = !!(item.thread_context && /^Sophie:/m.test(item.thread_context));
+    item.is_dm = isDm;
+    item.sophie_in_thread = sophieInThread;
+
+    // Classify via Haiku API
+    const classResult = await classifyItem({
+      sender: item.sender || "unknown",
+      sender_privilege: item.sender_privilege || item.priority_signals?.from_ceo ? "ceo" : "unknown",
+      channel: item.channel || "unknown",
+      service,
+      content: item.content || item.subject || "",
+      is_reply: item.is_reply || false,
+      thread_context: item.thread_context || null,
+      subject: item.subject || "",
+      is_dm: isDm,
+      is_group: !isDm && service === "slack",
+      sophie_in_thread: sophieInThread,
+    });
+
+    recordClassification(true);
+
+    logEvent("classifications", {
+      item_id: itemId,
+      sender: item.sender,
+      service,
+      ...classResult,
+    });
+
+    console.log(`[daemon] Classified: [${classResult.priority}] ${classResult.summary} → ${classResult.model} (${classResult.action})`);
+
+    // Skip ignored items
+    if (classResult.category === "ignore" || classResult.action === "ignore") {
+      markProcessed(item, service);
+      return;
+    }
+
+    // DIRECTED-MESSAGE GATE: In channels and group chats, only respond to
+    // messages that are clearly directed at Sophie. DMs always pass.
+    // This prevents Sophie from inserting herself into every conversation.
+    //
+    // Two-layer check:
+    //   1. If LLM says NOT directed → verify with rules (catch missed @mentions, CEO, DMs)
+    //   2. If LLM says directed BUT it's a non-DM channel → verify with rules (catch over-eager LLM)
+    if (service === "slack") {
+      const isDm = item.is_dm || (item.channel || "").startsWith("dm/") || (item.channel_id || "").startsWith("D");
+
+      if (!classResult.directed_at_sophie) {
+        // LLM says not directed — double-check with rule-based heuristics
+        // to catch clear signals the LLM may have missed (DM, CEO, @mention)
+        const ruleCheck = isDirectedAtSophie(item);
+        if (!ruleCheck) {
+          console.log(`[daemon] Directed-message filter: skipping non-directed message from ${item.sender} in ${item.channel}`);
+          logEvent("classifications", {
+            item_id: itemId,
+            sender: item.sender,
+            service,
+            skipped: true,
+            reason: "not_directed_at_sophie",
+            classifier_directed: false,
+            rule_directed: false,
+            summary: classResult.summary,
+          });
+          markProcessed(item, service);
+          return;
+        }
+        // Rule-based says directed — override LLM
+        console.log(`[daemon] Directed-message override: LLM said not directed but rule-based detected direction (${item.sender} in ${item.channel})`);
+      } else if (!isDm) {
+        // LLM says directed in a channel/group — sanity-check with rules.
+        // If rule-based also agrees, proceed. If rules say no AND the message
+        // doesn't contain Sophie's name, the LLM was probably over-eager.
+        const ruleCheck = isDirectedAtSophie(item);
+        const content = (item.content || "").toLowerCase();
+        const mentionsSophie = content.includes("sophie") || content.includes("<@U09");
+        if (!ruleCheck && !mentionsSophie) {
+          console.log(`[daemon] Directed-message filter (LLM override): LLM said directed but rules disagree for ${item.sender} in ${item.channel} — skipping`);
+          logEvent("classifications", {
+            item_id: itemId,
+            sender: item.sender,
+            service,
+            skipped: true,
+            reason: "llm_over_eager_directed",
+            classifier_directed: true,
+            rule_directed: false,
+            summary: classResult.summary,
+          });
+          markProcessed(item, service);
+          return;
+        }
+      }
+    }
+
+    // THREAD / CHANNEL DEDUP: Prevent multiple sessions from responding to the
+    // same conversation. For thread replies, lock by thread_ts. For non-threaded
+    // DMs, lock by channel_id (so rapid-fire DMs don't each spawn a session).
+    // This check runs BEFORE quick reply to prevent ALL duplicate responses.
+    {
+      const channel = item.channel_id || (item.raw_ref ? item.raw_ref.match(/slack:([^:]+):/)?.[1] : null) || item.channel;
+      const threadTs = item.thread_id || (isDm ? `dm-channel` : null);
+      if (threadTs && channel) {
+        const threadCheck = acquireThreadLock(channel, threadTs);
+        if (!threadCheck.allowed) {
+          console.log(`[daemon] Thread/channel dedup: skipping item from ${item.sender} — ${threadCheck.reason}`);
+          logEvent("classifications", {
+            item_id: itemId,
+            sender: item.sender,
+            service,
+            skipped: true,
+            reason: `thread_dedup: ${threadCheck.reason}`,
+          });
+          markProcessed(item, service);
+          return;
+        }
+      }
+    }
+
+    // REQUEST CLAIMING: For action_required items, claim the request so
+    // parallel sessions (from overlapping poll cycles or backlog sweep)
+    // don't independently process the same "email me the brief" request.
+    // Claim key: action_type + recipient channel + summary (normalized).
+    if (classResult.category === "action_required" && (classResult.action === "respond" || classResult.action === "draft" || classResult.action === "research")) {
+      const claimAttrs = {
+        recipient: item.channel_id || item.channel || item.sender || "unknown",
+        subject: classResult.summary || item.subject || "",
+        action_type: classResult.action,
+      };
+      const claim = claimRequest(claimAttrs);
+      if (!claim.claimed) {
+        console.log(`[daemon] Request claim denied for ${item.sender}: ${claim.reason}`);
+        logEvent("classifications", {
+          item_id: itemId,
+          sender: item.sender,
+          service,
+          skipped: true,
+          reason: `request_claim_denied: ${claim.reason}`,
+          summary: classResult.summary,
+        });
+        markProcessed(item, service);
+        return;
+      }
+    }
+
+    // QUICK REPLY PATH: Direct responses via API + Slack/Gmail posting.
+    // No claude --print session needed. ~4-8 seconds total.
+    if (isQuickReply(classResult)) {
+      console.log(`[daemon] Quick reply path for ${item.sender} (${classResult.model})`);
+      const result = await sendQuickResponse(item, classResult);
+      if (result.sent) {
+        markProcessed(item, service);
+        return;
+      }
+      // If quick reply failed to send or was blocked by validation, fall through to dispatch a full session
+      const reason = result.blocked ? `validation blocked: ${result.issues?.map(i => i.rule).join(", ")}` : "send failed";
+      console.warn(`[daemon] Quick reply not sent (${reason}), falling through to session dispatch`);
+    }
+
+    // COMPLEX WORK PATH: Send immediate holding message, then dispatch full session
+    let holdingText = null;
+    try {
+      if (classResult.action === "respond" || classResult.action === "draft" || classResult.action === "research") {
+        const holdResult = await sendHoldingMessage(item, classResult);
+        holdingText = holdResult.sent ? holdResult.holdingText : null;
+        if (holdResult.sent) {
+          updateLock(itemId, { holdingSent: true });
+        }
+      }
+    } catch (err) {
+      console.error(`[daemon] Holding message failed (non-fatal): ${err.message}`);
+    }
+
+    // Build prompt with holding message context and dispatch
+    const prompt = await buildPrompt(item, classResult, {
+      type: "inbox",
+      holdingMessage: holdingText,
+    });
+    dispatch(prompt, item, classResult);
+    recordSession(false); // spawned
+    markProcessed(item, service);
+
+  } catch (err) {
+    console.error(`[daemon] Failed to process item ${itemId}:`, err.message);
+    recordClassification(false);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Mark inbox item as processed
+// ---------------------------------------------------------------------------
+
+function markProcessed(item, service) {
+  // The pollers write files to state/inbox/{service}/
+  // We mark them by renaming to .processed
+  try {
+    const inboxDir = join(SOPHIE_AI_DIR, "state", "inbox", service);
+    const files = readdirSync(inboxDir).filter(
+      (f) => !f.endsWith(".processed") && (f.includes(item.id) || f.includes(item.raw_ref))
+    );
+    for (const file of files) {
+      const src = join(inboxDir, file);
+      const dst = join(inboxDir, file + ".processed");
+      renameSync(src, dst);
+    }
+  } catch {
+    // Not all items have inbox files (e.g. backlog items)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Backlog Sweep
+// ---------------------------------------------------------------------------
+
+async function sweepBacklog() {
+  const slots = availableSlots();
+  if (slots <= 0) return; // No capacity
+
+  try {
+    const queueDir = join(SOPHIE_AI_DIR, "state", "queues");
+    const files = readdirSync(queueDir).filter((f) => f.endsWith(".yaml"));
+    const actionableItems = [];
+
+    for (const file of files) {
+      try {
+        const content = readFileSync(join(queueDir, file), "utf-8");
+        // Simple YAML parsing — look for items with status: open
+        const items = content.split(/^-\s+/m).filter(Boolean);
+        for (const itemBlock of items) {
+          // Strip comment lines to avoid matching schema templates (e.g. "# title: string")
+          const cleanBlock = itemBlock.split("\n").filter((l) => !l.trimStart().startsWith("#")).join("\n");
+          // Match both quoted ("title":) and unquoted (title:) YAML keys
+          const statusMatch = cleanBlock.match(/"?status"?:\s*["']?(open|in_progress)["']?/);
+          const blockedMatch = cleanBlock.match(/"?status"?:\s*["']?blocked["']?/);
+          if (statusMatch && !blockedMatch) {
+            const titleMatch = cleanBlock.match(/"?title"?:\s*["']?(.+?)["']?\s*\n/);
+            const priorityMatch = cleanBlock.match(/"?priority"?:\s*["']?(critical|high|normal|low)["']?/);
+            const nextActionMatch = cleanBlock.match(/"?next_action"?:\s*["']?(.+?)["']?\s*\n/);
+            if (titleMatch && nextActionMatch) {
+              actionableItems.push({
+                title: titleMatch[1],
+                priority: priorityMatch?.[1] || "normal",
+                next_action: nextActionMatch[1],
+                source_file: file,
+                raw: cleanBlock.substring(0, 500),
+              });
+            }
+          }
+        }
+      } catch {
+        // Skip unparseable queue files
+      }
+    }
+
+    // Sort: critical first, then high, then normal
+    const priorityOrder = { critical: 0, high: 1, normal: 2, low: 3 };
+    actionableItems.sort((a, b) => (priorityOrder[a.priority] || 3) - (priorityOrder[b.priority] || 3));
+
+    // Filter out items that already have active sessions or exceeded retries
+    const dispatchable = actionableItems.filter((qi) => {
+      const check = canDispatchBacklog(qi);
+      if (!check.allowed) {
+        if (check.reason === "max_retries_exceeded") {
+          console.log(`[daemon] Backlog skip: "${qi.title}" — retries exhausted`);
+        }
+        return false;
+      }
+      return true;
+    });
+
+    // Dispatch top items up to available slots
+    const toDispatch = dispatchable.slice(0, Math.min(slots, 5));
+    let dispatched = 0;
+
+    for (const queueItem of toDispatch) {
+      const classResult = {
+        priority: queueItem.priority,
+        action: "queue",
+        model: queueItem.priority === "critical" ? "opus" : "sonnet",
+        summary: queueItem.title,
+        category: "action_required",
+      };
+
+      const prompt = await buildPrompt(null, classResult, {
+        type: "backlog",
+        queueItem,
+      });
+
+      dispatch(prompt, queueItem, classResult, "backlog");
+      recordSession(false);
+      dispatched++;
+    }
+
+    if (dispatched > 0) {
+      console.log(`[daemon] Backlog sweep: dispatched ${dispatched} items (${actionableItems.length - dispatchable.length} skipped: in-flight or exhausted)`);
+    }
+  } catch (err) {
+    console.error(`[daemon] Backlog sweep error:`, err.message);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+  console.log("╔══════════════════════════════════════════════════════════╗");
+  console.log("║          Sophie Daemon — Reactive Event Processor       ║");
+  console.log("╠══════════════════════════════════════════════════════════╣");
+  console.log(`║  Directory:   ${SOPHIE_AI_DIR}`);
+  console.log(`║  Poll:        every ${POLL_INTERVAL / 1000}s`);
+  console.log(`║  Backlog:     every ${BACKLOG_INTERVAL / 1000}s`);
+  console.log(`║  Concurrency: up to ${process.env.DAEMON_MAX_CONCURRENT || 10} parallel sessions`);
+  console.log("╚══════════════════════════════════════════════════════════╝");
+
+  // Check for emergency stop
+  try {
+    readFileSync(join(SOPHIE_AI_DIR, ".emergency-stop"));
+    console.error("[daemon] Emergency stop active — exiting");
+    process.exit(0);
+  } catch {
+    // No emergency stop — continue
+  }
+
+  // Clear orphaned state from prior daemon instances
+  const staleCleared = scanStaleLocks();
+  if (staleCleared > 0) {
+    console.log(`[daemon] Cleared ${staleCleared} stale session locks`);
+  }
+  resetActiveSessions();
+
+  // Initial poll immediately
+  await poll();
+
+  // Poll loop
+  setInterval(async () => {
+    try {
+      // Check emergency stop
+      try { readFileSync(join(SOPHIE_AI_DIR, ".emergency-stop")); process.exit(0); } catch {}
+      await poll();
+    } catch (err) {
+      console.error("[daemon] Poll loop error:", err.message);
+    }
+  }, POLL_INTERVAL);
+
+  // Backlog sweep
+  setInterval(async () => {
+    try {
+      await sweepBacklog();
+    } catch (err) {
+      console.error("[daemon] Backlog sweep error:", err.message);
+    }
+  }, BACKLOG_INTERVAL);
+
+  // Health dashboard
+  setInterval(() => {
+    try {
+      writeHealthDashboard();
+    } catch (err) {
+      console.error("[daemon] Health write error:", err.message);
+    }
+  }, HEALTH_INTERVAL);
+
+  // Graceful shutdown — clean up active.json so next startup doesn't see stale sessions
+  process.on("SIGTERM", () => {
+    console.log("[daemon] SIGTERM received, shutting down gracefully");
+    resetActiveSessions();
+    writeHealthDashboard();
+    process.exit(0);
+  });
+
+  process.on("SIGINT", () => {
+    console.log("[daemon] SIGINT received, shutting down");
+    resetActiveSessions();
+    writeHealthDashboard();
+    process.exit(0);
+  });
+
+  console.log("[daemon] Running. Ctrl+C to stop.");
+}
+
+main().catch((err) => {
+  console.error("[daemon] Fatal:", err);
+  process.exit(1);
+});