@cohortapp/agent-sdk 2.3.0

package/scripts/poller/slack-cloud-relay-client.mjs

@@ -0,0 +1,368 @@
+/**
+ * Maestro — Slack Cloud-Relay Client
+ *
+ * Persistent SSE consumer that connects to the Railway-hosted cloud-relay
+ * (scripts/cloud-relay/) and turns the Slack user-scope event stream into
+ * inbox YAML files via writeInboxItem. Drop-in replacement for the
+ * bot-scoped Socket Mode listener when the agent's primary message
+ * surface is the agent's USER account rather than a bot user.
+ *
+ * Lifecycle:
+ *   1. Honour .emergency-stop before any I/O.
+ *   2. Open an HTTPS SSE connection to CLOUD_RELAY_URL/events/stream with
+ *      a Bearer header containing RELAY_SHARED_SECRET.
+ *   3. Parse `data:` lines as event_callback envelopes; reuse the
+ *      shouldKeepEvent + eventToInboxItem helpers from slack-socket-mode
+ *      so the inbox YAML shape is identical to what Socket Mode produces.
+ *   4. On disconnect, reconnect with exponential back-off + jitter.
+ *
+ * Env (read from .env):
+ *   CLOUD_RELAY_URL           — Railway service base URL (https://...)
+ *   RELAY_SHARED_SECRET       — auth bearer; must match Railway env var
+ *   SLACK_BOT_TOKEN           — used by shouldKeepEvent for own/peer
+ *                               filtering (no requests made on this path)
+ */
+
+import { config as loadEnv } from "dotenv";
+import { join, dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { existsSync, readFileSync, appendFileSync, mkdirSync, readdirSync } from "node:fs";
+
+import {
+  shouldKeepEvent,
+} from "./slack-socket-mode.mjs";
+import { createSseParser, isReplayInSiblings } from "./lib/cloud-relay-dedup.mjs";
+import { writeFileSync } from "node:fs";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const AGENT_REPO_DIR = process.env.AGENT_DIR || resolve(__dirname, "../..");
+loadEnv({ path: join(AGENT_REPO_DIR, ".env") });
+
+const CLOUD_RELAY_URL = (process.env.CLOUD_RELAY_URL || "").replace(/\/$/, "");
+const RELAY_SHARED_SECRET = process.env.RELAY_SHARED_SECRET || "";
+
+const RECONNECT_INITIAL_MS = 2_000;
+const RECONNECT_MAX_MS = 60_000;
+const HEARTBEAT_GRACE_MS = 90_000; // SSE heartbeat is 25s — be generous before declaring dead
+
+// ---------------------------------------------------------------------------
+// Identity loading (mirrors slack-socket-mode)
+// ---------------------------------------------------------------------------
+
+function loadAgent() {
+  const path = join(AGENT_REPO_DIR, "config", "agent.json");
+  if (!existsSync(path)) return { firstName: "Agent" };
+  try {
+    return JSON.parse(readFileSync(path, "utf-8"));
+  } catch {
+    return { firstName: "Agent" };
+  }
+}
+
+function loadPeerSlackIds() {
+  // Read agents/index.json if present; otherwise empty set (lone agent).
+  const peers = new Set();
+  const idxPath = join(AGENT_REPO_DIR, "agents", "index.json");
+  if (!existsSync(idxPath)) return peers;
+  try {
+    const idx = JSON.parse(readFileSync(idxPath, "utf-8"));
+    for (const a of idx.agents || []) {
+      if (a.slackMemberId) peers.add(a.slackMemberId);
+    }
+  } catch { /* */ }
+  return peers;
+}
+
+// ---------------------------------------------------------------------------
+// Logging
+// ---------------------------------------------------------------------------
+
+const LOG_DIR = join(AGENT_REPO_DIR, "logs", "polling");
+try { mkdirSync(LOG_DIR, { recursive: true }); } catch { /* */ }
+
+function log(level, message, data) {
+  const ts = new Date().toISOString();
+  const line = { ts, level, message, ...(data || {}) };
+  if (level === "error") console.error(`[slack-cloud-relay] ${message}`, data || "");
+  else console.log(`[slack-cloud-relay] ${message}`, data || "");
+  try {
+    appendFileSync(
+      join(LOG_DIR, `${ts.slice(0, 10)}-slack-cloud-relay.jsonl`),
+      JSON.stringify(line) + "\n",
+    );
+  } catch { /* best-effort */ }
+}
+
+// SSE parsing (createSseParser) and filesystem replay dedup
+// (isReplayInSiblings) live in ./lib/cloud-relay-dedup.mjs so they can be
+// unit-tested without importing dotenv or opening a live connection.
+
+// ---------------------------------------------------------------------------
+// Exponential back-off (mirrors slack-socket-mode)
+// ---------------------------------------------------------------------------
+
+function nextBackoffMs(currentMs) {
+  const doubled = Math.min(RECONNECT_MAX_MS, Math.max(RECONNECT_INITIAL_MS, currentMs * 2));
+  const jitter = Math.floor(Math.random() * 0.25 * doubled);
+  return Math.min(RECONNECT_MAX_MS, doubled + jitter);
+}
+
+// ---------------------------------------------------------------------------
+// Main loop
+// ---------------------------------------------------------------------------
+
+let stopped = false;
+let backoffMs = RECONNECT_INITIAL_MS;
+
+// Bounded dedup set — Slack retries the same event_id up to 3 times within a
+// short window. Keep the last 2000 IDs in memory; that's well above any
+// plausible burst while staying cheap on RAM.
+const seenEventIds = new Set();
+const MAX_SEEN_EVENT_IDS = 2000;
+
+async function connect(identity, peerSlackIds) {
+  if (stopped) return;
+  if (existsSync(join(AGENT_REPO_DIR, ".emergency-stop"))) {
+    log("info", ".emergency-stop present — refusing to connect");
+    return;
+  }
+  if (!CLOUD_RELAY_URL || !RELAY_SHARED_SECRET) {
+    log("error", "CLOUD_RELAY_URL or RELAY_SHARED_SECRET not set — cannot connect");
+    return;
+  }
+
+  const url = `${CLOUD_RELAY_URL}/events/stream`;
+  log("info", "connecting", { url });
+
+  let response;
+  try {
+    response = await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${RELAY_SHARED_SECRET}`,
+        Accept: "text/event-stream",
+      },
+    });
+  } catch (err) {
+    log("error", `connect failed: ${err.message}`);
+    scheduleReconnect(identity, peerSlackIds);
+    return;
+  }
+
+  if (!response.ok) {
+    log("error", `relay rejected: HTTP ${response.status}`);
+    scheduleReconnect(identity, peerSlackIds);
+    return;
+  }
+
+  log("info", "connected");
+  backoffMs = RECONNECT_INITIAL_MS; // reset on a successful connect
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  const parser = createSseParser();
+  let lastByteAt = Date.now();
+
+  // Heartbeat watchdog — if we haven't seen ANY bytes in HEARTBEAT_GRACE_MS,
+  // declare the connection dead and reconnect. Relay sends `: hb\n\n` every
+  // 25s, so a 90s grace catches both true silence and a proxy that froze.
+  const watchdog = setInterval(() => {
+    if (Date.now() - lastByteAt > HEARTBEAT_GRACE_MS) {
+      log("warn", "watchdog tripped — no bytes from relay", {
+        silent_ms: Date.now() - lastByteAt,
+      });
+      clearInterval(watchdog);
+      try { reader.cancel(); } catch { /* */ }
+    }
+  }, 30_000);
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        log("info", "stream closed by relay");
+        break;
+      }
+      lastByteAt = Date.now();
+      const chunk = decoder.decode(value, { stream: true });
+      const events = parser.push(chunk);
+      for (const raw of events) {
+        try {
+          const webhook = JSON.parse(raw);
+          handleWebhook(webhook, identity, peerSlackIds);
+        } catch (err) {
+          log("warn", `bad event JSON: ${err.message}`, { sample: raw.slice(0, 200) });
+        }
+      }
+    }
+  } catch (err) {
+    log("warn", `stream error: ${err.message}`);
+  } finally {
+    clearInterval(watchdog);
+  }
+
+  scheduleReconnect(identity, peerSlackIds);
+}
+
+function scheduleReconnect(identity, peerSlackIds) {
+  if (stopped) return;
+  const delay = backoffMs;
+  backoffMs = nextBackoffMs(backoffMs);
+  log("info", "reconnect scheduled", { delay_ms: delay, next_backoff_ms: backoffMs });
+  setTimeout(() => connect(identity, peerSlackIds), delay);
+}
+
+/**
+ * Drop the Slack Events API webhook envelope into `state/inbox/slack/` as a
+ * `.json` file. This is the format that `slack-poller.mjs`'s events-server
+ * scanner reads (it walks `*.json && !.processed` every poll cycle and
+ * translates them into items[] via the `data.event` fallback path), so the
+ * daemon picks up cloud-relay events through the exact same code path it
+ * already uses for the legacy slack-events-server.mjs webhook receiver.
+ *
+ * We deliberately do NOT write the agent-shaped YAML inbox file here:
+ * the daemon's poll cycle returns items[] from in-memory API calls plus
+ * the events-server `.json` scanner, NOT from `.yaml` files in the inbox
+ * directory. Earlier YAML writes were invisible to the daemon (observed
+ * on 2026-05-12 when 4 Alex DMs landed as YAML but were never picked
+ * up because pollSlack only scans `.json`).
+ *
+ * Slack Events API webhook payload shape:
+ *   { token, team_id, api_app_id, type: "event_callback",
+ *     event: { type, user, text, channel, ts, ... },
+ *     event_id, event_time }
+ *
+ * slack-poller.mjs reads `data.event` (line ~775) so writing the raw
+ * webhook body works directly.
+ */
+function handleWebhook(webhook, identity, peerSlackIds) {
+  if (!webhook || webhook.type !== "event_callback") {
+    log("info", "ignored-non-event-callback", { type: webhook?.type });
+    return;
+  }
+
+  // Re-use shouldKeepEvent so we don't store bot-echo + self-message noise
+  // on disk for the scanner to filter later. Same filter Socket Mode used.
+  const envelope = { type: "events_api", payload: webhook };
+  const decision = shouldKeepEvent(envelope, {
+    ownSlackId: identity.slackMemberId,
+    peerSlackIds,
+  });
+  if (!decision.keep) {
+    log("info", "event-dropped", { reason: decision.reason, event_id: webhook.event_id });
+    return;
+  }
+
+  // Dedup: Slack retries failed deliveries with the same event_id. If we've
+  // already written this event to the inbox, discard quietly.
+  const eventId = webhook.event_id;
+  if (eventId) {
+    if (seenEventIds.has(eventId)) {
+      log("info", "event-id-dedup-skipped", { event_id: eventId });
+      return;
+    }
+    seenEventIds.add(eventId);
+    // Trim oldest entries when the set grows beyond the cap.
+    if (seenEventIds.size > MAX_SEEN_EVENT_IDS) {
+      const oldest = seenEventIds.values().next().value;
+      seenEventIds.delete(oldest);
+    }
+  }
+
+  const event = webhook.event || {};
+  const ts = event.ts || webhook.event_time || `${Date.now() / 1000}`;
+  const isoTs = new Date().toISOString().replace(/[:.]/g, "-");
+  // Filename mirrors the events-server convention so duplicate dedup works:
+  // <iso-arrival>-<slack-ts-with-dash>.json
+  const tsKey = String(ts).replace(".", "-");
+  const filename = `${isoTs}-${tsKey}-cloud-relay.json`;
+  const inboxDir = join(AGENT_REPO_DIR, "state", "inbox", "slack");
+  const path = join(inboxDir, filename);
+
+  // Persistent dedup: the in-process seenEventIds Set is reset on every
+  // process restart, and Slack's cloud-relay reconnects can replay the
+  // same event_id after a reconnect — observed 13x on 2026-05-19 for
+  // event_id=Ev0B4MAFCX62 across ~15-min restart cycles, each spawning
+  // a fresh duplicate reply. Gate on filesystem state: if any sibling
+  // file for this (ts,cloud-relay) already exists — pending OR
+  // .processed — discard quietly.
+  try {
+    const siblings = readdirSync(inboxDir);
+    const isReplay = isReplayInSiblings(siblings, tsKey);
+    if (isReplay) {
+      log("info", "event-ts-fs-dedup-skipped", {
+        event_id: webhook.event_id,
+        ts,
+      });
+      return;
+    }
+  } catch (err) {
+    log("warn", `inbox-readdir-failed: ${err.message}`);
+  }
+
+  // Augment the webhook envelope with a `source` marker so audit can tell
+  // relay-delivered events apart from poller- or socket-mode-delivered ones,
+  // and add `channel_id` at the top level (the scanner reads it from there).
+  const payload = {
+    ...webhook,
+    channel_id: event.channel || "",
+    received_at: new Date().toISOString(),
+    source: "cloud-relay",
+  };
+
+  try {
+    writeFileSync(path, JSON.stringify(payload, null, 2));
+    log("info", "inbox-json-written", {
+      filename,
+      channel: event.channel,
+      user: event.user,
+      event_id: webhook.event_id,
+    });
+  } catch (err) {
+    log("error", `writeFileSync failed: ${err.message}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Bootstrap
+// ---------------------------------------------------------------------------
+
+/**
+ * Wire up signal handlers and open the relay connection. Extracted into a
+ * function so the module can be imported by the test suite without spinning up
+ * a live SSE connection — only a direct `node slack-cloud-relay-client.mjs`
+ * invocation (the launchd wrapper's entrypoint) starts the client.
+ */
+function startClient() {
+  const identity = loadAgent();
+  const peerSlackIds = loadPeerSlackIds();
+
+  const shutdown = (signal, code) => {
+    log("info", `received ${signal}, shutting down (exit ${code})`);
+    stopped = true;
+    // Match the slack-socket-mode pattern: SIGTERM signals a respawn from
+    // launchd (exit 143), SIGINT is human-initiated stop (exit 0).
+    setTimeout(() => process.exit(code), 250);
+  };
+  process.on("SIGTERM", () => shutdown("SIGTERM", 143));
+  process.on("SIGINT", () => shutdown("SIGINT", 0));
+  process.on("unhandledRejection", (reason) => {
+    log("error", `unhandled rejection: ${reason?.message || reason}`);
+  });
+  process.on("uncaughtException", (err) => {
+    log("error", `uncaught exception: ${err.message}`);
+    setTimeout(() => process.exit(1), 250);
+  });
+
+  connect(identity, peerSlackIds);
+}
+
+// Boot the client only when this file is the entrypoint (so it can be imported
+// as a module by the test suite). The `process.argv[1]` check mirrors
+// slack-socket-mode.mjs; the launchd wrapper execs this file by absolute path,
+// so production behaviour is unchanged.
+const isDirectInvocation = process.argv[1] && resolve(process.argv[1]) === __filename;
+
+if (isDirectInvocation) {
+  startClient();
+}