@cohortapp/agent-sdk 2.3.0

package/lib/org/mesh.mjs

@@ -0,0 +1,459 @@
+/**
+ * lib/org/mesh.mjs — the agent's RUNTIME membership of the Cohort org mesh.
+ *
+ * THE GAP this closes: the Cohort server exists and lib/org/client.mjs can talk
+ * to it, but nothing made an agent actually JOIN the mesh at runtime — it never
+ * registered itself, never presence-beat, and never acted on the server's
+ * directives (the kill switch, policy-stale nudges, resync). connectOrgMesh wires
+ * that in at daemon boot:
+ *
+ *   1. register() — publish this agent's directory entry at boot (idempotent).
+ *   2. presence-beat loop (~30s) — collect the SHARED STATUS SNAPSHOT
+ *      (lib/telemetry/collect.collectStatus) and ride it on the beat as
+ *      `params.status` (fail-open: a collector error beats WITHOUT status, never
+ *      breaks the beat); a critical alert in that snapshot also escalates locally
+ *      (notify the operator). Then call presence.beat and ACT on the returned
+ *      directives[] (and on an error-frame `directive`, e.g. when a deactivated
+ *      token is rejected at auth):
+ *        - "halt"         → trip the local kill switch: write .emergency-stop and
+ *                           signal the daemon to stop (requestStop()).
+ *        - "rotate_token" → log + drop a re-pair flag (state/org/repair-needed).
+ *        - "policy_stale" → re-fetch + re-verify the signed policy bundle
+ *                           (lib/org/policy.fetchAndApply against the PINNED root).
+ *        - "resync"       → re-fetch the directory (client.fetchDirectory) into the
+ *                           local peers cache (state/org/peers/<id>.json).
+ *   3. stop() — clears the interval so the daemon can shut the mesh down cleanly.
+ *
+ * FAIL-OPEN by design (mirrors lib/org/client): if the org is disabled
+ * connectOrgMesh is a no-op; if the org is unreachable every step swallows the
+ * failure and the daemon keeps running standalone. Nothing here ever throws past
+ * the boundary — a mesh problem must never take the agent down.
+ *
+ * G16 (zero-LLM-in-the-coordination-loop): pure HTTP + signature verify + file
+ * I/O. ZERO LLM calls.
+ *
+ * Node builtins only. The clock (`now`), interval scheduler (`setInterval`),
+ * `fetchImpl`, and the org `client` are all injectable for deterministic tests.
+ *
+ * @module lib/org/mesh
+ */
+
+"use strict";
+
+import { join, resolve } from "node:path";
+import { mkdirSync, writeFileSync } from "node:fs";
+
+import * as defaultClient from "./client.mjs";
+import * as policy from "./policy.mjs";
+import { assembleSelfEntry } from "./registry.mjs";
+import {
+  maybeRefresh,
+  currentToolsetVersion,
+  writeIntegrationToolsToDisk,
+} from "./integration-tools.mjs";
+import { collectStatus as defaultCollectStatus } from "../telemetry/collect.mjs";
+import { hasCritical } from "../telemetry/alerts.mjs";
+
+/** How often to presence-beat (ms). Matches the server's liveness expectation. */
+export const DEFAULT_BEAT_MS = 30_000;
+
+/**
+ * A directive we recognise → its handler. Anything else is ignored (forward-
+ * compatible). `toolset_changed` (SP5) forces an immediate integration-tool
+ * refresh ahead of the 30s poll; hq MAY later surface it as a presence.beat
+ * directive — until then the per-beat poll (refreshIntegrationTools, below)
+ * covers it, and this entry means the moment hq emits the frame it Just Works.
+ */
+const KNOWN_DIRECTIVES = new Set([
+  "halt",
+  "rotate_token",
+  "policy_stale",
+  "resync",
+  "toolset_changed",
+]);
+
+function logInfo(msg) { try { console.log(`[org-mesh] ${msg}`); } catch { /* never throw from logging */ } }
+function logWarn(msg) { try { console.warn(`[org-mesh] ${msg}`); } catch { /* never throw from logging */ } }
+
+/**
+ * Default stop signal: drop the daemon out of its run loop. The daemon's own
+ * SIGTERM handler clears the cadence consumer + this mesh handle. We send to
+ * ourselves (process group leader); fail-open if signalling isn't permitted.
+ */
+function defaultRequestStop() {
+  try { process.kill(process.pid, "SIGTERM"); } catch { /* fail-open */ }
+}
+
+/**
+ * Trip the local kill switch in response to a "halt" directive.
+ *   1. Write `.emergency-stop` (the same flag emergency-stop.sh drops) so a
+ *      launchd restart is refused at the boot gate — the halt is durable across
+ *      a crash/restart, not just this process.
+ *   2. requestStop() — signal the daemon to exit cleanly NOW.
+ * Idempotent + fail-open. Never throws.
+ */
+function tripKillSwitch(agentRoot, requestStop, now) {
+  const at = new Date((typeof now === "function" ? now() : Date.now())).toISOString();
+  try {
+    writeFileSync(join(agentRoot, ".emergency-stop"), `${at}\n`);
+    logWarn(`HALT directive received — wrote .emergency-stop (${at}) and stopping`);
+  } catch (err) {
+    logWarn(`HALT directive received — failed to write .emergency-stop (${err && err.message}); stopping anyway`);
+  }
+  try { requestStop(); } catch { /* fail-open */ }
+}
+
+/**
+ * Re-pair flag in response to a "rotate_token" directive: the agent's token is
+ * expired/rotated and a human must re-pair it. We don't auto-pair (that needs a
+ * human-issued code); we drop a durable flag + log so the operator/doctor sees it.
+ * Fail-open.
+ */
+function flagRepairNeeded(agentRoot, now) {
+  const at = new Date((typeof now === "function" ? now() : Date.now())).toISOString();
+  try {
+    mkdirSync(join(agentRoot, "state", "org"), { recursive: true });
+    writeFileSync(
+      join(agentRoot, "state", "org", "repair-needed.json"),
+      JSON.stringify({ reason: "rotate_token", at }, null, 2),
+    );
+    logWarn(`rotate_token directive — token rotation required; flagged state/org/repair-needed.json (${at})`);
+  } catch (err) {
+    logWarn(`rotate_token directive — failed to flag re-pair (${err && err.message})`);
+  }
+}
+
+/**
+ * Locally escalate a critical telemetry alert to the operator: log it loudly and
+ * drop a durable flag (state/org/alert-critical.json) the operator/doctor can see,
+ * mirroring flagRepairNeeded. Idempotent overwrite + fail-open; never throws. This
+ * is the local half of "a critical alert escalates" — the snapshot also rides the
+ * beat so the server/directory sees it.
+ */
+function escalateCriticalAlerts(agentRoot, alerts, now) {
+  const at = new Date((typeof now === "function" ? now() : Date.now())).toISOString();
+  const crit = (Array.isArray(alerts) ? alerts : []).filter((a) => a && a.severity === "critical");
+  if (crit.length === 0) return;
+  for (const a of crit) {
+    logWarn(`CRITICAL telemetry alert [${a.id || a.kind || "?"}]: ${a.detail || ""}`);
+  }
+  try {
+    mkdirSync(join(agentRoot, "state", "org"), { recursive: true });
+    writeFileSync(
+      join(agentRoot, "state", "org", "alert-critical.json"),
+      JSON.stringify({ at, alerts: crit }, null, 2),
+    );
+  } catch (err) {
+    logWarn(`failed to persist critical alert flag (${err && err.message})`);
+  }
+}
+
+/**
+ * Collect the SHARED STATUS SNAPSHOT for this beat, fully fail-open: any collector
+ * error yields `undefined` so the beat goes out WITHOUT a status (never broken).
+ * Returns the snapshot or undefined.
+ */
+async function collectBeatStatus(ctx) {
+  try {
+    const fn = typeof ctx.collectStatus === "function" ? ctx.collectStatus : defaultCollectStatus;
+    const status = await fn({ agentRoot: ctx.agentRoot, now: ctx.now });
+    return status && typeof status === "object" ? status : undefined;
+  } catch (err) {
+    logWarn(`status collection failed (${err && err.message}) — beating without status`);
+    return undefined;
+  }
+}
+
+/**
+ * Re-fetch the org directory and persist each peer into the local peers cache
+ * (state/org/peers/<id>.json), which registry.readDirectory merges. In response
+ * to a "resync" directive. Returns the count written. Fail-open.
+ */
+async function resyncDirectory(o) {
+  const { agentRoot, client, cfg, fetchImpl } = o;
+  let peers = [];
+  try {
+    peers = await client.fetchDirectory({ cfg, fetchImpl });
+  } catch { peers = []; }
+  if (!Array.isArray(peers) || peers.length === 0) {
+    logInfo("resync directive — directory empty/unreachable, peers cache left as-is");
+    return 0;
+  }
+  const peersDir = join(agentRoot, "state", "org", "peers");
+  let written = 0;
+  try { mkdirSync(peersDir, { recursive: true }); } catch { /* fail-open */ }
+  for (const p of peers) {
+    if (!p || !p.id) continue;
+    const id = String(p.id).toLowerCase().replace(/[^a-z0-9._-]/gi, "_");
+    if (!id) continue;
+    try {
+      writeFileSync(join(peersDir, `${id}.json`), JSON.stringify({ ...p, source: "synced" }, null, 2));
+      written += 1;
+    } catch { /* skip this peer, keep going */ }
+  }
+  logInfo(`resync directive — refreshed ${written} peer(s) into the local cache`);
+  return written;
+}
+
+/**
+ * Refetch + re-verify the signed policy bundle(s) against the PINNED trust root,
+ * in response to a "policy_stale" directive. Quarantines an unverifiable bundle
+ * (policy.fetchAndApply's contract); leaves the last-good cache in place. We use
+ * handlePolicyStale so all currently-cached kinds converge; it delegates to
+ * fetchAndApply per kind. Fail-open.
+ */
+async function refreshPolicy(o) {
+  const { agentRoot, cfg, fetchImpl, now } = o;
+  try {
+    const r = await policy.handlePolicyStale({ cfg, agentRoot, fetchImpl, now });
+    logInfo(`policy_stale directive — refreshed ${r.refreshed} bundle(s)`);
+    return r;
+  } catch (err) {
+    logWarn(`policy_stale directive — refresh failed (${err && err.message})`);
+    return { refreshed: 0, results: [] };
+  }
+}
+
+/**
+ * Refresh the workspace's GRANTED integration tools (SP5) and, when the version
+ * changed, persist them to state/org/toolset.json so per-session cohort-mcp
+ * subprocesses + the native tool snapshot pick up the new set WITHOUT a daemon
+ * restart. Cheap: one `integration.toolsetVersion` poll, refetching the manifest
+ * only on a version change. Fully fail-open — a refresh hiccup must never break
+ * the beat that calls it. Returns the cache (or null on a caught failure).
+ * @param {object} ctx the shared loop context
+ */
+async function refreshIntegrationTools(ctx) {
+  try {
+    const prev = currentToolsetVersion();
+    const call = (method, params) =>
+      ctx.client.call(method, params, {
+        base: ctx.base,
+        token: ctx.token,
+        orgId: ctx.orgId,
+        fetchImpl: ctx.fetchImpl,
+      });
+    const cache = await maybeRefresh({
+      call,
+      agentMemberId: ctx.agentMemberId,
+      now: ctx.now,
+    });
+    if (cache.version !== prev) {
+      writeIntegrationToolsToDisk(ctx.agentRoot, {
+        version: cache.version,
+        tools: cache.tools,
+      });
+      logInfo(
+        `integration tools refreshed → v${cache.version} (${cache.tools.length} tool(s)) — wrote state/org/toolset.json`,
+      );
+    }
+    return cache;
+  } catch (err) {
+    logWarn(`integration refresh failed (${err && err.message}) — keeping last-good tools`);
+    return null;
+  }
+}
+
+/**
+ * Act on one directive. Pure dispatch; each handler is independently fail-open.
+ * @param {string} directive one of KNOWN_DIRECTIVES
+ * @param {object} ctx the shared loop context
+ */
+async function handleDirective(directive, ctx) {
+  switch (directive) {
+    case "halt":
+      tripKillSwitch(ctx.agentRoot, ctx.requestStop, ctx.now);
+      // Once halted, stop beating: the daemon is going down.
+      ctx.stop();
+      return;
+    case "rotate_token":
+      flagRepairNeeded(ctx.agentRoot, ctx.now);
+      return;
+    case "policy_stale":
+      await refreshPolicy(ctx);
+      return;
+    case "resync":
+      await resyncDirectory(ctx);
+      return;
+    case "toolset_changed":
+      // Force an immediate refresh + disk write, ahead of the 30s poll.
+      await refreshIntegrationTools(ctx);
+      return;
+    default:
+      return; // forward-compatible: ignore unknown directives
+  }
+}
+
+/**
+ * Extract the directive list from a presence.beat outcome. Handles both shapes:
+ *   - success res frame: { ok:true, result:{ directives:[...] } }
+ *   - error res frame (e.g. a deactivated token rejected at auth):
+ *       { ok:false, error:{ directive:"halt", ... } }
+ * Returns a de-duped array limited to KNOWN_DIRECTIVES. Never throws.
+ */
+function extractDirectives(frame) {
+  const out = new Set();
+  try {
+    if (frame && frame.ok && frame.result && Array.isArray(frame.result.directives)) {
+      for (const d of frame.result.directives) if (KNOWN_DIRECTIVES.has(d)) out.add(d);
+    }
+    if (frame && frame.ok === false && frame.error && KNOWN_DIRECTIVES.has(frame.error.directive)) {
+      out.add(frame.error.directive);
+    }
+  } catch { /* fail-open */ }
+  return [...out];
+}
+
+/**
+ * Run a single presence-beat: call presence.beat, read the directives, and act.
+ * Exported (via _internals) for direct testing without the interval timing.
+ * Fully fail-open. Never throws.
+ * @param {object} ctx the shared loop context
+ */
+async function beatOnce(ctx) {
+  // Collect the status snapshot first (fail-open: undefined => beat without it).
+  const status = await collectBeatStatus(ctx);
+  // A critical alert escalates locally too (notify the operator), independent of
+  // whether the org is reachable.
+  if (status && hasCritical(status.alerts)) {
+    try { escalateCriticalAlerts(ctx.agentRoot, status.alerts, ctx.now); } catch { /* fail-open */ }
+  }
+
+  let frame;
+  try {
+    const params = { focus: ctx.beatFocus || undefined };
+    if (status) params.status = status; // rides presence.beat params (no protocol change)
+    frame = await ctx.client.presenceBeat(
+      params,
+      { base: ctx.base, token: ctx.token, idempotencyKey: undefined, fetchImpl: ctx.fetchImpl },
+    );
+  } catch (err) {
+    logWarn(`presence beat failed (${err && err.message}) — org may be down, continuing standalone`);
+    return [];
+  }
+  const directives = extractDirectives(frame);
+  for (const d of directives) {
+    try { await handleDirective(d, ctx); } catch { /* each directive is independently fail-open */ }
+    if (ctx.stopped) break; // a halt stopped us mid-list
+  }
+  // Poll the granted integration tools once per beat (SP5): cheap version check,
+  // refetch + disk-write only on a change. Skipped once halted. Fail-open — the
+  // refresh never affects the beat's returned directives. A `toolset_changed`
+  // directive above already forced this ahead of the poll when hq pushes one.
+  if (!ctx.stopped) await refreshIntegrationTools(ctx);
+  return directives;
+}
+
+/**
+ * Join the org mesh at runtime. Idempotent registration at boot + a presence-beat
+ * loop that acts on server directives. Returns a handle with stop().
+ *
+ * No-op (returns an inert handle) when the org integration is disabled. Never
+ * throws — every failure path is swallowed so a mesh problem can't take the
+ * daemon down.
+ *
+ * @param {object} o
+ * @param {string} o.agentRoot                 the agent repo root
+ * @param {object} o.cfg                        the collective/agent config
+ * @param {Function} [o.fetchImpl]             injected fetch (tests/back-compat)
+ * @param {Function} [o.now]                   injected clock () => ms
+ * @param {Function} [o.setInterval]           injected interval scheduler
+ * @param {Function} [o.clearInterval]         injected interval clearer
+ * @param {object}  [o.client]                 injected org client (defaults to lib/org/client)
+ * @param {Function}[o.requestStop]            injected stop signal (defaults to SIGTERM self)
+ * @param {number}  [o.beatMs]                 beat interval (defaults to DEFAULT_BEAT_MS)
+ * @param {Function}[o.collectStatus]         injected status collector (defaults to lib/telemetry/collect)
+ * @returns {Promise<{stop:Function, isEnabled:boolean}>}
+ */
+export async function connectOrgMesh(o = {}) {
+  const client = o.client || defaultClient;
+  const cfg = o.cfg || {};
+
+  // No-op when the org is disabled. Returns an inert handle so the caller's
+  // shutdown path (handle.stop()) is always safe to call.
+  if (!client.isEnabled(cfg)) {
+    return { stop() {}, isEnabled: false };
+  }
+
+  const agentRoot = resolve(o.agentRoot || process.env.AGENT_ROOT || process.env.AGENT_DIR || process.cwd());
+  const now = typeof o.now === "function" ? o.now : Date.now;
+  const setIntervalFn = typeof o.setInterval === "function" ? o.setInterval : setInterval;
+  const clearIntervalFn = typeof o.clearInterval === "function" ? o.clearInterval : clearInterval;
+  const beatMs = Number.isFinite(o.beatMs) && o.beatMs > 0 ? o.beatMs : DEFAULT_BEAT_MS;
+  const requestStop = typeof o.requestStop === "function" ? o.requestStop : defaultRequestStop;
+
+  let conn = {};
+  try { conn = client.configFromAgent(cfg) || {}; } catch { conn = {}; }
+
+  const ctx = {
+    agentRoot,
+    cfg,
+    client,
+    fetchImpl: o.fetchImpl,
+    now,
+    requestStop,
+    base: conn.base,
+    token: conn.token,
+    orgId: conn.orgId,
+    // The seat whose GRANTED tools we fetch (SP5). Set from the registration
+    // entry id below; absent → the workspace-wide granted set (hq's fallback).
+    agentMemberId: null,
+    beatFocus: o.beatFocus || null,
+    collectStatus: typeof o.collectStatus === "function" ? o.collectStatus : defaultCollectStatus,
+    timer: null,
+    stopped: false,
+    stop: null, // set below
+  };
+
+  ctx.stop = function stop() {
+    if (ctx.stopped) return;
+    ctx.stopped = true;
+    if (ctx.timer != null) {
+      try { clearIntervalFn(ctx.timer); } catch { /* fail-open */ }
+      ctx.timer = null;
+    }
+  };
+
+  // 1. Register at boot (idempotent). Best-effort: a failure just means we're not
+  //    in the directory yet; the next presence-beat / org-sync cadence retries.
+  try {
+    const entry = await assembleSelfEntry(agentRoot, {});
+    if (entry && entry.id) {
+      // Fetch THIS seat's granted tools (SP5); falls back to workspace-wide when absent.
+      ctx.agentMemberId = entry.id;
+      await client.register(entry, {
+        base: conn.base,
+        token: conn.token,
+        idempotencyKey: `registry.register:${entry.id}`,
+        fetchImpl: o.fetchImpl,
+      });
+      logInfo(`registered ${entry.id} with the org mesh`);
+    }
+  } catch (err) {
+    logWarn(`registration failed (${err && err.message}) — continuing; presence-beat will retry`);
+  }
+
+  // 2. Presence-beat loop. The first beat fires on the interval (not immediately)
+  //    so boot isn't blocked on a second round-trip; tests drive it explicitly.
+  ctx.timer = setIntervalFn(() => { void beatOnce(ctx); }, beatMs);
+  // Don't let the beat timer keep the process alive on its own (Node only).
+  try { if (ctx.timer && typeof ctx.timer.unref === "function") ctx.timer.unref(); } catch { /* */ }
+
+  return { stop: ctx.stop, isEnabled: true, _ctx: ctx };
+}
+
+export const _internals = {
+  beatOnce,
+  extractDirectives,
+  handleDirective,
+  tripKillSwitch,
+  flagRepairNeeded,
+  resyncDirectory,
+  refreshPolicy,
+  refreshIntegrationTools,
+  collectBeatStatus,
+  escalateCriticalAlerts,
+  KNOWN_DIRECTIVES,
+};
+
+export default { connectOrgMesh, DEFAULT_BEAT_MS, _internals };