@adaptic/maestro 1.7.3 → 1.8.0

package/lib/cadence-bus.mjs

@@ -0,0 +1,625 @@
+/**
+ * Maestro — Cadence Bus
+ *
+ * Local, file-backed event queue that decouples scheduled cadence ticks
+ * (launchd, manual, daemon, init-maestro, upgrade) from the persistent main
+ * Maestro session that actually services them.
+ *
+ * Why this exists
+ * ---------------
+ * Before the bus, every scheduled cadence tick (inbox-processor every 5m,
+ * backlog-executor every 10m, the daily/weekly/quarterly ones, …) was wired
+ * to launch a fresh `claude --print` session via run-trigger.sh. That meant
+ * dozens of full Claude Code spawns per day, each paying full auth/context/
+ * token overhead even when the tick had nothing to do. It also made
+ * emergency-stop, deduplication, rate-limiting, and quota gating ad-hoc.
+ *
+ * After the bus, launchd just enqueues a tiny JSON event onto a directory.
+ * The persistent daemon (maestro-daemon.mjs) consumes events in-process,
+ * handles lightweight scan/classify/route work inline, and only spawns a
+ * sub-session when the work genuinely warrants it.
+ *
+ * Storage layout (all paths relative to AGENT_ROOT)
+ * --------------------------------------------------
+ *   state/cadence-bus/
+ *     inbox/                  — newly enqueued events (one JSON per event)
+ *     claimed/                — events currently being processed
+ *     processed/<YYYY-MM-DD>/ — successful completions, archived by date
+ *     failed/                 — last-known-bad event payloads (transient errors)
+ *     dlq/                    — events that exceeded retry budget (terminal)
+ *     queue.jsonl             — append-only fallback log (every enqueue lands
+ *                               here too, so no event is lost even if inbox/
+ *                               is unavailable on a given enqueue)
+ *     health.json             — heartbeat written by the consumer
+ *
+ * Concurrency model
+ * -----------------
+ * Atomic enqueue:   write to a temp file in the same directory, then
+ *                   rename(tmp, final). rename(2) is atomic on macOS APFS
+ *                   when source and destination live on the same volume.
+ *
+ * Atomic claim:     rename(inbox/<id>.json → claimed/<id>.json). If two
+ *                   consumers race for the same file, exactly one rename
+ *                   succeeds; the loser gets ENOENT and skips the event.
+ *                   No locks, no stale-lock pathology.
+ *
+ * Stale recovery:   on consumer startup (and periodically while running),
+ *                   any file in claimed/ older than STALE_CLAIM_MS is
+ *                   either returned to inbox/ (under the retry budget) or
+ *                   moved to dlq/. This handles crashes mid-processing.
+ *
+ * Event payload schema
+ * --------------------
+ *   {
+ *     "id":            "evt-2026-05-12T15:30:00.123Z-<random>",
+ *     "type":          "cadence_tick" | "manual_tick" | "internal",
+ *     "source":        "launchd" | "manual" | "daemon" | "init-maestro" | "upgrade",
+ *     "ts":            "2026-05-12T15:30:00.123Z",
+ *     "cadence":       "inbox-processor" | "backlog-executor" | "weekly-strategic-memo" | …,
+ *     "workflow":      "continuous/inbox-processor" (optional),
+ *     "correlation_id": "<external id, optional>",
+ *     "priority":      "high" | "normal" | "low" (default "normal"),
+ *     "metadata":      { … free-form, kept for auditing },
+ *     "attempts":      0 (auto-managed by consumer)
+ *   }
+ *
+ * The bus does not interpret `cadence` itself — it only routes by it.
+ * Handler selection and execution live in scripts/daemon/cadence-handlers.mjs
+ * and scripts/daemon/cadence-consumer.mjs.
+ */
+
+import {
+  appendFileSync,
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import { randomBytes } from "node:crypto";
+
+// ---------------------------------------------------------------------------
+// Constants (override-friendly via env for tests)
+// ---------------------------------------------------------------------------
+
+export const BUS_RELATIVE = "state/cadence-bus";
+export const LOGS_RELATIVE = "logs/cadence-bus";
+
+// Bus schema version. If we change the on-disk layout in a backwards-
+// incompatible way, bump this and write a migration. Doctor reads this.
+export const BUS_VERSION = "1";
+
+// Stale claim threshold — claims older than this are recovered to inbox or
+// the dlq. Defaults assume the longest live cadence handler runs well under
+// 30 minutes; tune via env for long-running migrations.
+const DEFAULT_STALE_CLAIM_MS = 30 * 60 * 1000;
+
+// Maximum attempts before an event goes to dlq. Cadence ticks are mostly
+// non-essential; we don't want a single stuck tick to clog the bus.
+const DEFAULT_MAX_ATTEMPTS = 5;
+
+// ---------------------------------------------------------------------------
+// Path resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the agent root. Prefers explicit arg, then AGENT_ROOT env, then
+ * AGENT_DIR env, then process.cwd(). Returns an absolute path.
+ */
+export function resolveAgentRoot(agentRoot) {
+  return resolve(
+    agentRoot ||
+      process.env.AGENT_ROOT ||
+      process.env.AGENT_DIR ||
+      process.cwd()
+  );
+}
+
+/**
+ * Return all bus paths derived from an agent root.
+ */
+export function getBusPaths(agentRoot) {
+  const root = resolveAgentRoot(agentRoot);
+  const base = join(root, BUS_RELATIVE);
+  return {
+    agentRoot: root,
+    base,
+    inbox: join(base, "inbox"),
+    claimed: join(base, "claimed"),
+    processed: join(base, "processed"),
+    failed: join(base, "failed"),
+    dlq: join(base, "dlq"),
+    queueJsonl: join(base, "queue.jsonl"),
+    health: join(base, "health.json"),
+    version: join(base, "VERSION"),
+    logsDir: join(root, LOGS_RELATIVE),
+    emergencyStop: join(root, ".emergency-stop"),
+  };
+}
+
+/**
+ * Idempotently create the cadence-bus directory tree. Safe to call on every
+ * enqueue/consume; no-op once present.
+ */
+export function ensureBusDirs(agentRoot) {
+  const paths = getBusPaths(agentRoot);
+  for (const dir of [paths.inbox, paths.claimed, paths.processed, paths.failed, paths.dlq, paths.logsDir]) {
+    mkdirSync(dir, { recursive: true });
+  }
+  // Touch a version marker so doctor / upgrade can detect a freshly-created
+  // bus and reason about schema migrations later.
+  if (!existsSync(paths.version)) {
+    writeFileSync(paths.version, `${BUS_VERSION}\n`);
+  }
+  return paths;
+}
+
+// ---------------------------------------------------------------------------
+// Event id
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate a unique, sortable event id of the form:
+ *   evt-2026-05-12T15:30:00.123Z-<8 hex>
+ * The ISO prefix makes inbox/ directory listings naturally sorted by time.
+ * The random suffix avoids collisions for events enqueued in the same ms.
+ */
+export function nextEventId(now = new Date()) {
+  const iso = now.toISOString();
+  const rand = randomBytes(4).toString("hex");
+  return `evt-${iso}-${rand}`;
+}
+
+// ---------------------------------------------------------------------------
+// Logging
+// ---------------------------------------------------------------------------
+
+function todayUtc(now = new Date()) {
+  return now.toISOString().slice(0, 10);
+}
+
+/**
+ * Append a structured log line to logs/cadence-bus/<date>.jsonl. Best-effort;
+ * caller never blocks on log failures. Returns true on success, false on a
+ * caught I/O error.
+ */
+export function logBusEvent(agentRoot, entry) {
+  try {
+    const paths = getBusPaths(agentRoot);
+    mkdirSync(paths.logsDir, { recursive: true });
+    const file = join(paths.logsDir, `${todayUtc()}.jsonl`);
+    const line = JSON.stringify({ ts: new Date().toISOString(), ...entry }) + "\n";
+    appendFileSync(file, line);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Atomic write
+// ---------------------------------------------------------------------------
+
+/**
+ * Atomically write JSON to `targetPath`. Writes a sibling `.tmp` file then
+ * renames it. fsync the temp file's directory after rename to flush metadata
+ * (best-effort; on macOS APFS the rename is durable once it returns).
+ */
+function writeJsonAtomic(targetPath, obj) {
+  mkdirSync(dirname(targetPath), { recursive: true });
+  const tmp = `${targetPath}.tmp.${process.pid}.${Date.now()}.${randomBytes(2).toString("hex")}`;
+  writeFileSync(tmp, JSON.stringify(obj, null, 2) + "\n");
+  try {
+    renameSync(tmp, targetPath);
+  } catch (err) {
+    // Clean up the orphan tmp file if rename failed.
+    try { unlinkSync(tmp); } catch { /* ignore */ }
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Enqueue
+// ---------------------------------------------------------------------------
+
+/**
+ * Append a one-line JSON record to the fallback queue.jsonl. Used both as a
+ * tamper-evident audit log of every enqueue and as a last-resort fallback
+ * when writing to inbox/ fails (e.g. disk full, permissions).
+ */
+function appendFallbackQueue(paths, event) {
+  try {
+    mkdirSync(dirname(paths.queueJsonl), { recursive: true });
+    appendFileSync(paths.queueJsonl, JSON.stringify(event) + "\n");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Enqueue a cadence event onto the bus.
+ *
+ * @param {object} input
+ * @param {string} input.cadence  Cadence name (e.g. "inbox-processor"). Required.
+ * @param {string} [input.type]   Event type, default "cadence_tick".
+ * @param {string} [input.source] Originator label, default "manual".
+ * @param {string} [input.workflow]
+ * @param {string} [input.correlation_id]
+ * @param {"high"|"normal"|"low"} [input.priority]
+ * @param {object} [input.metadata]
+ * @param {string} [input.agentRoot]  Override AGENT_ROOT for tests.
+ * @returns {{ id: string, path: string, fallbackOnly: boolean, skipped?: string }}
+ */
+export function enqueueTick(input = {}) {
+  if (!input || typeof input !== "object") {
+    throw new TypeError("enqueueTick: input must be an object");
+  }
+  const cadence = String(input.cadence || "").trim();
+  if (!cadence) {
+    throw new Error("enqueueTick: cadence is required");
+  }
+
+  const paths = ensureBusDirs(input.agentRoot);
+
+  // Emergency-stop short-circuit. The bus still logs the event so we can
+  // see what would have run, but no file lands in inbox/.
+  if (existsSync(paths.emergencyStop)) {
+    const event = buildEvent(input, cadence);
+    appendFallbackQueue(paths, { ...event, _suppressed: "emergency-stop" });
+    logBusEvent(paths.agentRoot, {
+      level: "info",
+      stage: "enqueue_suppressed",
+      cadence,
+      reason: "emergency-stop",
+      id: event.id,
+    });
+    return { id: event.id, path: null, fallbackOnly: true, skipped: "emergency-stop" };
+  }
+
+  const event = buildEvent(input, cadence);
+  const target = join(paths.inbox, `${event.id}.json`);
+
+  // Always write the audit row first. If inbox write fails the event is
+  // still recoverable from queue.jsonl.
+  appendFallbackQueue(paths, event);
+
+  let fallbackOnly = false;
+  try {
+    writeJsonAtomic(target, event);
+  } catch (err) {
+    fallbackOnly = true;
+    logBusEvent(paths.agentRoot, {
+      level: "error",
+      stage: "enqueue_inbox_failed",
+      cadence,
+      id: event.id,
+      error: err.message,
+    });
+  }
+
+  logBusEvent(paths.agentRoot, {
+    level: "info",
+    stage: fallbackOnly ? "enqueued_fallback" : "enqueued",
+    cadence,
+    source: event.source,
+    priority: event.priority,
+    id: event.id,
+  });
+
+  return { id: event.id, path: fallbackOnly ? null : target, fallbackOnly };
+}
+
+function buildEvent(input, cadence) {
+  return {
+    id: input.id || nextEventId(),
+    type: input.type || "cadence_tick",
+    source: input.source || "manual",
+    ts: input.ts || new Date().toISOString(),
+    cadence,
+    workflow: input.workflow || null,
+    correlation_id: input.correlation_id || null,
+    priority: ["high", "normal", "low"].includes(input.priority) ? input.priority : "normal",
+    metadata: input.metadata && typeof input.metadata === "object" ? input.metadata : {},
+    attempts: typeof input.attempts === "number" ? input.attempts : 0,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Consume
+// ---------------------------------------------------------------------------
+
+/**
+ * List inbox event ids in chronological order. Returns ids without `.json`.
+ */
+export function listInbox(agentRoot) {
+  const paths = getBusPaths(agentRoot);
+  if (!existsSync(paths.inbox)) return [];
+  return readdirSync(paths.inbox)
+    .filter((n) => n.endsWith(".json") && !n.endsWith(".tmp"))
+    .sort()
+    .map((n) => n.slice(0, -5));
+}
+
+/**
+ * List currently-claimed event ids. Used by stale-claim recovery.
+ */
+export function listClaimed(agentRoot) {
+  const paths = getBusPaths(agentRoot);
+  if (!existsSync(paths.claimed)) return [];
+  return readdirSync(paths.claimed)
+    .filter((n) => n.endsWith(".json") && !n.endsWith(".tmp"))
+    .sort()
+    .map((n) => n.slice(0, -5));
+}
+
+/**
+ * Atomically claim the oldest inbox event for processing. Returns the parsed
+ * event payload (with absolute paths attached), or null if the inbox was
+ * empty / all candidates lost the race.
+ *
+ * Other consumers racing for the same event will see ENOENT on rename and
+ * naturally move on.
+ */
+export function claimNextTick(agentRoot) {
+  const paths = ensureBusDirs(agentRoot);
+  const ids = listInbox(paths.agentRoot);
+  for (const id of ids) {
+    const src = join(paths.inbox, `${id}.json`);
+    const dst = join(paths.claimed, `${id}.json`);
+    try {
+      renameSync(src, dst);
+    } catch (err) {
+      if (err.code === "ENOENT") continue; // someone else claimed it
+      throw err;
+    }
+    // Load the event from its claimed path. If it's malformed, fail it and
+    // try the next one.
+    let event;
+    try {
+      event = JSON.parse(readFileSync(dst, "utf-8"));
+    } catch (err) {
+      logBusEvent(paths.agentRoot, {
+        level: "error",
+        stage: "claim_parse_failed",
+        id,
+        error: err.message,
+      });
+      failTick(paths.agentRoot, id, `parse-failed: ${err.message}`);
+      continue;
+    }
+    event.attempts = (typeof event.attempts === "number" ? event.attempts : 0) + 1;
+    // Persist the bumped attempt count so a crash mid-handler still reflects
+    // that we tried once.
+    try {
+      writeJsonAtomic(dst, event);
+    } catch {
+      /* best-effort */
+    }
+    logBusEvent(paths.agentRoot, {
+      level: "info",
+      stage: "claimed",
+      id,
+      cadence: event.cadence,
+      attempts: event.attempts,
+    });
+    return { event, claimedPath: dst };
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Lifecycle transitions
+// ---------------------------------------------------------------------------
+
+function moveClaimed(paths, id, destDir, suffix = ".json") {
+  const src = join(paths.claimed, `${id}.json`);
+  mkdirSync(destDir, { recursive: true });
+  const dst = join(destDir, `${id}${suffix}`);
+  if (!existsSync(src)) return null;
+  renameSync(src, dst);
+  return dst;
+}
+
+/**
+ * Mark a claimed event as successfully processed. Moves it to
+ * processed/<YYYY-MM-DD>/<id>.json and stores the handler result alongside.
+ */
+export function completeTick(agentRoot, id, result = {}) {
+  const paths = ensureBusDirs(agentRoot);
+  const date = todayUtc();
+  const dst = moveClaimed(paths, id, join(paths.processed, date));
+  if (!dst) {
+    logBusEvent(paths.agentRoot, { level: "warn", stage: "complete_missing", id });
+    return null;
+  }
+  // Annotate the archived event with the result.
+  try {
+    const event = JSON.parse(readFileSync(dst, "utf-8"));
+    event.result = result;
+    event.completed_at = new Date().toISOString();
+    writeJsonAtomic(dst, event);
+  } catch {
+    /* best-effort */
+  }
+  logBusEvent(paths.agentRoot, {
+    level: "info",
+    stage: "processed",
+    id,
+    cadence: result.cadence || null,
+    decision: result.decision || null,
+    duration_ms: result.duration_ms ?? null,
+  });
+  return dst;
+}
+
+/**
+ * Mark a claimed event as failed. If under the retry budget, returns it to
+ * inbox/ for another attempt; otherwise routes to dlq/.
+ */
+export function failTick(agentRoot, id, errorOrReason, opts = {}) {
+  const paths = ensureBusDirs(agentRoot);
+  const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const srcClaimed = join(paths.claimed, `${id}.json`);
+  let event = null;
+  if (existsSync(srcClaimed)) {
+    try { event = JSON.parse(readFileSync(srcClaimed, "utf-8")); } catch { /* */ }
+  }
+  if (!event) {
+    // Caller already moved or never had a claim; nothing to do.
+    logBusEvent(paths.agentRoot, { level: "warn", stage: "fail_missing", id });
+    return null;
+  }
+  event.last_error = typeof errorOrReason === "string"
+    ? errorOrReason
+    : (errorOrReason?.message || String(errorOrReason));
+  event.failed_at = new Date().toISOString();
+
+  const attempts = typeof event.attempts === "number" ? event.attempts : 0;
+  if (attempts >= maxAttempts || opts.terminal === true) {
+    // Move to dlq/
+    moveClaimed(paths, id, paths.dlq);
+    logBusEvent(paths.agentRoot, {
+      level: "error",
+      stage: "dlq",
+      id,
+      cadence: event.cadence,
+      attempts,
+      reason: event.last_error,
+    });
+    return { destination: "dlq" };
+  }
+
+  // Re-enqueue: write fresh in inbox/, drop the claim.
+  const newTarget = join(paths.inbox, `${id}.json`);
+  writeJsonAtomic(newTarget, event);
+  try { unlinkSync(srcClaimed); } catch { /* */ }
+  logBusEvent(paths.agentRoot, {
+    level: "warn",
+    stage: "retry_requeued",
+    id,
+    cadence: event.cadence,
+    attempts,
+    reason: event.last_error,
+  });
+  return { destination: "inbox" };
+}
+
+// ---------------------------------------------------------------------------
+// Stale claim recovery
+// ---------------------------------------------------------------------------
+
+/**
+ * Sweep claimed/ for entries older than `maxAgeMs`. Each stale claim is
+ * either returned to inbox/ (under the retry budget) or moved to dlq/.
+ * Safe to run on consumer startup *and* periodically while running, because
+ * the consumer is the only writer to claimed/.
+ *
+ * Returns the count of events recovered + count moved to dlq.
+ */
+export function recoverStaleClaims(agentRoot, maxAgeMs = DEFAULT_STALE_CLAIM_MS, now = Date.now()) {
+  const paths = ensureBusDirs(agentRoot);
+  const stats = { recovered: 0, dlq: 0, scanned: 0 };
+  if (!existsSync(paths.claimed)) return stats;
+  for (const name of readdirSync(paths.claimed)) {
+    if (!name.endsWith(".json")) continue;
+    stats.scanned++;
+    const claimedPath = join(paths.claimed, name);
+    let st;
+    try { st = statSync(claimedPath); } catch { continue; }
+    if (now - st.mtimeMs < maxAgeMs) continue;
+
+    const id = name.slice(0, -5);
+    const outcome = failTick(paths.agentRoot, id, "stale-claim-recovery", {
+      maxAttempts: DEFAULT_MAX_ATTEMPTS,
+    });
+    if (outcome?.destination === "dlq") stats.dlq++;
+    else if (outcome?.destination === "inbox") stats.recovered++;
+  }
+  if (stats.scanned > 0) {
+    logBusEvent(paths.agentRoot, {
+      level: "info",
+      stage: "stale_recovery_complete",
+      ...stats,
+    });
+  }
+  return stats;
+}
+
+// ---------------------------------------------------------------------------
+// Heartbeat
+// ---------------------------------------------------------------------------
+
+/**
+ * Write the consumer's heartbeat state. Doctor / healthcheck reads this to
+ * confirm the persistent main session is alive.
+ */
+export function writeHealth(agentRoot, state = {}) {
+  const paths = ensureBusDirs(agentRoot);
+  const payload = {
+    version: BUS_VERSION,
+    ts: new Date().toISOString(),
+    pid: process.pid,
+    ...state,
+  };
+  try {
+    writeJsonAtomic(paths.health, payload);
+  } catch {
+    /* best-effort */
+  }
+  return payload;
+}
+
+/**
+ * Read the consumer's heartbeat state. Returns null if missing or unreadable.
+ */
+export function readHealth(agentRoot) {
+  const paths = getBusPaths(agentRoot);
+  if (!existsSync(paths.health)) return null;
+  try {
+    return JSON.parse(readFileSync(paths.health, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Inspect bus depth — counts of events at each stage. Used by doctor and
+ * tests; cheap enough to call frequently.
+ */
+export function busDepth(agentRoot) {
+  const paths = getBusPaths(agentRoot);
+  const dirCount = (d) => existsSync(d) ? readdirSync(d).filter((n) => n.endsWith(".json")).length : 0;
+  return {
+    inbox: dirCount(paths.inbox),
+    claimed: dirCount(paths.claimed),
+    dlq: dirCount(paths.dlq),
+    failed: dirCount(paths.failed),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CLI usability
+// ---------------------------------------------------------------------------
+
+/**
+ * Touch each bus directory so a fresh checkout/install has the expected
+ * shape on disk. Equivalent to calling ensureBusDirs + readHealth probe
+ * + writing a `.gitkeep` per dir so empty-bus repos still track structure.
+ */
+export function bootstrapBus(agentRoot) {
+  const paths = ensureBusDirs(agentRoot);
+  for (const dir of [paths.inbox, paths.claimed, paths.processed, paths.failed, paths.dlq, paths.logsDir]) {
+    const keep = join(dir, ".gitkeep");
+    if (!existsSync(keep)) {
+      try { closeSync(openSync(keep, "a")); } catch { /* ignore */ }
+    }
+  }
+  return paths;
+}