clawmatrix 0.2.8 → 0.2.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmatrix",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "description": "Decentralized mesh cluster plugin for OpenClaw — inter-gateway communication, model proxy, task handoff, and tool proxy.",
   "type": "module",
   "license": "MIT",

package/src/acp-proxy.ts

@@ -584,7 +584,7 @@ export class AcpProxy {
             let newSession: AcpSession;
             if (resumeAcpSessionId) {
               debug("acp", `Session "${sessionId}" not found, resuming via acpSessionId "${resumeAcpSessionId.slice(0, 8)}..."`);
-              const effectiveCwd = cwd ?? process.cwd();
+              const effectiveCwd = cwd || process.cwd();
               try {
                 newSession = await this.createSessionWithResume(agent, resumeAcpSessionId, effectiveCwd, from);
               } catch (resumeErr) {
@@ -611,7 +611,7 @@ export class AcpProxy {
               });
             }
             if (mode === "oneshot" && task) {
-              this.returnToReusePool(newSession, cwd ?? process.cwd());
+              this.returnToReusePool(newSession, cwd || process.cwd());
             }
           } else {
             // Re-create native session
@@ -671,7 +671,7 @@ export class AcpProxy {
             });
           }
           if (mode === "oneshot" && task) {
-            this.returnToReusePool(session, cwd ?? process.cwd());
+            this.returnToReusePool(session, cwd || process.cwd());
           }
         } else {
           // Create new native OpenClaw session
@@ -778,7 +778,8 @@ export class AcpProxy {
   /** Handle resume session request (receiver side): resume an ACP session and track it. */
   async handleResumeRequest(frame: AcpResumeRequest): Promise<void> {
     const { id, from, payload } = frame;
-    const { agent, acpSessionId, cwd } = payload;
+    const { agent, acpSessionId } = payload;
+    const cwd = payload.cwd || process.cwd();
 
     try {
       // Enforce concurrent session limit
@@ -1089,6 +1090,30 @@ export class AcpProxy {
       }
     } catch { /* no claude projects directory */ }
 
+    // 3. Search Codex sessions (~/.codex/sessions/YYYY/MM/DD/rollout-*-{sessionId}.jsonl)
+    // Reverse sort so we search recent dates first (most likely to match)
+    const codexSessionsDir = join(homedir(), ".codex", "sessions");
+    try {
+      const years = (await readdir(codexSessionsDir)).sort().reverse();
+      for (const year of years) {
+        const monthsDir = join(codexSessionsDir, year);
+        let months: string[];
+        try { months = (await readdir(monthsDir)).sort().reverse(); } catch { continue; }
+        for (const month of months) {
+          const daysDir = join(monthsDir, month);
+          let days: string[];
+          try { days = (await readdir(daysDir)).sort().reverse(); } catch { continue; }
+          for (const day of days) {
+            const dayDir = join(daysDir, day);
+            let files: string[];
+            try { files = await readdir(dayDir); } catch { continue; }
+            const match = files.find((f) => f.endsWith(`-${sessionId}.jsonl`));
+            if (match) return join(dayDir, match);
+          }
+        }
+      }
+    } catch { /* no codex sessions directory */ }
+
     return null;
   }
 
@@ -1313,7 +1338,7 @@ export class AcpProxy {
   // ── Internal: ACP session management (receiver) ────────────────
 
   private async createSession(agent: string, cwd: string | undefined, from: string): Promise<AcpSession> {
-    const effectiveCwd = cwd ?? process.cwd();
+    const effectiveCwd = cwd || process.cwd();
 
     // 1. Try daemon pool first (long-lived process, instant newSession)
     try {
@@ -2256,6 +2281,62 @@ async function fetchSessionListFromDisk(): Promise<AcpSessionInfo[]> {
     // history.jsonl missing or unreadable — skip
   }
 
+  // 3. Codex session history (~/.codex/history.jsonl + session_index.jsonl)
+  try {
+    const codexDir = join(homedir(), ".codex");
+    // Read session index for titles
+    const titleMap = new Map<string, string>();
+    try {
+      const indexContent = await readFileText(join(codexDir, "session_index.jsonl"));
+      for (const line of indexContent.split("\n")) {
+        if (!line.trim()) continue;
+        try {
+          const entry = JSON.parse(line) as { id?: string; thread_name?: string };
+          if (entry.id && entry.thread_name) titleMap.set(entry.id, entry.thread_name);
+        } catch { /* skip */ }
+      }
+    } catch { /* index missing */ }
+
+    const codexHistoryPath = join(codexDir, "history.jsonl");
+    const codexContent = await readFileTail(codexHistoryPath, 512 * 1024);
+    const codexSessions = new Map<string, { title: string; updatedAt: number }>();
+
+    for (const line of codexContent.split("\n")) {
+      if (!line.trim()) continue;
+      try {
+        const entry = JSON.parse(line) as { session_id?: string; ts?: number; text?: string };
+        if (!entry.session_id) continue;
+        const existing = codexSessions.get(entry.session_id);
+        if (!existing) {
+          const title = (entry.text ?? "").replace(/\s+/g, " ").trim().slice(0, 120);
+          codexSessions.set(entry.session_id, {
+            title,
+            updatedAt: entry.ts ? entry.ts * 1000 : 0, // ts is in seconds
+          });
+        } else if (entry.ts && entry.ts * 1000 > existing.updatedAt) {
+          existing.updatedAt = entry.ts * 1000;
+        }
+      } catch { /* skip */ }
+    }
+
+    const codexResults: AcpSessionInfo[] = [];
+    const existingIds2 = new Set(results.map((r) => r.sessionId));
+    for (const [sessionId, info] of codexSessions) {
+      if (existingIds2.has(sessionId)) continue;
+      codexResults.push({
+        sessionId,
+        cwd: "",
+        title: titleMap.get(sessionId) ?? info.title,
+        updatedAt: info.updatedAt ? new Date(info.updatedAt).toISOString() : undefined,
+        agent: "codex",
+      });
+    }
+    codexResults.sort((a, b) => (b.updatedAt ?? "").localeCompare(a.updatedAt ?? ""));
+    results.push(...codexResults);
+  } catch {
+    // Codex history missing or unreadable — skip
+  }
+
   return results;
 }
 
@@ -2444,7 +2525,12 @@ function normalizeTranscriptMessages(lines: string[], limit: number): ChatHistor
   for (const line of lines) {
     try {
       const parsed = JSON.parse(line);
-      const msg = parsed?.message;
+
+      // Support both OpenClaw/Claude format (parsed.message) and Codex format (parsed.payload with type=response_item)
+      let msg: Record<string, unknown> | undefined = parsed?.message;
+      if (!msg && parsed?.type === "response_item" && parsed?.payload?.role) {
+        msg = parsed.payload;
+      }
       if (!msg) continue;
 
       const role = typeof msg.role === "string" ? msg.role : "";

package/src/cluster-service.ts

@@ -16,6 +16,7 @@ import { AcpProxy, readAllSessionStoresFromDisk } from "./acp-proxy.ts";
 import { TerminalManager } from "./terminal.ts";
 import { WebHandler } from "./web.ts";
 import { KnowledgeSync } from "./knowledge-sync.ts";
+import { HealthTracker } from "./health-tracker.ts";
 import { SentinelManager } from "./sentinel-manager.ts";
 import type {
   AnyClusterFrame,
@@ -28,6 +29,7 @@ import type {
   HandoffInputRequired,
   HandoffInput,
   KnowledgeSyncFrame,
+  HealthSyncFrame,
   ModelRequest,
   ModelResponse,
   ModelStreamChunk,
@@ -85,6 +87,7 @@ export class ClusterRuntime {
   readonly acpProxy: AcpProxy | null;
   readonly terminalManager: TerminalManager;
   knowledgeSync: KnowledgeSync | null = null;
+  healthTracker: HealthTracker;
   webHandler: WebHandler | null = null;
   private sentinelManager: SentinelManager | null = null;
   private logger: PluginLogger;
@@ -104,6 +107,10 @@ export class ClusterRuntime {
     const acpEnabled = config.acp?.enabled || (openclawConfig as Record<string, any>).acp?.enabled;
     this.acpProxy = acpEnabled ? new AcpProxy(config, this.peerManager, openclawConfig as Record<string, unknown>, gatewayInfo) : null;
     this.terminalManager = new TerminalManager(config, this.peerManager);
+    this.healthTracker = new HealthTracker({
+      nodeId: config.nodeId,
+      peerManager: this.peerManager,
+    });
   }
 
   start() {
@@ -115,11 +122,15 @@ export class ClusterRuntime {
     this.peerManager.on("peerConnected", (nodeId) => {
       this.logger.info(`[clawmatrix] Peer connected: ${nodeId}`);
       this.refreshDiscoveredModels();
+      this.healthTracker.recordPeerOnline(nodeId, "direct");
+      this.healthTracker.initPeerSync(nodeId);
     });
 
     this.peerManager.on("peerDisconnected", (nodeId) => {
       this.logger.info(`[clawmatrix] Peer disconnected: ${nodeId}`);
       this.refreshDiscoveredModels();
+      this.healthTracker.recordPeerOffline(nodeId);
+      this.healthTracker.removePeerSync(nodeId);
     });
 
     this.peerManager.on("peerCapabilitiesChanged", () => {
@@ -132,6 +143,7 @@ export class ClusterRuntime {
       this.peerManager.setHttpHandler((req, res) => this.webHandler!.handle(req, res));
       // Enable satellite tool routing through WebHandler
       this.toolProxy.setSatelliteHandler(this.webHandler);
+      this.webHandler.setHealthTracker(this.healthTracker);
       this.logger.info(`[clawmatrix] Web dashboard enabled on listen port`);
     }
 
@@ -166,8 +178,9 @@ export class ClusterRuntime {
     }
 
     // Auto-detect ACP agents if ACP is enabled but no agents are explicitly configured
-    if (this.acpProxy && this.config.acp?.enabled && (!this.config.acp.agents || this.config.acp.agents.length === 0)) {
-      AcpProxy.detectAvailableAgents(this.config.acp.commands).then((detected) => {
+    // Check both ClawMatrix and OpenClaw configs (consistent with acpProxy creation above)
+    if (this.acpProxy && (!this.config.acp?.agents || this.config.acp.agents.length === 0)) {
+      AcpProxy.detectAvailableAgents(this.config.acp?.commands).then((detected) => {
         if (detected.length > 0) {
           this.logger.info(`[clawmatrix] Auto-detected ACP agents: ${detected.map((a) => a.id).join(", ")}`);
           this.peerManager.updateAcpAgents(detected);
@@ -177,6 +190,11 @@ export class ClusterRuntime {
       });
     }
 
+    // Health tracker (always enabled, no config gate)
+    this.healthTracker.start().catch((err) => {
+      this.logger.error(`[clawmatrix] Health tracker failed to start: ${err}`);
+    });
+
     // Start subsystems
     this.peerManager.start();
     this.modelProxy.start();
@@ -213,6 +231,7 @@ export class ClusterRuntime {
     // NOTE: intentionally do NOT stop sentinel here.
     // Sentinel must survive gateway shutdown — that's its entire purpose.
     // It will be replaced by killOldSentinel() on next gateway start.
+    await this.healthTracker.stop();
     await this.knowledgeSync?.stop();
     this.webHandler?.destroy();
     this.handoffManager.destroy();
@@ -316,6 +335,9 @@ export class ClusterRuntime {
           this.logger.error(`[clawmatrix] Knowledge sync error: ${err}`);
         });
         break;
+      case "health_sync":
+        this.healthTracker.handleSyncMessage(frame as HealthSyncFrame);
+        break;
       case "acp_req":
         if (this.acpProxy) {
           this.acpProxy.handleRequest(frame as AcpTaskRequest).catch((err) => {

package/src/health-tracker.ts

@@ -0,0 +1,581 @@
+import * as Automerge from "@automerge/automerge";
+import path from "node:path";
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+
+import { debug } from "./debug.ts";
+import type { PeerManager } from "./peer-manager.ts";
+import type { HealthSyncFrame } from "./types.ts";
+
+const TAG = "health";
+
+/** Retention period for health events (default 90 days). */
+const DEFAULT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
+
+/** Compact interval: every 24 hours. */
+const COMPACT_INTERVAL = 24 * 60 * 60 * 1000;
+
+/** Save debounce interval (5 seconds). */
+const SAVE_DEBOUNCE = 5_000;
+
+// ── Document schema ─────────────────────────────────────────────
+
+export interface HealthEvent {
+  ts: number;
+  type: "start" | "stop" | "peer_online" | "peer_offline";
+  peer?: string;
+  via?: string;   // "direct" | "relay"
+  reason?: string; // disconnect reason
+}
+
+interface NodeHealthEntry {
+  events: HealthEvent[];
+  lastUpdated: number;
+}
+
+export interface HealthDoc {
+  nodes: Record<string, NodeHealthEntry>;
+}
+
+// ── Timeline aggregation ────────────────────────────────────────
+
+export type BucketState = "up" | "degraded" | "down" | "unknown";
+
+export interface NodeTimeline {
+  nodeId: string;
+  firstSeen: number;
+  lastSeen: number;
+  buckets: BucketState[];
+  uptimeRatio: number;
+}
+
+export interface AvailabilityResult {
+  range: string;
+  bucketMinutes: number;
+  startTs: number;
+  endTs: number;
+  nodes: NodeTimeline[];
+  /** Time periods when the observing node was down (cannot observe). */
+  gaps: Array<[number, number]>;
+}
+
+// ── HealthTracker ───────────────────────────────────────────────
+
+export interface HealthTrackerOptions {
+  nodeId: string;
+  peerManager: PeerManager;
+  retentionMs?: number;
+  /** Override state directory (for tests). */
+  stateDir?: string;
+}
+
+export class HealthTracker {
+  private doc: Automerge.Doc<HealthDoc>;
+  private syncStates = new Map<string, Automerge.SyncState>();
+  private readonly nodeId: string;
+  private readonly peerManager: PeerManager;
+  private readonly retentionMs: number;
+  private readonly docPath: string;
+  private compactTimer: ReturnType<typeof setInterval> | null = null;
+  private saveTimer: ReturnType<typeof setTimeout> | null = null;
+  private dirty = false;
+
+  constructor(opts: HealthTrackerOptions) {
+    this.nodeId = opts.nodeId;
+    this.peerManager = opts.peerManager;
+    this.retentionMs = opts.retentionMs ?? DEFAULT_RETENTION_MS;
+
+    const stateDir = opts.stateDir ?? path.join(homedir() || tmpdir(), ".openclaw", "clawmatrix");
+    this.docPath = path.join(stateDir, "health.automerge");
+
+    // Initialize empty doc (will be replaced by load if file exists)
+    this.doc = Automerge.init<HealthDoc>();
+    this.doc = Automerge.change(this.doc, (d) => {
+      (d as HealthDoc).nodes = {};
+    });
+  }
+
+  async start() {
+    // Load persisted doc
+    await this.load();
+
+    // Record self start
+    this.recordEvent({ ts: Date.now(), type: "start" });
+
+    // Compact old events on start
+    this.compact();
+
+    // Schedule periodic compact
+    this.compactTimer = setInterval(() => this.compact(), COMPACT_INTERVAL);
+
+    debug(TAG, `health tracker started for node "${this.nodeId}"`);
+  }
+
+  async stop() {
+    // Record self stop
+    this.recordEvent({ ts: Date.now(), type: "stop" });
+
+    if (this.compactTimer) {
+      clearInterval(this.compactTimer);
+      this.compactTimer = null;
+    }
+    if (this.saveTimer) {
+      clearTimeout(this.saveTimer);
+      this.saveTimer = null;
+    }
+
+    // Final save
+    await this.save();
+    debug(TAG, "health tracker stopped");
+  }
+
+  // ── Event recording ─────────────────────────────────────────
+
+  recordEvent(event: HealthEvent) {
+    this.doc = Automerge.change(this.doc, (d) => {
+      if (!d.nodes[this.nodeId]) {
+        d.nodes[this.nodeId] = { events: [], lastUpdated: 0 };
+      }
+      const entry = d.nodes[this.nodeId]!;
+      entry.events.push({ ...event });
+      entry.lastUpdated = Date.now();
+    });
+    this.scheduleSave();
+    this.broadcastSync();
+  }
+
+  recordPeerOnline(peerId: string, via: "direct" | "relay") {
+    this.recordEvent({ ts: Date.now(), type: "peer_online", peer: peerId, via });
+  }
+
+  recordPeerOffline(peerId: string, reason?: string) {
+    this.recordEvent({ ts: Date.now(), type: "peer_offline", peer: peerId, reason });
+  }
+
+  // ── Sync protocol ──────────────────────────────────────────
+
+  /** Handle incoming health_sync frame from a peer. */
+  handleSyncMessage(frame: HealthSyncFrame) {
+    const peerId = frame.from;
+    const message = new Uint8Array(Buffer.from(frame.payload.data, "base64"));
+    const syncKey = peerId;
+
+    try {
+      const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
+      const [newDoc, newSyncState] = Automerge.receiveSyncMessage(this.doc, syncState, message);
+      this.doc = newDoc;
+      this.syncStates.set(syncKey, newSyncState);
+      this.scheduleSave();
+
+      // Send our response
+      this.sendSyncMessage(peerId);
+    } catch (err) {
+      debug(TAG, `error handling sync from ${peerId}: ${err}`);
+    }
+  }
+
+  /** Initiate sync with a peer (called on peer connect). */
+  initPeerSync(peerId: string) {
+    if (peerId === this.nodeId) return;
+    this.syncStates.set(peerId, Automerge.initSyncState());
+    this.sendSyncMessage(peerId);
+  }
+
+  /** Clean up sync state for a disconnected peer. */
+  removePeerSync(peerId: string) {
+    this.syncStates.delete(peerId);
+  }
+
+  private sendSyncMessage(peerId: string) {
+    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
+    const [newSyncState, message] = Automerge.generateSyncMessage(this.doc, syncState);
+    this.syncStates.set(peerId, newSyncState);
+
+    if (!message) return;
+
+    debug(TAG, `sending health sync to ${peerId} (${message.byteLength} bytes)`);
+
+    const frame: HealthSyncFrame = {
+      type: "health_sync",
+      from: this.nodeId,
+      to: peerId,
+      timestamp: Date.now(),
+      payload: {
+        data: Buffer.from(message).toString("base64"),
+      },
+    };
+
+    this.peerManager.router.sendTo(peerId, frame);
+  }
+
+  private broadcastSync() {
+    const peers = this.peerManager.router.getAllPeers();
+    for (const peer of peers) {
+      this.sendSyncMessage(peer.nodeId);
+    }
+  }
+
+  // ── Timeline aggregation ──────────────────────────────────
+
+  /**
+   * Build availability timeline for all known nodes.
+   * @param range - "24h", "7d", or "90d"
+   */
+  getAvailability(range: "24h" | "7d" | "90d" = "24h"): AvailabilityResult {
+    const now = Date.now();
+    let durationMs: number;
+    let bucketMinutes: number;
+
+    switch (range) {
+      case "24h":
+        durationMs = 24 * 60 * 60 * 1000;
+        bucketMinutes = 30;
+        break;
+      case "7d":
+        durationMs = 7 * 24 * 60 * 60 * 1000;
+        bucketMinutes = 60 * 4; // 4-hour buckets
+        break;
+      case "90d":
+        durationMs = 90 * 24 * 60 * 60 * 1000;
+        bucketMinutes = 60 * 24; // 1-day buckets
+        break;
+    }
+
+    const startTs = now - durationMs;
+    const endTs = now;
+    const bucketMs = bucketMinutes * 60 * 1000;
+    const bucketCount = Math.ceil(durationMs / bucketMs);
+
+    // Find observation gaps (periods where THIS node was down)
+    const gaps = this.getObservationGaps(this.nodeId, startTs, endTs);
+
+    // Build timeline for each node (including self)
+    const nodes: NodeTimeline[] = [];
+
+    for (const [nodeId, entry] of Object.entries(this.doc.nodes)) {
+      // For self, use start/stop events to determine uptime
+      // For other nodes, use peer_online/peer_offline from the observing node
+      const timeline = this.buildNodeTimeline(
+        nodeId,
+        entry,
+        startTs,
+        endTs,
+        bucketMs,
+        bucketCount,
+        gaps,
+      );
+      if (timeline) nodes.push(timeline);
+    }
+
+    return {
+      range,
+      bucketMinutes,
+      startTs,
+      endTs,
+      nodes,
+      gaps,
+    };
+  }
+
+  private buildNodeTimeline(
+    nodeId: string,
+    entry: NodeHealthEntry,
+    startTs: number,
+    endTs: number,
+    bucketMs: number,
+    bucketCount: number,
+    observerGaps: Array<[number, number]>,
+  ): NodeTimeline | null {
+    const events = [...entry.events].sort((a, b) => a.ts - b.ts);
+    if (events.length === 0) return null;
+
+    const firstSeen = events[0]!.ts;
+    const lastSeen = entry.lastUpdated;
+
+    // Build online intervals for this node
+    const intervals = this.buildOnlineIntervals(nodeId, events, startTs, endTs);
+
+    // Calculate per-bucket state
+    const buckets: BucketState[] = [];
+    let totalOnline = 0;
+    let totalObservable = 0;
+
+    for (let i = 0; i < bucketCount; i++) {
+      const bStart = startTs + i * bucketMs;
+      const bEnd = Math.min(bStart + bucketMs, endTs);
+
+      // How much of this bucket is observable (subtract observer gaps)
+      const observableMs = this.observableTimeInRange(bStart, bEnd, observerGaps);
+
+      if (observableMs === 0) {
+        buckets.push("unknown");
+        continue;
+      }
+
+      // How much of this bucket the node was online
+      const onlineMs = this.overlapMs(intervals, bStart, bEnd);
+      const ratio = onlineMs / observableMs;
+
+      totalOnline += onlineMs;
+      totalObservable += observableMs;
+
+      if (ratio >= 0.95) buckets.push("up");
+      else if (ratio >= 0.05) buckets.push("degraded");
+      else buckets.push("down");
+    }
+
+    const uptimeRatio = totalObservable > 0 ? totalOnline / totalObservable : 0;
+
+    return { nodeId, firstSeen, lastSeen, buckets, uptimeRatio };
+  }
+
+  /**
+   * Build online intervals for a node.
+   * - For self node: uses start/stop events
+   * - For other nodes: uses peer_online/peer_offline events from all observers
+   */
+  private buildOnlineIntervals(
+    nodeId: string,
+    events: HealthEvent[],
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const intervals: Array<[number, number]> = [];
+
+    if (nodeId === this.nodeId) {
+      // Self: start/stop events define uptime
+      // But we're looking at all nodes' data, so check if this nodeId
+      // has start/stop events (each node writes its own start/stop)
+      return this.buildSelfIntervals(events, startTs, endTs);
+    }
+
+    // For remote nodes: gather peer_online/peer_offline events from all observer nodes
+    // We have the CRDT doc with all nodes' events merged
+    // Look through ALL nodes' events for peer_online/peer_offline referencing this nodeId
+    return this.buildPeerIntervals(nodeId, startTs, endTs);
+  }
+
+  private buildSelfIntervals(
+    events: HealthEvent[],
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const intervals: Array<[number, number]> = [];
+    let onlineSince: number | null = null;
+
+    for (const ev of events) {
+      if (ev.ts < startTs) {
+        // Track state before window
+        if (ev.type === "start") onlineSince = ev.ts;
+        else if (ev.type === "stop") onlineSince = null;
+        continue;
+      }
+      if (ev.ts > endTs) break;
+
+      if (ev.type === "start") {
+        onlineSince = ev.ts;
+      } else if (ev.type === "stop" && onlineSince !== null) {
+        intervals.push([Math.max(onlineSince, startTs), ev.ts]);
+        onlineSince = null;
+      }
+    }
+
+    // If still online at end of window
+    if (onlineSince !== null) {
+      intervals.push([Math.max(onlineSince, startTs), endTs]);
+    }
+
+    return intervals;
+  }
+
+  private buildPeerIntervals(
+    targetNodeId: string,
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const intervals: Array<[number, number]> = [];
+
+    // Collect peer_online/peer_offline events from all observer nodes
+    const relevantEvents: HealthEvent[] = [];
+    for (const [, entry] of Object.entries(this.doc.nodes)) {
+      for (const ev of entry.events) {
+        if (ev.peer === targetNodeId && (ev.type === "peer_online" || ev.type === "peer_offline")) {
+          relevantEvents.push(ev);
+        }
+      }
+    }
+    relevantEvents.sort((a, b) => a.ts - b.ts);
+
+    let onlineSince: number | null = null;
+
+    for (const ev of relevantEvents) {
+      if (ev.ts < startTs) {
+        if (ev.type === "peer_online") onlineSince = ev.ts;
+        else if (ev.type === "peer_offline") onlineSince = null;
+        continue;
+      }
+      if (ev.ts > endTs) break;
+
+      if (ev.type === "peer_online") {
+        if (onlineSince === null) onlineSince = ev.ts;
+      } else if (ev.type === "peer_offline" && onlineSince !== null) {
+        intervals.push([Math.max(onlineSince, startTs), ev.ts]);
+        onlineSince = null;
+      }
+    }
+
+    if (onlineSince !== null) {
+      intervals.push([Math.max(onlineSince, startTs), endTs]);
+    }
+
+    return intervals;
+  }
+
+  /** Get observation gaps: periods when the local node was not running. */
+  private getObservationGaps(
+    nodeId: string,
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const entry = this.doc.nodes[nodeId];
+    if (!entry) return [[startTs, endTs]]; // no data = entire range is a gap
+
+    const selfIntervals = this.buildSelfIntervals(
+      [...entry.events].sort((a, b) => a.ts - b.ts),
+      startTs,
+      endTs,
+    );
+
+    // Gaps are the complement of self intervals within [startTs, endTs]
+    const gaps: Array<[number, number]> = [];
+    let cursor = startTs;
+
+    for (const [start, end] of selfIntervals) {
+      if (start > cursor) {
+        gaps.push([cursor, start]);
+      }
+      cursor = Math.max(cursor, end);
+    }
+
+    if (cursor < endTs) {
+      gaps.push([cursor, endTs]);
+    }
+
+    return gaps;
+  }
+
+  /** Calculate observable time in a range, excluding gaps. */
+  private observableTimeInRange(
+    start: number,
+    end: number,
+    gaps: Array<[number, number]>,
+  ): number {
+    let total = end - start;
+    for (const [gStart, gEnd] of gaps) {
+      const overlapStart = Math.max(start, gStart);
+      const overlapEnd = Math.min(end, gEnd);
+      if (overlapStart < overlapEnd) {
+        total -= overlapEnd - overlapStart;
+      }
+    }
+    return Math.max(0, total);
+  }
+
+  /** Calculate total overlap between intervals and a range. */
+  private overlapMs(intervals: Array<[number, number]>, start: number, end: number): number {
+    let total = 0;
+    for (const [iStart, iEnd] of intervals) {
+      const overlapStart = Math.max(start, iStart);
+      const overlapEnd = Math.min(end, iEnd);
+      if (overlapStart < overlapEnd) {
+        total += overlapEnd - overlapStart;
+      }
+    }
+    return total;
+  }
+
+  // ── Compact ───────────────────────────────────────────────
+
+  private compact() {
+    const cutoff = Date.now() - this.retentionMs;
+    let pruned = 0;
+
+    this.doc = Automerge.change(this.doc, (d) => {
+      for (const [, node] of Object.entries(d.nodes)) {
+        const before = node.events.length;
+        // Keep events newer than cutoff; also keep the last event before cutoff
+        // to preserve state continuity
+        let lastBeforeCutoff = -1;
+        for (let i = 0; i < node.events.length; i++) {
+          if (node.events[i]!.ts < cutoff) lastBeforeCutoff = i;
+        }
+        if (lastBeforeCutoff > 0) {
+          // Remove all events before the last one before cutoff
+          node.events.splice(0, lastBeforeCutoff);
+          pruned += before - node.events.length;
+        }
+      }
+    });
+
+    if (pruned > 0) {
+      debug(TAG, `compacted ${pruned} old events`);
+      // Re-save to discard old ops
+      this.recompact();
+    }
+  }
+
+  /** Re-serialize to discard Automerge op history for removed data. */
+  private recompact() {
+    const bytes = Automerge.save(this.doc);
+    this.doc = Automerge.load<HealthDoc>(bytes);
+    this.scheduleSave();
+  }
+
+  // ── Persistence ───────────────────────────────────────────
+
+  private async load() {
+    try {
+      const data = await readFile(this.docPath);
+      this.doc = Automerge.load<HealthDoc>(new Uint8Array(data));
+      debug(TAG, `loaded health doc from ${this.docPath}`);
+    } catch {
+      debug(TAG, "no existing health doc, starting fresh");
+    }
+  }
+
+  private async save() {
+    try {
+      const data = Automerge.save(this.doc);
+      await mkdir(path.dirname(this.docPath), { recursive: true });
+      await writeFile(this.docPath, Buffer.from(data));
+    } catch (err) {
+      debug(TAG, `failed to save health doc: ${err}`);
+    }
+  }
+
+  private scheduleSave() {
+    this.dirty = true;
+    if (this.saveTimer) return;
+    this.saveTimer = setTimeout(() => {
+      this.saveTimer = null;
+      if (this.dirty) {
+        this.dirty = false;
+        this.save().catch((err) => {
+          debug(TAG, `deferred save error: ${err}`);
+        });
+      }
+    }, SAVE_DEBOUNCE);
+  }
+
+  // ── Public accessors ──────────────────────────────────────
+
+  /** Get all known node IDs (including offline ones). */
+  getKnownNodes(): string[] {
+    return Object.keys(this.doc.nodes);
+  }
+
+  /** Get raw events for a specific node. */
+  getNodeEvents(nodeId: string): HealthEvent[] {
+    return [...(this.doc.nodes[nodeId]?.events ?? [])];
+  }
+}

package/src/model-proxy.ts

@@ -1090,13 +1090,28 @@ export class ModelProxy {
         ?? this.config.models.find((m) => m.id === payload.model)
       : this.config.models.find((m) => m.id === payload.model);
     if (!model) {
+      // Model not available locally — try forwarding to a remote node that has it
+      const originalTarget = frame.to;
+      const exclude = new Set([this.config.nodeId]);
+      if (originalTarget) exclude.add(originalTarget);
+      const alternatives = this.peerManager.router.findNodesForModel(payload.model, exclude);
+      for (const alt of alternatives) {
+        if (this.peerManager.sendTo(alt.nodeId, { ...frame, to: alt.nodeId })) {
+          debug("model_req", `failover: ${originalTarget ?? "local"} → ${alt.nodeId} for "${payload.model}"`);
+          return;
+        }
+      }
+      // No alternative found
+      const hint = originalTarget && originalTarget !== this.config.nodeId
+        ? `Target node "${originalTarget}" is unreachable and no alternative nodes provide model "${payload.model}"`
+        : `Model "${payload.model}" not available locally`;
       this.peerManager.sendTo(from, {
         type: "model_res",
         id,
         from: this.config.nodeId,
         to: from,
         timestamp: Date.now(),
-        payload: { success: false, error: `Model "${payload.model}" not available locally` },
+        payload: { success: false, error: hint },
       } satisfies ModelResponse);
       return;
     }

package/src/peer-manager.ts

@@ -604,8 +604,11 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
   // ── Message handling ───────────────────────────────────────────
   private onFrame(frame: AnyClusterFrame, from: Connection) {
-    // Ignore self-echo: frames with our own nodeId that were relayed back to us
-    if (frame.from === this.config.nodeId) return;
+    // Ignore self-echo: frames with our own nodeId that were relayed back to us.
+    // Exception: frames from same-nodeId satellite connections (Mac/iOS client)
+    // are legitimate requests that must be processed or relayed.
+    const isSatellite = from.remoteNodeId === this.config.nodeId;
+    if (frame.from === this.config.nodeId && !isSatellite) return;
 
     // Validate from field: must be the direct peer or a known node (relayed)
     if (frame.from && frame.from !== from.remoteNodeId && !this.router.getRoute(frame.from)) {
@@ -674,8 +677,26 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     }
 
     if (frame.to && frame.to !== this.config.nodeId) {
-      this.router.tryRelay(frame);
-      return;
+      if (this.router.tryRelay(frame)) return;
+
+      // Relay failed — for model_req, try alternative nodes or fall through to local handling
+      if (frame.type === "model_req") {
+        const modelId = (frame as any).payload?.model;
+        if (modelId) {
+          const exclude = new Set([frame.to, this.config.nodeId]);
+          const alternatives = this.router.findNodesForModel(modelId, exclude);
+          for (const alt of alternatives) {
+            if (this.sendTo(alt.nodeId, { ...frame, to: alt.nodeId })) {
+              debug("peer", `model_req failover: ${frame.to} → ${alt.nodeId}`);
+              return;
+            }
+          }
+        }
+        // No remote alternative — fall through to local handling
+        // (model-proxy will handle locally or send error back)
+      } else {
+        return;
+      }
     }
 
     // Forward to same-nodeId satellite connection (e.g. Mac desktop app) so that

package/src/router.ts

@@ -233,6 +233,31 @@ export class Router {
     return this.routes.get(target);
   }
 
+  /** Find reachable nodes that provide a specific model, sorted by latency.
+   *  Excludes nodes in the `exclude` set. */
+  findNodesForModel(modelId: string, exclude?: Set<string>): RouteEntry[] {
+    const candidates: RouteEntry[] = [];
+    for (const entry of this.routes.values()) {
+      if (exclude?.has(entry.nodeId)) continue;
+      if (!entry.models.some((m) => m.id === modelId)) continue;
+      // Check reachability
+      if (entry.connection?.isOpen) {
+        candidates.push(entry);
+      } else if (entry.reachableVia) {
+        const relay = this.connections.get(entry.reachableVia);
+        if (relay?.isOpen) candidates.push(entry);
+      }
+    }
+    // Sort: direct first, then by latency
+    candidates.sort((a, b) => {
+      const aDirect = a.connection ? 0 : 1;
+      const bDirect = b.connection ? 0 : 1;
+      if (aDirect !== bDirect) return aDirect - bDirect;
+      return a.latencyMs - b.latencyMs;
+    });
+    return candidates;
+  }
+
   // ── Message sending and relay ──────────────────────────────────
   /** Send a frame to a specific node, relaying if necessary. Returns true if sent. */
   sendTo(targetNodeId: string, frame: ClusterFrame | AnyClusterFrame): boolean {

package/src/sentinel.ts

@@ -569,6 +569,28 @@ process.on("exit", (code) => {
   try { process.stderr.write(`[svc ${ts}] Exit code=${code}\n`); } catch { /* ignore */ }
 });
 
+/** Connect to all configured peers (called when gateway dies). */
+function connectAllPeers() {
+  for (const peer of config.peers) {
+    if (!connections.has(peer.nodeId) && !reconnectTimers.has(peer.nodeId)) {
+      connectToPeer(peer);
+    }
+  }
+}
+
+/** Disconnect from all peers (called when gateway recovers). */
+function disconnectAllPeers() {
+  for (const [nodeId, conn] of connections) {
+    conn.close(1000, "gateway recovered");
+    connections.delete(nodeId);
+  }
+  for (const [nodeId, timer] of reconnectTimers) {
+    clearTimeout(timer);
+    reconnectTimers.delete(nodeId);
+  }
+  reconnectAttempts.clear();
+}
+
 /** Periodically check if the gateway process is still alive via kill(pid, 0). */
 function startGatewayHealthCheck() {
   if (healthCheckTimer || !gatewayPid) return;
@@ -581,11 +603,15 @@ function startGatewayHealthCheck() {
         log("Gateway process detected — back online");
         // Release the port so the gateway can reclaim it
         stopListening();
+        // Disconnect from peers — gateway handles mesh connections
+        disconnectAllPeers();
       }
     } catch {
       if (gatewayAlive) {
         gatewayAlive = false;
         log(`Gateway process (pid ${gatewayPid}) gone — entering standalone mode`);
+        // Connect to peers now that gateway is down
+        connectAllPeers();
         // Take over the gateway's listen port
         if (config.listenPort) {
           // Small delay to let the OS release the port from the dead process
@@ -608,11 +634,7 @@ function boot() {
   writePidFile();
   log(`Started (pid ${process.pid}, gateway ${gatewayPid}, nodeId ${sentinelNodeId()}, takeover port ${config.listenPort || "none"})`);
 
-  // Connect to all configured peers
-  for (const peer of config.peers) {
-    connectToPeer(peer);
-  }
-
-  // Note: we do NOT start listening here.
-  // Listening only starts when gateway dies (port takeover mode).
+  // Do NOT connect to peers on boot — gateway handles mesh connections.
+  // Sentinel only connects when gateway dies (standalone mode).
+  // Listening also only starts when gateway dies (port takeover mode).
 }

package/src/types.ts

@@ -411,6 +411,14 @@ export interface KnowledgeSyncFrame extends ClusterFrame {
   };
 }
 
+// ── Health sync ──────────────────────────────────────────────────
+export interface HealthSyncFrame extends ClusterFrame {
+  type: "health_sync";
+  payload: {
+    data: string; // base64-encoded Automerge sync message
+  };
+}
+
 // ── Diagnostic (sentinel) ────────────────────────────────────────
 export interface DiagnosticExec extends ClusterFrame {
   type: "diagnostic_exec";
@@ -831,4 +839,5 @@ export type AnyClusterFrame =
   | TerminalData
   | TerminalResize
   | TerminalCloseRequest
-  | TerminalCloseResponse;
+  | TerminalCloseResponse
+  | HealthSyncFrame;

package/src/web.ts

@@ -3,6 +3,7 @@ import type { PeerManager } from "./peer-manager.ts";
 import type { HandoffManager } from "./handoff.ts";
 import type { ClawMatrixConfig } from "./config.ts";
 import type { SatelliteContext, IngestedEvent } from "./types.ts";
+import type { HealthTracker } from "./health-tracker.ts";
 import { timingSafeEqual } from "./auth.ts";
 import { renderDashboard } from "./web-ui.ts";
 import { readBody } from "./http-utils.ts";
@@ -46,6 +47,7 @@ export class WebHandler {
   private ingestedEvents: IngestedEvent[] = []; // ring buffer for ingested events
   private loginAttempts = new Map<string, { count: number; resetAt: number }>(); // IP → rate limit
   private loginCleanupTimer: ReturnType<typeof setInterval> | null = null;
+  private healthTracker: HealthTracker | null = null;
   private onPeerConnected: (nodeId: string) => void;
   private onPeerDisconnected: (nodeId: string) => void;
 
@@ -91,6 +93,11 @@ export class WebHandler {
     peerManager.on("peerDisconnected", this.onPeerDisconnected);
   }
 
+  /** Set the health tracker for availability API. */
+  setHealthTracker(tracker: HealthTracker) {
+    this.healthTracker = tracker;
+  }
+
   /** Clean up timers and pending requests on shutdown. */
   destroy() {
     // Remove event listeners to prevent post-destroy callbacks
@@ -181,6 +188,11 @@ export class WebHandler {
       return;
     }
 
+    if (path === "/api/availability" && req.method === "GET") {
+      this.handleAvailability(req, res);
+      return;
+    }
+
     if (path === "/api/satellite/poll" && req.method === "GET") {
       this.handleSatellitePoll(req, res);
       return;
@@ -271,6 +283,27 @@ export class WebHandler {
     }
   }
 
+  private handleAvailability(req: IncomingMessage, res: ServerResponse) {
+    if (!this.healthTracker) {
+      res.writeHead(503, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Health tracker not available" }));
+      return;
+    }
+
+    const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
+    const range = (url.searchParams.get("range") ?? "24h") as "24h" | "7d" | "90d";
+
+    if (!["24h", "7d", "90d"].includes(range)) {
+      res.writeHead(400, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Invalid range. Use 24h, 7d, or 90d" }));
+      return;
+    }
+
+    const result = this.healthTracker.getAvailability(range);
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(result));
+  }
+
   private handleStatus(res: ServerResponse) {
     const peers = this.peerManager.router.getAllPeers();
     const localNode = {