clawmatrix 0.2.7 → 0.2.9

package/src/health-tracker.ts

@@ -0,0 +1,581 @@
+import * as Automerge from "@automerge/automerge";
+import path from "node:path";
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+
+import { debug } from "./debug.ts";
+import type { PeerManager } from "./peer-manager.ts";
+import type { HealthSyncFrame } from "./types.ts";
+
+const TAG = "health";
+
+/** Retention period for health events (default 90 days). */
+const DEFAULT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
+
+/** Compact interval: every 24 hours. */
+const COMPACT_INTERVAL = 24 * 60 * 60 * 1000;
+
+/** Save debounce interval (5 seconds). */
+const SAVE_DEBOUNCE = 5_000;
+
+// ── Document schema ─────────────────────────────────────────────
+
+export interface HealthEvent {
+  ts: number;
+  type: "start" | "stop" | "peer_online" | "peer_offline";
+  peer?: string;
+  via?: string;   // "direct" | "relay"
+  reason?: string; // disconnect reason
+}
+
+interface NodeHealthEntry {
+  events: HealthEvent[];
+  lastUpdated: number;
+}
+
+export interface HealthDoc {
+  nodes: Record<string, NodeHealthEntry>;
+}
+
+// ── Timeline aggregation ────────────────────────────────────────
+
+export type BucketState = "up" | "degraded" | "down" | "unknown";
+
+export interface NodeTimeline {
+  nodeId: string;
+  firstSeen: number;
+  lastSeen: number;
+  buckets: BucketState[];
+  uptimeRatio: number;
+}
+
+export interface AvailabilityResult {
+  range: string;
+  bucketMinutes: number;
+  startTs: number;
+  endTs: number;
+  nodes: NodeTimeline[];
+  /** Time periods when the observing node was down (cannot observe). */
+  gaps: Array<[number, number]>;
+}
+
+// ── HealthTracker ───────────────────────────────────────────────
+
+export interface HealthTrackerOptions {
+  nodeId: string;
+  peerManager: PeerManager;
+  retentionMs?: number;
+  /** Override state directory (for tests). */
+  stateDir?: string;
+}
+
+export class HealthTracker {
+  private doc: Automerge.Doc<HealthDoc>;
+  private syncStates = new Map<string, Automerge.SyncState>();
+  private readonly nodeId: string;
+  private readonly peerManager: PeerManager;
+  private readonly retentionMs: number;
+  private readonly docPath: string;
+  private compactTimer: ReturnType<typeof setInterval> | null = null;
+  private saveTimer: ReturnType<typeof setTimeout> | null = null;
+  private dirty = false;
+
+  constructor(opts: HealthTrackerOptions) {
+    this.nodeId = opts.nodeId;
+    this.peerManager = opts.peerManager;
+    this.retentionMs = opts.retentionMs ?? DEFAULT_RETENTION_MS;
+
+    const stateDir = opts.stateDir ?? path.join(homedir() || tmpdir(), ".openclaw", "clawmatrix");
+    this.docPath = path.join(stateDir, "health.automerge");
+
+    // Initialize empty doc (will be replaced by load if file exists)
+    this.doc = Automerge.init<HealthDoc>();
+    this.doc = Automerge.change(this.doc, (d) => {
+      (d as HealthDoc).nodes = {};
+    });
+  }
+
+  async start() {
+    // Load persisted doc
+    await this.load();
+
+    // Record self start
+    this.recordEvent({ ts: Date.now(), type: "start" });
+
+    // Compact old events on start
+    this.compact();
+
+    // Schedule periodic compact
+    this.compactTimer = setInterval(() => this.compact(), COMPACT_INTERVAL);
+
+    debug(TAG, `health tracker started for node "${this.nodeId}"`);
+  }
+
+  async stop() {
+    // Record self stop
+    this.recordEvent({ ts: Date.now(), type: "stop" });
+
+    if (this.compactTimer) {
+      clearInterval(this.compactTimer);
+      this.compactTimer = null;
+    }
+    if (this.saveTimer) {
+      clearTimeout(this.saveTimer);
+      this.saveTimer = null;
+    }
+
+    // Final save
+    await this.save();
+    debug(TAG, "health tracker stopped");
+  }
+
+  // ── Event recording ─────────────────────────────────────────
+
+  recordEvent(event: HealthEvent) {
+    this.doc = Automerge.change(this.doc, (d) => {
+      if (!d.nodes[this.nodeId]) {
+        d.nodes[this.nodeId] = { events: [], lastUpdated: 0 };
+      }
+      const entry = d.nodes[this.nodeId]!;
+      entry.events.push({ ...event });
+      entry.lastUpdated = Date.now();
+    });
+    this.scheduleSave();
+    this.broadcastSync();
+  }
+
+  recordPeerOnline(peerId: string, via: "direct" | "relay") {
+    this.recordEvent({ ts: Date.now(), type: "peer_online", peer: peerId, via });
+  }
+
+  recordPeerOffline(peerId: string, reason?: string) {
+    this.recordEvent({ ts: Date.now(), type: "peer_offline", peer: peerId, reason });
+  }
+
+  // ── Sync protocol ──────────────────────────────────────────
+
+  /** Handle incoming health_sync frame from a peer. */
+  handleSyncMessage(frame: HealthSyncFrame) {
+    const peerId = frame.from;
+    const message = new Uint8Array(Buffer.from(frame.payload.data, "base64"));
+    const syncKey = peerId;
+
+    try {
+      const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
+      const [newDoc, newSyncState] = Automerge.receiveSyncMessage(this.doc, syncState, message);
+      this.doc = newDoc;
+      this.syncStates.set(syncKey, newSyncState);
+      this.scheduleSave();
+
+      // Send our response
+      this.sendSyncMessage(peerId);
+    } catch (err) {
+      debug(TAG, `error handling sync from ${peerId}: ${err}`);
+    }
+  }
+
+  /** Initiate sync with a peer (called on peer connect). */
+  initPeerSync(peerId: string) {
+    if (peerId === this.nodeId) return;
+    this.syncStates.set(peerId, Automerge.initSyncState());
+    this.sendSyncMessage(peerId);
+  }
+
+  /** Clean up sync state for a disconnected peer. */
+  removePeerSync(peerId: string) {
+    this.syncStates.delete(peerId);
+  }
+
+  private sendSyncMessage(peerId: string) {
+    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
+    const [newSyncState, message] = Automerge.generateSyncMessage(this.doc, syncState);
+    this.syncStates.set(peerId, newSyncState);
+
+    if (!message) return;
+
+    debug(TAG, `sending health sync to ${peerId} (${message.byteLength} bytes)`);
+
+    const frame: HealthSyncFrame = {
+      type: "health_sync",
+      from: this.nodeId,
+      to: peerId,
+      timestamp: Date.now(),
+      payload: {
+        data: Buffer.from(message).toString("base64"),
+      },
+    };
+
+    this.peerManager.router.sendTo(peerId, frame);
+  }
+
+  private broadcastSync() {
+    const peers = this.peerManager.router.getAllPeers();
+    for (const peer of peers) {
+      this.sendSyncMessage(peer.nodeId);
+    }
+  }
+
+  // ── Timeline aggregation ──────────────────────────────────
+
+  /**
+   * Build availability timeline for all known nodes.
+   * @param range - "24h", "7d", or "90d"
+   */
+  getAvailability(range: "24h" | "7d" | "90d" = "24h"): AvailabilityResult {
+    const now = Date.now();
+    let durationMs: number;
+    let bucketMinutes: number;
+
+    switch (range) {
+      case "24h":
+        durationMs = 24 * 60 * 60 * 1000;
+        bucketMinutes = 30;
+        break;
+      case "7d":
+        durationMs = 7 * 24 * 60 * 60 * 1000;
+        bucketMinutes = 60 * 4; // 4-hour buckets
+        break;
+      case "90d":
+        durationMs = 90 * 24 * 60 * 60 * 1000;
+        bucketMinutes = 60 * 24; // 1-day buckets
+        break;
+    }
+
+    const startTs = now - durationMs;
+    const endTs = now;
+    const bucketMs = bucketMinutes * 60 * 1000;
+    const bucketCount = Math.ceil(durationMs / bucketMs);
+
+    // Find observation gaps (periods where THIS node was down)
+    const gaps = this.getObservationGaps(this.nodeId, startTs, endTs);
+
+    // Build timeline for each node (including self)
+    const nodes: NodeTimeline[] = [];
+
+    for (const [nodeId, entry] of Object.entries(this.doc.nodes)) {
+      // For self, use start/stop events to determine uptime
+      // For other nodes, use peer_online/peer_offline from the observing node
+      const timeline = this.buildNodeTimeline(
+        nodeId,
+        entry,
+        startTs,
+        endTs,
+        bucketMs,
+        bucketCount,
+        gaps,
+      );
+      if (timeline) nodes.push(timeline);
+    }
+
+    return {
+      range,
+      bucketMinutes,
+      startTs,
+      endTs,
+      nodes,
+      gaps,
+    };
+  }
+
+  private buildNodeTimeline(
+    nodeId: string,
+    entry: NodeHealthEntry,
+    startTs: number,
+    endTs: number,
+    bucketMs: number,
+    bucketCount: number,
+    observerGaps: Array<[number, number]>,
+  ): NodeTimeline | null {
+    const events = [...entry.events].sort((a, b) => a.ts - b.ts);
+    if (events.length === 0) return null;
+
+    const firstSeen = events[0]!.ts;
+    const lastSeen = entry.lastUpdated;
+
+    // Build online intervals for this node
+    const intervals = this.buildOnlineIntervals(nodeId, events, startTs, endTs);
+
+    // Calculate per-bucket state
+    const buckets: BucketState[] = [];
+    let totalOnline = 0;
+    let totalObservable = 0;
+
+    for (let i = 0; i < bucketCount; i++) {
+      const bStart = startTs + i * bucketMs;
+      const bEnd = Math.min(bStart + bucketMs, endTs);
+
+      // How much of this bucket is observable (subtract observer gaps)
+      const observableMs = this.observableTimeInRange(bStart, bEnd, observerGaps);
+
+      if (observableMs === 0) {
+        buckets.push("unknown");
+        continue;
+      }
+
+      // How much of this bucket the node was online
+      const onlineMs = this.overlapMs(intervals, bStart, bEnd);
+      const ratio = onlineMs / observableMs;
+
+      totalOnline += onlineMs;
+      totalObservable += observableMs;
+
+      if (ratio >= 0.95) buckets.push("up");
+      else if (ratio >= 0.05) buckets.push("degraded");
+      else buckets.push("down");
+    }
+
+    const uptimeRatio = totalObservable > 0 ? totalOnline / totalObservable : 0;
+
+    return { nodeId, firstSeen, lastSeen, buckets, uptimeRatio };
+  }
+
+  /**
+   * Build online intervals for a node.
+   * - For self node: uses start/stop events
+   * - For other nodes: uses peer_online/peer_offline events from all observers
+   */
+  private buildOnlineIntervals(
+    nodeId: string,
+    events: HealthEvent[],
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const intervals: Array<[number, number]> = [];
+
+    if (nodeId === this.nodeId) {
+      // Self: start/stop events define uptime
+      // But we're looking at all nodes' data, so check if this nodeId
+      // has start/stop events (each node writes its own start/stop)
+      return this.buildSelfIntervals(events, startTs, endTs);
+    }
+
+    // For remote nodes: gather peer_online/peer_offline events from all observer nodes
+    // We have the CRDT doc with all nodes' events merged
+    // Look through ALL nodes' events for peer_online/peer_offline referencing this nodeId
+    return this.buildPeerIntervals(nodeId, startTs, endTs);
+  }
+
+  private buildSelfIntervals(
+    events: HealthEvent[],
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const intervals: Array<[number, number]> = [];
+    let onlineSince: number | null = null;
+
+    for (const ev of events) {
+      if (ev.ts < startTs) {
+        // Track state before window
+        if (ev.type === "start") onlineSince = ev.ts;
+        else if (ev.type === "stop") onlineSince = null;
+        continue;
+      }
+      if (ev.ts > endTs) break;
+
+      if (ev.type === "start") {
+        onlineSince = ev.ts;
+      } else if (ev.type === "stop" && onlineSince !== null) {
+        intervals.push([Math.max(onlineSince, startTs), ev.ts]);
+        onlineSince = null;
+      }
+    }
+
+    // If still online at end of window
+    if (onlineSince !== null) {
+      intervals.push([Math.max(onlineSince, startTs), endTs]);
+    }
+
+    return intervals;
+  }
+
+  private buildPeerIntervals(
+    targetNodeId: string,
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const intervals: Array<[number, number]> = [];
+
+    // Collect peer_online/peer_offline events from all observer nodes
+    const relevantEvents: HealthEvent[] = [];
+    for (const [, entry] of Object.entries(this.doc.nodes)) {
+      for (const ev of entry.events) {
+        if (ev.peer === targetNodeId && (ev.type === "peer_online" || ev.type === "peer_offline")) {
+          relevantEvents.push(ev);
+        }
+      }
+    }
+    relevantEvents.sort((a, b) => a.ts - b.ts);
+
+    let onlineSince: number | null = null;
+
+    for (const ev of relevantEvents) {
+      if (ev.ts < startTs) {
+        if (ev.type === "peer_online") onlineSince = ev.ts;
+        else if (ev.type === "peer_offline") onlineSince = null;
+        continue;
+      }
+      if (ev.ts > endTs) break;
+
+      if (ev.type === "peer_online") {
+        if (onlineSince === null) onlineSince = ev.ts;
+      } else if (ev.type === "peer_offline" && onlineSince !== null) {
+        intervals.push([Math.max(onlineSince, startTs), ev.ts]);
+        onlineSince = null;
+      }
+    }
+
+    if (onlineSince !== null) {
+      intervals.push([Math.max(onlineSince, startTs), endTs]);
+    }
+
+    return intervals;
+  }
+
+  /** Get observation gaps: periods when the local node was not running. */
+  private getObservationGaps(
+    nodeId: string,
+    startTs: number,
+    endTs: number,
+  ): Array<[number, number]> {
+    const entry = this.doc.nodes[nodeId];
+    if (!entry) return [[startTs, endTs]]; // no data = entire range is a gap
+
+    const selfIntervals = this.buildSelfIntervals(
+      [...entry.events].sort((a, b) => a.ts - b.ts),
+      startTs,
+      endTs,
+    );
+
+    // Gaps are the complement of self intervals within [startTs, endTs]
+    const gaps: Array<[number, number]> = [];
+    let cursor = startTs;
+
+    for (const [start, end] of selfIntervals) {
+      if (start > cursor) {
+        gaps.push([cursor, start]);
+      }
+      cursor = Math.max(cursor, end);
+    }
+
+    if (cursor < endTs) {
+      gaps.push([cursor, endTs]);
+    }
+
+    return gaps;
+  }
+
+  /** Calculate observable time in a range, excluding gaps. */
+  private observableTimeInRange(
+    start: number,
+    end: number,
+    gaps: Array<[number, number]>,
+  ): number {
+    let total = end - start;
+    for (const [gStart, gEnd] of gaps) {
+      const overlapStart = Math.max(start, gStart);
+      const overlapEnd = Math.min(end, gEnd);
+      if (overlapStart < overlapEnd) {
+        total -= overlapEnd - overlapStart;
+      }
+    }
+    return Math.max(0, total);
+  }
+
+  /** Calculate total overlap between intervals and a range. */
+  private overlapMs(intervals: Array<[number, number]>, start: number, end: number): number {
+    let total = 0;
+    for (const [iStart, iEnd] of intervals) {
+      const overlapStart = Math.max(start, iStart);
+      const overlapEnd = Math.min(end, iEnd);
+      if (overlapStart < overlapEnd) {
+        total += overlapEnd - overlapStart;
+      }
+    }
+    return total;
+  }
+
+  // ── Compact ───────────────────────────────────────────────
+
+  private compact() {
+    const cutoff = Date.now() - this.retentionMs;
+    let pruned = 0;
+
+    this.doc = Automerge.change(this.doc, (d) => {
+      for (const [, node] of Object.entries(d.nodes)) {
+        const before = node.events.length;
+        // Keep events newer than cutoff; also keep the last event before cutoff
+        // to preserve state continuity
+        let lastBeforeCutoff = -1;
+        for (let i = 0; i < node.events.length; i++) {
+          if (node.events[i]!.ts < cutoff) lastBeforeCutoff = i;
+        }
+        if (lastBeforeCutoff > 0) {
+          // Remove all events before the last one before cutoff
+          node.events.splice(0, lastBeforeCutoff);
+          pruned += before - node.events.length;
+        }
+      }
+    });
+
+    if (pruned > 0) {
+      debug(TAG, `compacted ${pruned} old events`);
+      // Re-save to discard old ops
+      this.recompact();
+    }
+  }
+
+  /** Re-serialize to discard Automerge op history for removed data. */
+  private recompact() {
+    const bytes = Automerge.save(this.doc);
+    this.doc = Automerge.load<HealthDoc>(bytes);
+    this.scheduleSave();
+  }
+
+  // ── Persistence ───────────────────────────────────────────
+
+  private async load() {
+    try {
+      const data = await readFile(this.docPath);
+      this.doc = Automerge.load<HealthDoc>(new Uint8Array(data));
+      debug(TAG, `loaded health doc from ${this.docPath}`);
+    } catch {
+      debug(TAG, "no existing health doc, starting fresh");
+    }
+  }
+
+  private async save() {
+    try {
+      const data = Automerge.save(this.doc);
+      await mkdir(path.dirname(this.docPath), { recursive: true });
+      await writeFile(this.docPath, Buffer.from(data));
+    } catch (err) {
+      debug(TAG, `failed to save health doc: ${err}`);
+    }
+  }
+
+  private scheduleSave() {
+    this.dirty = true;
+    if (this.saveTimer) return;
+    this.saveTimer = setTimeout(() => {
+      this.saveTimer = null;
+      if (this.dirty) {
+        this.dirty = false;
+        this.save().catch((err) => {
+          debug(TAG, `deferred save error: ${err}`);
+        });
+      }
+    }, SAVE_DEBOUNCE);
+  }
+
+  // ── Public accessors ──────────────────────────────────────
+
+  /** Get all known node IDs (including offline ones). */
+  getKnownNodes(): string[] {
+    return Object.keys(this.doc.nodes);
+  }
+
+  /** Get raw events for a specific node. */
+  getNodeEvents(nodeId: string): HealthEvent[] {
+    return [...(this.doc.nodes[nodeId]?.events ?? [])];
+  }
+}

package/src/model-proxy.ts

@@ -1090,13 +1090,28 @@ export class ModelProxy {
         ?? this.config.models.find((m) => m.id === payload.model)
       : this.config.models.find((m) => m.id === payload.model);
     if (!model) {
+      // Model not available locally — try forwarding to a remote node that has it
+      const originalTarget = frame.to;
+      const exclude = new Set([this.config.nodeId]);
+      if (originalTarget) exclude.add(originalTarget);
+      const alternatives = this.peerManager.router.findNodesForModel(payload.model, exclude);
+      for (const alt of alternatives) {
+        if (this.peerManager.sendTo(alt.nodeId, { ...frame, to: alt.nodeId })) {
+          debug("model_req", `failover: ${originalTarget ?? "local"} → ${alt.nodeId} for "${payload.model}"`);
+          return;
+        }
+      }
+      // No alternative found
+      const hint = originalTarget && originalTarget !== this.config.nodeId
+        ? `Target node "${originalTarget}" is unreachable and no alternative nodes provide model "${payload.model}"`
+        : `Model "${payload.model}" not available locally`;
       this.peerManager.sendTo(from, {
         type: "model_res",
         id,
         from: this.config.nodeId,
         to: from,
         timestamp: Date.now(),
-        payload: { success: false, error: `Model "${payload.model}" not available locally` },
+        payload: { success: false, error: hint },
       } satisfies ModelResponse);
       return;
     }