clawmatrix 0.4.2 → 0.5.0

package/src/health-tracker.ts

@@ -1,24 +1,19 @@
-import * as Automerge from "@automerge/automerge";
-import path from "node:path";
-import { readFile, writeFile, mkdir } from "node:fs/promises";
-import { homedir, tmpdir } from "node:os";
+/**
+ * Health event tracker — records node lifecycle and peer connectivity events.
+ *
+ * Events are persisted to SQLite via Store and synced across peers via
+ * LogReplicator (sequence-based replication). Replaces the previous
+ * Automerge-based implementation with a lighter append-only log approach.
+ */
 
 import { debug } from "./debug.ts";
-import type { PeerManager } from "./peer-manager.ts";
-import type { HealthSyncFrame } from "./types.ts";
+import type { Store } from "./store.ts";
+import type { LogReplicator } from "./log-replication.ts";
+import type { HealthRow } from "./store.ts";
 
 const TAG = "health";
 
-/** Retention period for health events (default 90 days). */
-const DEFAULT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
-
-/** Compact interval: every 24 hours. */
-const COMPACT_INTERVAL = 24 * 60 * 60 * 1000;
-
-/** Save debounce interval (5 seconds). */
-const SAVE_DEBOUNCE = 5_000;
-
-// ── Document schema ─────────────────────────────────────────────
+// ── Public types (unchanged from previous implementation) ───────
 
 export interface HealthEvent {
   ts: number;
@@ -28,17 +23,6 @@ export interface HealthEvent {
   reason?: string; // disconnect reason
 }
 
-interface NodeHealthEntry {
-  events: HealthEvent[];
-  lastUpdated: number;
-}
-
-export interface HealthDoc {
-  nodes: Record<string, NodeHealthEntry>;
-}
-
-// ── Timeline aggregation ────────────────────────────────────────
-
 export type BucketState = "up" | "degraded" | "down" | "unknown";
 
 export interface NodeTimeline {
@@ -63,100 +47,54 @@ export interface AvailabilityResult {
 
 export interface HealthTrackerOptions {
   nodeId: string;
-  peerManager: PeerManager;
-  retentionMs?: number;
-  /** Override state directory (for tests). */
-  stateDir?: string;
+  store?: Store;
+  logReplicator?: LogReplicator;
 }
 
 export class HealthTracker {
-  private doc: Automerge.Doc<HealthDoc>;
-  private syncStates = new Map<string, Automerge.SyncState>();
   private readonly nodeId: string;
-  private readonly peerManager: PeerManager;
-  private readonly retentionMs: number;
-  private readonly docPath: string;
-  private compactTimer: ReturnType<typeof setInterval> | null = null;
-  private saveTimer: ReturnType<typeof setTimeout> | null = null;
-  private dirty = false;
-  /** Debounce timer for broadcastSync (prevents rapid-fire broadcasts). */
-  private broadcastTimer: ReturnType<typeof setTimeout> | null = null;
-  /** Round counter per peer to detect non-converging sync loops. */
-  private syncRounds = new Map<string, number>();
-  private static readonly MAX_SYNC_ROUNDS = 10;
-  private static readonly BROADCAST_DEBOUNCE = 500; // ms
+  private store: Store | null;
+  private logReplicator: LogReplicator | null;
 
   constructor(opts: HealthTrackerOptions) {
     this.nodeId = opts.nodeId;
-    this.peerManager = opts.peerManager;
-    this.retentionMs = opts.retentionMs ?? DEFAULT_RETENTION_MS;
-
-    const stateDir = opts.stateDir ?? path.join(homedir() || tmpdir(), ".openclaw", "clawmatrix");
-    this.docPath = path.join(stateDir, "health.automerge");
+    this.store = opts.store ?? null;
+    this.logReplicator = opts.logReplicator ?? null;
+  }
 
-    // Initialize empty doc (will be replaced by load if file exists)
-    this.doc = Automerge.init<HealthDoc>();
-    this.doc = Automerge.change(this.doc, (d) => {
-      (d as HealthDoc).nodes = {};
-    });
+  /** Set store after construction (for late initialization). */
+  setStore(store: Store, logReplicator?: LogReplicator) {
+    this.store = store;
+    this.logReplicator = logReplicator ?? this.logReplicator;
   }
 
   async start() {
-    // Load persisted doc
-    await this.load();
-
-    // Record self start
     this.recordEvent({ ts: Date.now(), type: "start" });
-
-    // Compact old events on start
-    this.compact();
-
-    // Schedule periodic compact
-    this.compactTimer = setInterval(() => this.compact(), COMPACT_INTERVAL);
-
     debug(TAG, `health tracker started for node "${this.nodeId}"`);
   }
 
   async stop() {
-    // Record self stop
     this.recordEvent({ ts: Date.now(), type: "stop" });
-
-    if (this.compactTimer) {
-      clearInterval(this.compactTimer);
-      this.compactTimer = null;
-    }
-    if (this.broadcastTimer) {
-      clearTimeout(this.broadcastTimer);
-      this.broadcastTimer = null;
-    }
-    if (this.saveTimer) {
-      clearTimeout(this.saveTimer);
-      this.saveTimer = null;
-    }
-
-    // Final save
-    await this.save();
     debug(TAG, "health tracker stopped");
   }
 
   // ── Event recording ─────────────────────────────────────────
 
   recordEvent(event: HealthEvent) {
-    // Strip undefined values — Automerge rejects them
-    const clean: Record<string, unknown> = {};
-    for (const [k, v] of Object.entries(event)) {
-      if (v !== undefined) clean[k] = v;
+    if (!this.store) return;
+    try {
+      this.store.insertHealth({
+        nodeId: this.nodeId,
+        ts: event.ts,
+        type: event.type,
+        peer: event.peer,
+        via: event.via,
+        reason: event.reason,
+      });
+      this.logReplicator?.notifyLocalInsert("health_events");
+    } catch (err) {
+      debug(TAG, `failed to record event: ${err}`);
     }
-    this.doc = Automerge.change(this.doc, (d) => {
-      if (!d.nodes[this.nodeId]) {
-        d.nodes[this.nodeId] = { events: [], lastUpdated: 0 };
-      }
-      const entry = d.nodes[this.nodeId]!;
-      entry.events.push(clean as HealthEvent);
-      entry.lastUpdated = Date.now();
-    });
-    this.scheduleSave();
-    this.broadcastSync();
   }
 
   recordPeerOnline(peerId: string, via: "direct" | "relay") {
@@ -167,90 +105,10 @@ export class HealthTracker {
     this.recordEvent({ ts: Date.now(), type: "peer_offline", peer: peerId, reason });
   }
 
-  // ── Sync protocol ──────────────────────────────────────────
-
-  /** Handle incoming health_sync frame from a peer. */
-  handleSyncMessage(frame: HealthSyncFrame) {
-    const peerId = frame.from;
-    const message = new Uint8Array(Buffer.from(frame.payload.data, "base64"));
-    const syncKey = peerId;
-
-    // Guard against non-converging sync loops
-    const rounds = (this.syncRounds.get(peerId) ?? 0) + 1;
-    if (rounds > HealthTracker.MAX_SYNC_ROUNDS) {
-      debug(TAG, `sync with ${peerId} exceeded ${HealthTracker.MAX_SYNC_ROUNDS} rounds, resetting`);
-      this.syncStates.set(syncKey, Automerge.initSyncState());
-      this.syncRounds.delete(peerId);
-      return;
-    }
-    this.syncRounds.set(peerId, rounds);
+  // ── Sync lifecycle (no-ops, handled by LogReplicator now) ───
 
-    try {
-      const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
-      const [newDoc, newSyncState] = Automerge.receiveSyncMessage(this.doc, syncState, message);
-      this.doc = newDoc;
-      this.syncStates.set(syncKey, newSyncState);
-      this.scheduleSave();
-
-      // Send our response (only if there's something to send)
-      this.sendSyncMessage(peerId);
-    } catch (err) {
-      debug(TAG, `error handling sync from ${peerId}: ${err}`);
-    }
-  }
-
-  /** Initialize sync state for a peer (called on peer connect).
-   *  Does NOT send a message — the subsequent recordPeerOnline → broadcastSync handles that.
-   *  Sending here would race with broadcastSync and corrupt the sync state. */
-  initPeerSync(peerId: string) {
-    if (peerId === this.nodeId) return;
-    this.syncStates.set(peerId, Automerge.initSyncState());
-  }
-
-  /** Clean up sync state for a disconnected peer. */
-  removePeerSync(peerId: string) {
-    this.syncStates.delete(peerId);
-  }
-
-  private sendSyncMessage(peerId: string) {
-    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
-    const [newSyncState, message] = Automerge.generateSyncMessage(this.doc, syncState);
-    this.syncStates.set(peerId, newSyncState);
-
-    if (!message) {
-      // Sync converged — reset round counter
-      this.syncRounds.delete(peerId);
-      return;
-    }
-
-    debug(TAG, `sending health sync to ${peerId} (${message.byteLength} bytes)`);
-
-    const frame: HealthSyncFrame = {
-      type: "health_sync",
-      from: this.nodeId,
-      to: peerId,
-      timestamp: Date.now(),
-      payload: {
-        data: Buffer.from(message).toString("base64"),
-      },
-    };
-
-    this.peerManager.router.sendTo(peerId, frame);
-  }
-
-  private broadcastSync() {
-    // Debounce: multiple events in quick succession → single broadcast
-    if (this.broadcastTimer) return;
-    this.broadcastTimer = setTimeout(() => {
-      this.broadcastTimer = null;
-      // Reset round counters — new broadcast starts fresh sync cycle
-      this.syncRounds.clear();
-      const peers = this.peerManager.router.getAllPeers();
-      for (const peer of peers) {
-        this.sendSyncMessage(peer.nodeId);
-      }
-    }, HealthTracker.BROADCAST_DEBOUNCE);
-  }
+  initPeerSync(_peerId: string) { /* handled by LogReplicator */ }
+  removePeerSync(_peerId: string) { /* handled by LogReplicator */ }
 
   // ── Timeline aggregation ──────────────────────────────────
 
@@ -270,11 +128,11 @@ export class HealthTracker {
         break;
       case "7d":
         durationMs = 7 * 24 * 60 * 60 * 1000;
-        bucketMinutes = 60 * 4; // 4-hour buckets
+        bucketMinutes = 60 * 4;
         break;
       case "90d":
         durationMs = 90 * 24 * 60 * 60 * 1000;
-        bucketMinutes = 60 * 24; // 1-day buckets
+        bucketMinutes = 60 * 24;
         break;
     }
 
@@ -284,67 +142,52 @@ export class HealthTracker {
     const bucketCount = Math.ceil(durationMs / bucketMs);
 
     // Find observation gaps (periods where THIS node was down)
-    const gaps = this.getObservationGaps(this.nodeId, startTs, endTs);
+    const selfEvents = this.getEventsForNode(this.nodeId);
+    const gaps = this.getObservationGaps(selfEvents, startTs, endTs);
 
     // Build timeline for each node (including self)
     const nodes: NodeTimeline[] = [];
 
-    // Collect nodeIds that have ever had peer_online events (actually connected)
+    // Collect nodeIds that have ever had peer_online events
     const everConnected = new Set<string>();
-    for (const [, entry] of Object.entries(this.doc.nodes)) {
-      for (const ev of entry.events) {
-        if (ev.type === "peer_online" && ev.peer) everConnected.add(ev.peer);
-      }
+    const allEvents = this.store ? this.store.queryHealth({ type: "peer_online" }) : [];
+    for (const ev of allEvents) {
+      if (ev.peer) everConnected.add(ev.peer);
     }
 
-    for (const [nodeId, entry] of Object.entries(this.doc.nodes)) {
-      // Remote nodes: must have successfully connected at some point
+    const knownNodes = this.getKnownNodes();
+    for (const nodeId of knownNodes) {
       if (nodeId !== this.nodeId && !everConnected.has(nodeId)) continue;
 
+      const events = this.getEventsForNode(nodeId);
       const timeline = this.buildNodeTimeline(
-        nodeId,
-        entry,
-        startTs,
-        endTs,
-        bucketMs,
-        bucketCount,
-        gaps,
+        nodeId, events, startTs, endTs, bucketMs, bucketCount, gaps,
       );
       if (!timeline) continue;
-      // Hide remote nodes that were offline for the entire requested range
       if (nodeId !== this.nodeId && timeline.uptimeRatio === 0) continue;
       nodes.push(timeline);
     }
 
-    return {
-      range,
-      bucketMinutes,
-      startTs,
-      endTs,
-      nodes,
-      gaps,
-    };
+    return { range, bucketMinutes, startTs, endTs, nodes, gaps };
   }
 
   private buildNodeTimeline(
     nodeId: string,
-    entry: NodeHealthEntry,
+    events: HealthEvent[],
     startTs: number,
     endTs: number,
     bucketMs: number,
     bucketCount: number,
     observerGaps: Array<[number, number]>,
   ): NodeTimeline | null {
-    const events = [...entry.events].sort((a, b) => a.ts - b.ts);
-    if (events.length === 0) return null;
+    const sorted = [...events].sort((a, b) => a.ts - b.ts);
+    if (sorted.length === 0) return null;
 
-    const firstSeen = events[0]!.ts;
-    const lastSeen = entry.lastUpdated;
+    const firstSeen = sorted[0]!.ts;
+    const lastSeen = sorted[sorted.length - 1]!.ts;
 
-    // Build online intervals for this node
-    const intervals = this.buildOnlineIntervals(nodeId, events, startTs, endTs);
+    const intervals = this.buildOnlineIntervals(nodeId, sorted, startTs, endTs);
 
-    // Calculate per-bucket state
     const buckets: BucketState[] = [];
     let totalOnline = 0;
     let totalObservable = 0;
@@ -353,15 +196,9 @@ export class HealthTracker {
       const bStart = startTs + i * bucketMs;
       const bEnd = Math.min(bStart + bucketMs, endTs);
 
-      // How much of this bucket is observable (subtract observer gaps)
       const observableMs = this.observableTimeInRange(bStart, bEnd, observerGaps);
+      if (observableMs === 0) { buckets.push("unknown"); continue; }
 
-      if (observableMs === 0) {
-        buckets.push("unknown");
-        continue;
-      }
-
-      // How much of this bucket the node was online
       const onlineMs = this.overlapMs(intervals, bStart, bEnd);
       const ratio = onlineMs / observableMs;
 
@@ -374,15 +211,9 @@ export class HealthTracker {
     }
 
     const uptimeRatio = totalObservable > 0 ? totalOnline / totalObservable : 0;
-
     return { nodeId, firstSeen, lastSeen, buckets, uptimeRatio };
   }
 
-  /**
-   * Build online intervals for a node.
-   * - For self node: uses start/stop events
-   * - For other nodes: uses peer_online/peer_offline events from all observers
-   */
   private buildOnlineIntervals(
     nodeId: string,
     events: HealthEvent[],
@@ -390,21 +221,14 @@ export class HealthTracker {
     endTs: number,
   ): Array<[number, number]> {
     if (nodeId === this.nodeId) {
-      // Self: start/stop events define uptime
       return this.buildSelfIntervals(events, startTs, endTs);
     }
-
-    // For remote nodes: use BOTH self-reported start/stop intervals AND
-    // peer_online/peer_offline observations, then merge for best accuracy.
-    // Self-reported intervals are the primary signal (the node knows when
-    // it was running); peer observations supplement for relay peers or
-    // when CRDT sync hasn't propagated the remote node's own events.
+    // For remote nodes: merge self-reported start/stop with peer observations
     const selfIntervals = this.buildSelfIntervals(events, startTs, endTs);
     const peerIntervals = this.buildPeerIntervals(nodeId, startTs, endTs);
     return this.mergeIntervals([...selfIntervals, ...peerIntervals]);
   }
 
-  /** Merge overlapping intervals into a sorted, non-overlapping set. */
   private mergeIntervals(intervals: Array<[number, number]>): Array<[number, number]> {
     if (intervals.length <= 1) return intervals;
     intervals.sort((a, b) => a[0] - b[0]);
@@ -431,7 +255,6 @@ export class HealthTracker {
 
     for (const ev of events) {
       if (ev.ts < startTs) {
-        // Track state before window
         if (ev.type === "start") onlineSince = ev.ts;
         else if (ev.type === "stop") onlineSince = null;
         continue;
@@ -446,11 +269,9 @@ export class HealthTracker {
       }
     }
 
-    // If still online at end of window
     if (onlineSince !== null) {
       intervals.push([Math.max(onlineSince, startTs), endTs]);
     }
-
     return intervals;
   }
 
@@ -459,19 +280,16 @@ export class HealthTracker {
     startTs: number,
     endTs: number,
   ): Array<[number, number]> {
-    const intervals: Array<[number, number]> = [];
+    if (!this.store) return [];
 
-    // Collect peer_online/peer_offline events from all observer nodes
-    const relevantEvents: HealthEvent[] = [];
-    for (const [, entry] of Object.entries(this.doc.nodes)) {
-      for (const ev of entry.events) {
-        if (ev.peer === targetNodeId && (ev.type === "peer_online" || ev.type === "peer_offline")) {
-          relevantEvents.push(ev);
-        }
-      }
-    }
+    // Collect peer_online/peer_offline events about this node from ALL observers
+    const relevantRows = this.store.queryHealth({ peer: targetNodeId });
+    const relevantEvents: HealthEvent[] = relevantRows
+      .filter(r => r.type === "peer_online" || r.type === "peer_offline")
+      .map(r => ({ ts: r.ts, type: r.type as HealthEvent["type"], peer: r.peer ?? undefined }));
     relevantEvents.sort((a, b) => a.ts - b.ts);
 
+    const intervals: Array<[number, number]> = [];
     let onlineSince: number | null = null;
 
     for (const ev of relevantEvents) {
@@ -493,155 +311,72 @@ export class HealthTracker {
     if (onlineSince !== null) {
       intervals.push([Math.max(onlineSince, startTs), endTs]);
     }
-
     return intervals;
   }
 
-  /** Get observation gaps: periods when the local node was not running. */
   private getObservationGaps(
-    nodeId: string,
+    selfEvents: HealthEvent[],
     startTs: number,
     endTs: number,
   ): Array<[number, number]> {
-    const entry = this.doc.nodes[nodeId];
-    if (!entry) return [[startTs, endTs]]; // no data = entire range is a gap
+    if (selfEvents.length === 0) return [[startTs, endTs]];
 
-    const selfIntervals = this.buildSelfIntervals(
-      [...entry.events].sort((a, b) => a.ts - b.ts),
-      startTs,
-      endTs,
-    );
+    const sorted = [...selfEvents].sort((a, b) => a.ts - b.ts);
+    const selfIntervals = this.buildSelfIntervals(sorted, startTs, endTs);
 
-    // Gaps are the complement of self intervals within [startTs, endTs]
     const gaps: Array<[number, number]> = [];
     let cursor = startTs;
 
     for (const [start, end] of selfIntervals) {
-      if (start > cursor) {
-        gaps.push([cursor, start]);
-      }
+      if (start > cursor) gaps.push([cursor, start]);
       cursor = Math.max(cursor, end);
     }
 
-    if (cursor < endTs) {
-      gaps.push([cursor, endTs]);
-    }
-
+    if (cursor < endTs) gaps.push([cursor, endTs]);
     return gaps;
   }
 
-  /** Calculate observable time in a range, excluding gaps. */
-  private observableTimeInRange(
-    start: number,
-    end: number,
-    gaps: Array<[number, number]>,
-  ): number {
+  private observableTimeInRange(start: number, end: number, gaps: Array<[number, number]>): number {
     let total = end - start;
     for (const [gStart, gEnd] of gaps) {
       const overlapStart = Math.max(start, gStart);
       const overlapEnd = Math.min(end, gEnd);
-      if (overlapStart < overlapEnd) {
-        total -= overlapEnd - overlapStart;
-      }
+      if (overlapStart < overlapEnd) total -= overlapEnd - overlapStart;
     }
     return Math.max(0, total);
   }
 
-  /** Calculate total overlap between intervals and a range. */
   private overlapMs(intervals: Array<[number, number]>, start: number, end: number): number {
     let total = 0;
     for (const [iStart, iEnd] of intervals) {
       const overlapStart = Math.max(start, iStart);
       const overlapEnd = Math.min(end, iEnd);
-      if (overlapStart < overlapEnd) {
-        total += overlapEnd - overlapStart;
-      }
+      if (overlapStart < overlapEnd) total += overlapEnd - overlapStart;
     }
     return total;
   }
 
-  // ── Compact ───────────────────────────────────────────────
-
-  private compact() {
-    const cutoff = Date.now() - this.retentionMs;
-    let pruned = 0;
-
-    this.doc = Automerge.change(this.doc, (d) => {
-      for (const [, node] of Object.entries(d.nodes)) {
-        const before = node.events.length;
-        // Keep events newer than cutoff; also keep the last event before cutoff
-        // to preserve state continuity
-        let lastBeforeCutoff = -1;
-        for (let i = 0; i < node.events.length; i++) {
-          if (node.events[i]!.ts < cutoff) lastBeforeCutoff = i;
-        }
-        if (lastBeforeCutoff > 0) {
-          // Remove all events before the last one before cutoff
-          node.events.splice(0, lastBeforeCutoff);
-          pruned += before - node.events.length;
-        }
-      }
-    });
-
-    if (pruned > 0) {
-      debug(TAG, `compacted ${pruned} old events`);
-      // Re-save to discard old ops
-      this.recompact();
-    }
-  }
-
-  /** Re-serialize to discard Automerge op history for removed data. */
-  private recompact() {
-    const bytes = Automerge.save(this.doc);
-    this.doc = Automerge.load<HealthDoc>(bytes);
-    this.scheduleSave();
-  }
-
-  // ── Persistence ───────────────────────────────────────────
-
-  private async load() {
-    try {
-      const data = await readFile(this.docPath);
-      this.doc = Automerge.load<HealthDoc>(new Uint8Array(data));
-      debug(TAG, `loaded health doc from ${this.docPath}`);
-    } catch {
-      debug(TAG, "no existing health doc, starting fresh");
-    }
-  }
-
-  private async save() {
-    try {
-      const data = Automerge.save(this.doc);
-      await mkdir(path.dirname(this.docPath), { recursive: true });
-      await writeFile(this.docPath, Buffer.from(data));
-    } catch (err) {
-      debug(TAG, `failed to save health doc: ${err}`);
-    }
-  }
-
-  private scheduleSave() {
-    this.dirty = true;
-    if (this.saveTimer) return;
-    this.saveTimer = setTimeout(() => {
-      this.saveTimer = null;
-      if (this.dirty) {
-        this.dirty = false;
-        this.save().catch((err) => {
-          debug(TAG, `deferred save error: ${err}`);
-        });
-      }
-    }, SAVE_DEBOUNCE);
-  }
-
   // ── Public accessors ──────────────────────────────────────
 
-  /** Get all known node IDs (including offline ones). */
   getKnownNodes(): string[] {
-    return Object.keys(this.doc.nodes);
+    return this.store?.getHealthNodeIds() ?? [];
   }
 
-  /** Get raw events for a specific node. */
   getNodeEvents(nodeId: string): HealthEvent[] {
-    return [...(this.doc.nodes[nodeId]?.events ?? [])];
+    return this.getEventsForNode(nodeId);
+  }
+
+  private getEventsForNode(nodeId: string): HealthEvent[] {
+    if (!this.store) return [];
+    const rows = this.store.queryHealth({ nodeId });
+    return rows.map(rowToEvent);
   }
 }
+
+function rowToEvent(row: HealthRow): HealthEvent {
+  const event: HealthEvent = { ts: row.ts, type: row.type as HealthEvent["type"] };
+  if (row.peer) event.peer = row.peer;
+  if (row.via) event.via = row.via;
+  if (row.reason) event.reason = row.reason;
+  return event;
+}