@sesamespace/hivemind 0.2.0 → 0.3.0

package/packages/runtime/src/fleet/memory-sync.ts

@@ -0,0 +1,362 @@
+/**
+ * Cross-machine memory sync.
+ *
+ * Workers periodically push key L3 knowledge back to the Primary.
+ * Primary can push Global context updates (L3 + L2 episodes) to workers.
+ *
+ * Conflict resolution:
+ *   - L3: last-write-wins (compared by updated_at timestamp)
+ *   - L2: append-only (episodes are never overwritten, only added)
+ *
+ * Sync protocol:
+ *   POST /sync/pull  — Worker sends L3 knowledge to Primary
+ *   POST /sync/push  — Primary sends Global context updates to Worker
+ */
+
+import type { MemoryClient, L3Entry, Episode } from "../memory-client.js";
+import type { PrimaryClient } from "./primary-client.js";
+import type { WorkerServer } from "./worker-server.js";
+import type {
+  SyncPullRequest,
+  SyncPullResponse,
+  SyncPushRequest,
+  SyncPushResponse,
+  SyncL3Entry,
+  SyncL2Episode,
+} from "./worker-protocol.js";
+import { DEFAULT_SYNC_INTERVAL_MS, HEALTH_TIMEOUT_MS, PRIMARY_ROUTES } from "./worker-protocol.js";
+
+// --- Helpers ---
+
+function l3ToSync(entry: L3Entry): SyncL3Entry {
+  return {
+    id: entry.id,
+    source_episode_id: entry.source_episode_id,
+    context_name: entry.context_name,
+    content: entry.content,
+    promoted_at: entry.promoted_at,
+    access_count: entry.access_count,
+    connection_density: entry.connection_density,
+    updated_at: entry.promoted_at,
+  };
+}
+
+function episodeToSync(ep: Episode): SyncL2Episode {
+  return {
+    id: ep.id,
+    timestamp: ep.timestamp,
+    context_name: ep.context_name,
+    role: ep.role,
+    content: ep.content,
+  };
+}
+
+// --- MemorySync (Worker side) ---
+
+export interface WorkerSyncOptions {
+  workerId: string;
+  primaryUrl: string;
+  memory: MemoryClient;
+  server: WorkerServer;
+  syncIntervalMs?: number;
+}
+
+/**
+ * Worker-side memory sync. Periodically sends L3 knowledge from assigned
+ * contexts back to the Primary, and accepts Global context pushes.
+ */
+export class WorkerMemorySync {
+  private workerId: string;
+  private primaryUrl: string;
+  private memory: MemoryClient;
+  private syncTimer: ReturnType<typeof setInterval> | null = null;
+  private syncIntervalMs: number;
+  private knownL2Ids: Set<string> = new Set();
+  private knownL3Ids: Map<string, string> = new Map(); // id -> updated_at
+
+  constructor(opts: WorkerSyncOptions) {
+    this.workerId = opts.workerId;
+    this.primaryUrl = opts.primaryUrl;
+    this.memory = opts.memory;
+    this.syncIntervalMs = opts.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS;
+
+    // Register handler for incoming Global context pushes from Primary
+    opts.server.onSyncPush((req) => this.handlePush(req));
+  }
+
+  /** Start periodic sync (Worker -> Primary). */
+  start(contextNames: () => string[]): void {
+    this.stop();
+    this.syncTimer = setInterval(() => {
+      this.pullTowardsPrimary(contextNames()).catch((err) => {
+        console.warn("[sync] Pull cycle failed:", (err as Error).message);
+      });
+    }, this.syncIntervalMs);
+  }
+
+  /** Stop periodic sync. */
+  stop(): void {
+    if (this.syncTimer) {
+      clearInterval(this.syncTimer);
+      this.syncTimer = null;
+    }
+  }
+
+  /**
+   * Pull L3 knowledge from all assigned contexts and send to Primary.
+   * Called periodically by the sync timer.
+   */
+  async pullTowardsPrimary(contextNames: string[]): Promise<void> {
+    for (const contextName of contextNames) {
+      try {
+        const entries = await this.memory.getL3Knowledge(contextName);
+        if (entries.length === 0) continue;
+
+        const syncEntries = entries.map(l3ToSync);
+
+        const req: SyncPullRequest = {
+          worker_id: this.workerId,
+          context_name: contextName,
+          entries: syncEntries,
+        };
+
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS);
+
+        const resp = await fetch(`${this.primaryUrl}${PRIMARY_ROUTES.syncPull}`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(req),
+          signal: controller.signal,
+        });
+        clearTimeout(timeout);
+
+        if (resp.ok) {
+          const result = (await resp.json()) as SyncPullResponse;
+          if (result.accepted > 0) {
+            console.log(`[sync] Pushed ${result.accepted} L3 entries from "${contextName}" to Primary`);
+          }
+        }
+      } catch {
+        // Primary unreachable — will retry next interval
+      }
+    }
+  }
+
+  /**
+   * Handle an incoming push of Global context updates from the Primary.
+   * L3: last-write-wins. L2: append-only.
+   */
+  async handlePush(req: SyncPushRequest): Promise<SyncPushResponse> {
+    let l3Accepted = 0;
+    let l2Appended = 0;
+
+    // Process L3 entries with last-write-wins
+    for (const entry of req.entries) {
+      const existing = this.knownL3Ids.get(entry.id);
+      if (existing && existing >= entry.updated_at) {
+        // Local version is same or newer — skip
+        continue;
+      }
+
+      // Store in local memory daemon (the memory daemon's L3 store handles upsert)
+      try {
+        // Store as an episode in the global context so the worker has access
+        await this.memory.storeEpisode({
+          context_name: entry.context_name,
+          role: "system",
+          content: entry.content,
+        });
+        this.knownL3Ids.set(entry.id, entry.updated_at);
+        l3Accepted++;
+      } catch {
+        // Non-fatal — entry will be retried on next push
+      }
+    }
+
+    // Process L2 episodes as append-only
+    for (const episode of req.episodes) {
+      if (this.knownL2Ids.has(episode.id)) {
+        continue; // Already stored
+      }
+
+      try {
+        await this.memory.storeEpisode({
+          context_name: episode.context_name,
+          role: episode.role,
+          content: episode.content,
+        });
+        this.knownL2Ids.add(episode.id);
+        l2Appended++;
+      } catch {
+        // Non-fatal
+      }
+    }
+
+    if (l3Accepted > 0 || l2Appended > 0) {
+      console.log(`[sync] Received push: ${l3Accepted} L3 accepted, ${l2Appended} L2 appended`);
+    }
+
+    return { l3_accepted: l3Accepted, l2_appended: l2Appended };
+  }
+
+  getSyncIntervalMs(): number {
+    return this.syncIntervalMs;
+  }
+}
+
+// --- PrimaryMemorySync ---
+
+export interface PrimarySyncOptions {
+  primary: PrimaryClient;
+  memory: MemoryClient;
+  syncIntervalMs?: number;
+}
+
+/**
+ * Primary-side memory sync. Handles incoming L3 pulls from workers and
+ * can push Global context updates out to all workers.
+ */
+export class PrimaryMemorySync {
+  private primary: PrimaryClient;
+  private memory: MemoryClient;
+  private syncIntervalMs: number;
+  private pushTimer: ReturnType<typeof setInterval> | null = null;
+  private l3Store: Map<string, SyncL3Entry> = new Map(); // id -> latest entry
+  private lastPushAt: string = new Date(0).toISOString();
+
+  constructor(opts: PrimarySyncOptions) {
+    this.primary = opts.primary;
+    this.memory = opts.memory;
+    this.syncIntervalMs = opts.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS;
+
+    // Register handler for incoming L3 pulls from workers
+    this.primary.onSyncPull((req) => this.handlePull(req));
+  }
+
+  /**
+   * Handle an incoming pull — Worker sending L3 knowledge to Primary.
+   * Uses last-write-wins for conflict resolution.
+   */
+  async handlePull(req: SyncPullRequest): Promise<SyncPullResponse> {
+    let accepted = 0;
+    let rejected = 0;
+
+    for (const entry of req.entries) {
+      const existing = this.l3Store.get(entry.id);
+
+      if (existing && existing.updated_at >= entry.updated_at) {
+        // Primary has same or newer version — reject
+        rejected++;
+        continue;
+      }
+
+      // Accept the entry — store locally
+      this.l3Store.set(entry.id, entry);
+
+      // Persist to Primary's memory daemon
+      try {
+        await this.memory.storeEpisode({
+          context_name: entry.context_name,
+          role: "system",
+          content: entry.content,
+        });
+      } catch {
+        // Non-fatal — the in-memory store is the source of truth for sync
+      }
+
+      accepted++;
+    }
+
+    if (accepted > 0) {
+      console.log(
+        `[sync] Accepted ${accepted} L3 entries from worker "${req.worker_id}" context "${req.context_name}"`,
+      );
+    }
+
+    return { accepted, rejected };
+  }
+
+  /**
+   * Push Global context updates to all workers.
+   * Fetches current Global L3 knowledge and recent L2 episodes,
+   * then pushes to every registered worker.
+   */
+  async pushGlobalToAll(): Promise<Map<string, SyncPushResponse | null>> {
+    // Gather Global context L3 knowledge
+    let l3Entries: SyncL3Entry[] = [];
+    try {
+      const knowledge = await this.memory.getL3Knowledge("global");
+      l3Entries = knowledge.map(l3ToSync);
+    } catch {
+      // Memory daemon may not have global context yet
+    }
+
+    // Gather recent Global L2 episodes (append-only)
+    let l2Episodes: SyncL2Episode[] = [];
+    try {
+      const episodes = await this.memory.getContext("global");
+      // Only send episodes newer than last push
+      l2Episodes = episodes
+        .filter((ep) => ep.timestamp > this.lastPushAt)
+        .map(episodeToSync);
+    } catch {
+      // Memory daemon may not have global context yet
+    }
+
+    if (l3Entries.length === 0 && l2Episodes.length === 0) {
+      return new Map();
+    }
+
+    const payload: SyncPushRequest = {
+      entries: l3Entries,
+      episodes: l2Episodes,
+    };
+
+    this.lastPushAt = new Date().toISOString();
+
+    const results = await this.primary.pushSyncToAll(payload);
+
+    let totalL3 = 0;
+    let totalL2 = 0;
+    for (const resp of results.values()) {
+      if (resp) {
+        totalL3 += resp.l3_accepted;
+        totalL2 += resp.l2_appended;
+      }
+    }
+
+    if (totalL3 > 0 || totalL2 > 0) {
+      console.log(`[sync] Pushed Global updates: ${totalL3} L3, ${totalL2} L2 across ${results.size} workers`);
+    }
+
+    return results;
+  }
+
+  /** Start periodic Global push to all workers. */
+  startPushLoop(): void {
+    this.stopPushLoop();
+    this.pushTimer = setInterval(() => {
+      this.pushGlobalToAll().catch((err) => {
+        console.warn("[sync] Push cycle failed:", (err as Error).message);
+      });
+    }, this.syncIntervalMs);
+  }
+
+  /** Stop periodic push. */
+  stopPushLoop(): void {
+    if (this.pushTimer) {
+      clearInterval(this.pushTimer);
+      this.pushTimer = null;
+    }
+  }
+
+  /** Get all synced L3 entries the Primary has received. */
+  getSyncedEntries(): SyncL3Entry[] {
+    return Array.from(this.l3Store.values());
+  }
+
+  getSyncIntervalMs(): number {
+    return this.syncIntervalMs;
+  }
+}

package/packages/runtime/src/fleet/primary-client.ts

@@ -0,0 +1,285 @@
+/**
+ * Primary-side client for managing Workers.
+ *
+ * The Primary uses this to:
+ *   - Track registered workers
+ *   - Poll worker health
+ *   - Assign/unassign contexts to workers
+ *   - Collect status reports
+ */
+
+import type {
+  WorkerInfo,
+  WorkerRegistrationRequest,
+  WorkerRegistrationResponse,
+  WorkerHealthResponse,
+  WorkerHealthStatus,
+  WorkerStatus,
+  WorkerStatusReport,
+  ContextAssignRequest,
+  ContextAssignResponse,
+  SyncPushRequest,
+  SyncPushResponse,
+  SyncPullRequest,
+  SyncPullResponse,
+} from "./worker-protocol.js";
+
+import {
+  PRIMARY_ROUTES,
+  WORKER_ROUTES,
+  DEFAULT_HEALTH_INTERVAL_MS,
+  HEALTH_TIMEOUT_MS,
+} from "./worker-protocol.js";
+
+export class PrimaryClient {
+  private workers: Map<string, WorkerInfo> = new Map();
+  private healthTimer: ReturnType<typeof setInterval> | null = null;
+  private nextId = 1;
+  private onSyncPullCallback: ((req: SyncPullRequest) => Promise<SyncPullResponse>) | null = null;
+
+  /** All registered workers. */
+  getWorkers(): WorkerInfo[] {
+    return Array.from(this.workers.values());
+  }
+
+  /** Get a single worker by ID. */
+  getWorker(workerId: string): WorkerInfo | undefined {
+    return this.workers.get(workerId);
+  }
+
+  /** Find which worker owns a context, if any. */
+  findWorkerForContext(contextName: string): WorkerInfo | undefined {
+    for (const w of this.workers.values()) {
+      if (w.assigned_contexts.includes(contextName)) return w;
+    }
+    return undefined;
+  }
+
+  // --- Registration (called when Worker POSTs to Primary) ---
+
+  handleRegistration(req: WorkerRegistrationRequest): WorkerRegistrationResponse {
+    const id = `worker-${this.nextId++}`;
+    const now = new Date().toISOString();
+
+    const info: WorkerInfo = {
+      id,
+      url: req.url,
+      capabilities: req.capabilities,
+      assigned_contexts: [],
+      registered_at: now,
+      last_heartbeat: now,
+    };
+
+    this.workers.set(id, info);
+    return { worker_id: id, registered_at: now };
+  }
+
+  /** Remove a worker from the registry. */
+  deregister(workerId: string): boolean {
+    return this.workers.delete(workerId);
+  }
+
+  // --- Health Checking ---
+
+  /** Poll a single worker's health endpoint. */
+  async checkHealth(workerId: string): Promise<WorkerHealthResponse | null> {
+    const worker = this.workers.get(workerId);
+    if (!worker) return null;
+
+    try {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS);
+
+      const resp = await fetch(`${worker.url}${WORKER_ROUTES.health}`, {
+        signal: controller.signal,
+      });
+      clearTimeout(timeout);
+
+      if (!resp.ok) {
+        this.markUnreachable(workerId);
+        return null;
+      }
+
+      const health = (await resp.json()) as WorkerHealthResponse;
+      worker.last_heartbeat = new Date().toISOString();
+      return health;
+    } catch {
+      this.markUnreachable(workerId);
+      return null;
+    }
+  }
+
+  /** Poll all workers and return their statuses. */
+  async checkAllHealth(): Promise<Map<string, WorkerHealthStatus>> {
+    const results = new Map<string, WorkerHealthStatus>();
+
+    const checks = Array.from(this.workers.keys()).map(async (id) => {
+      const health = await this.checkHealth(id);
+      results.set(id, health?.status ?? "unreachable");
+    });
+
+    await Promise.all(checks);
+    return results;
+  }
+
+  /** Start periodic health polling. */
+  startHealthPolling(intervalMs = DEFAULT_HEALTH_INTERVAL_MS): void {
+    this.stopHealthPolling();
+    this.healthTimer = setInterval(() => {
+      this.checkAllHealth().catch(() => {});
+    }, intervalMs);
+  }
+
+  /** Stop periodic health polling. */
+  stopHealthPolling(): void {
+    if (this.healthTimer) {
+      clearInterval(this.healthTimer);
+      this.healthTimer = null;
+    }
+  }
+
+  // --- Context Assignment ---
+
+  /** Assign a context to a worker. */
+  async assignContext(
+    workerId: string,
+    contextName: string,
+    contextDescription = "",
+  ): Promise<ContextAssignResponse> {
+    const worker = this.workers.get(workerId);
+    if (!worker) {
+      return { context_name: contextName, accepted: false, reason: "Worker not found" };
+    }
+
+    if (worker.assigned_contexts.length >= worker.capabilities.max_contexts) {
+      return { context_name: contextName, accepted: false, reason: "Worker at capacity" };
+    }
+
+    const body: ContextAssignRequest = {
+      context_name: contextName,
+      context_description: contextDescription,
+    };
+
+    try {
+      const resp = await fetch(`${worker.url}${WORKER_ROUTES.assign}`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+      });
+
+      if (!resp.ok) {
+        const text = await resp.text();
+        return { context_name: contextName, accepted: false, reason: `Worker rejected: ${text}` };
+      }
+
+      const result = (await resp.json()) as ContextAssignResponse;
+
+      if (result.accepted) {
+        worker.assigned_contexts.push(contextName);
+      }
+
+      return result;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return { context_name: contextName, accepted: false, reason: `Request failed: ${msg}` };
+    }
+  }
+
+  /** Unassign a context from a worker. */
+  async unassignContext(workerId: string, contextName: string): Promise<boolean> {
+    const worker = this.workers.get(workerId);
+    if (!worker) return false;
+
+    try {
+      const resp = await fetch(`${worker.url}${WORKER_ROUTES.unassign(contextName)}`, {
+        method: "DELETE",
+      });
+
+      if (resp.ok) {
+        worker.assigned_contexts = worker.assigned_contexts.filter((c) => c !== contextName);
+        return true;
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  }
+
+  // --- Status Collection ---
+
+  /** Handle an incoming status report from a Worker. */
+  handleStatusReport(workerId: string, report: WorkerStatusReport): WorkerStatus | null {
+    const worker = this.workers.get(workerId);
+    if (!worker) return null;
+
+    worker.last_heartbeat = new Date().toISOString();
+
+    return {
+      worker_id: workerId,
+      activity: report.activity,
+      current_context: report.current_context,
+      current_task: report.current_task,
+      error: report.error,
+      reported_at: new Date().toISOString(),
+    };
+  }
+
+  // --- Memory Sync ---
+
+  /** Register a handler for incoming sync pull requests from Workers. */
+  onSyncPull(cb: (req: SyncPullRequest) => Promise<SyncPullResponse>): void {
+    this.onSyncPullCallback = cb;
+  }
+
+  /** Handle an incoming sync pull (Worker sends L3 knowledge to Primary). */
+  async handleSyncPull(req: SyncPullRequest): Promise<SyncPullResponse> {
+    if (!this.onSyncPullCallback) {
+      return { accepted: 0, rejected: req.entries.length };
+    }
+    return this.onSyncPullCallback(req);
+  }
+
+  /** Push Global context updates to a specific worker. */
+  async pushSyncToWorker(workerId: string, payload: SyncPushRequest): Promise<SyncPushResponse | null> {
+    const worker = this.workers.get(workerId);
+    if (!worker) return null;
+
+    try {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS);
+
+      const resp = await fetch(`${worker.url}${WORKER_ROUTES.syncPush}`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: controller.signal,
+      });
+      clearTimeout(timeout);
+
+      if (!resp.ok) return null;
+      return (await resp.json()) as SyncPushResponse;
+    } catch {
+      return null;
+    }
+  }
+
+  /** Push Global context updates to all registered workers. */
+  async pushSyncToAll(payload: SyncPushRequest): Promise<Map<string, SyncPushResponse | null>> {
+    const results = new Map<string, SyncPushResponse | null>();
+
+    const pushes = Array.from(this.workers.keys()).map(async (id) => {
+      const result = await this.pushSyncToWorker(id, payload);
+      results.set(id, result);
+    });
+
+    await Promise.all(pushes);
+    return results;
+  }
+
+  // --- Internal ---
+
+  private markUnreachable(workerId: string): void {
+    // Worker stays registered but health is tracked via last_heartbeat staleness.
+    // Fleet manager (Phase 3.3) will decide when to evict.
+  }
+}