@agentpulselabs/cli-plugin-pulse 0.1.0 → 1.0.0

package/src/lib/mirrorService.js

@@ -0,0 +1,358 @@
+import { fullJitterDelay } from './backoff.js';
+
+// Every hub graph read writes hub.db and so raises a `hub` change event of its own; only windows passes read the
+// graph (edges and bus ride on their 168 h read), so targeted passes never feed that loop, and the echo of a windows
+// pass is not taken for a change (see onChange).
+export const CHANGE_SOURCES = Object.freeze({
+  channels: ['deliveries', 'channels', 'windows'],
+  hub: ['deliveries', 'channels', 'inbox', 'windows'],
+  agents: ['deliveries', 'windows'],
+  // A hub heartbeat: only who is online and when they were last seen moved, which only the windows carry.
+  presence: ['windows'],
+});
+
+const REFUSAL_CODES = new Set(['HUB_MIRROR_SCOPE_MISSING', 'HUB_SCOPE_DENIED']);
+
+export function isMirrorRefusal(code) {
+  return REFUSAL_CODES.has(code) || String(code).startsWith('machine_');
+}
+
+export class MirrorService {
+  constructor({
+    sync,
+    reader,
+    debounceMs = 300,
+    windowsGapMs = 2_000,
+    slowWindowsGapMs = 30_000,
+    echoMs = 1_500,
+    echoGapMs = 10_000,
+    presenceGapMs = 10_000,
+    reconcileMs = 5 * 60_000,
+    maxFullDelayMs = 15 * 60_000,
+    reconnectBaseMs = 1_000,
+    reconnectCapMs = 30_000,
+    refusedBaseMs = 60_000,
+    refusedCapMs = 15 * 60_000,
+    random = Math.random,
+    now = Date.now,
+    log = () => {},
+    onPassStart = () => null,
+    onPassEnd = () => {},
+  }) {
+    this.onPassStart = onPassStart;
+    this.onPassEnd = onPassEnd;
+    this.sync = sync;
+    this.reader = reader;
+    this.debounceMs = debounceMs;
+    this.windowsGapMs = windowsGapMs;
+    this.slowWindowsGapMs = slowWindowsGapMs;
+    this.lastSlowWindowsAt = -Infinity;
+    this.echoMs = echoMs;
+    this.echoGapMs = echoGapMs;
+    this.presenceGapMs = presenceGapMs;
+    this.windowsTimer = null;
+    this.windowsTimerGapMs = null;
+    this.lastWindowsAt = -Infinity;
+    this.windowsRunning = false;
+    this.windowsDeferredGapMs = null;
+    this.windowsChanged = false;
+    this.reconcileMs = reconcileMs;
+    this.maxFullDelayMs = maxFullDelayMs;
+    this.lastFullCompletedAt = -Infinity;
+    this.fullController = null;
+    this.reconnectBaseMs = reconnectBaseMs;
+    this.reconnectCapMs = reconnectCapMs;
+    this.refusedBaseMs = refusedBaseMs;
+    this.refusedCapMs = refusedCapMs;
+    this.random = random;
+    this.now = now;
+    this.log = log;
+    this.blocked = null;
+    this.refusals = 0;
+    this.retryTimer = null;
+    this.controller = null;
+    this.pendingSources = new Set();
+    this.pendingFull = false;
+    this.debounceTimer = null;
+    this.reconcileTimer = null;
+    this.running = null;
+    this.windowsPass = null;
+    this.listening = null;
+    this.connected = false;
+  }
+
+  start() {
+    if (this.controller) return;
+    this.controller = new AbortController();
+    this.#requestFull();
+    this.listening = this.#listen(this.controller.signal);
+  }
+
+  async stop() {
+    if (!this.controller) return;
+    this.controller.abort();
+    this.fullController?.abort();
+    clearTimeout(this.reconcileTimer);
+    this.reconcileTimer = null;
+    clearTimeout(this.debounceTimer);
+    clearTimeout(this.retryTimer);
+    clearTimeout(this.windowsTimer);
+    this.debounceTimer = null;
+    this.retryTimer = null;
+    this.windowsTimer = null;
+    this.pendingFull = false;
+    this.pendingSources.clear();
+    await Promise.allSettled([this.listening, this.running, this.windowsPass]);
+    this.controller = null;
+  }
+
+  onChange(kinds) {
+    if (this.blocked) return;
+    // A `hub` event during one of our graph reads or within echoMs after it may be that read's echo. Once a windows
+    // pass found nothing new, such an event only refreshes the windows after echoGapMs, so an idle hub is not re-read
+    // in a loop. While windows passes keep finding changes the hub is busy, and it refreshes at the normal gap.
+    const ours = this.reader.graphReadRecently?.(this.echoMs) ?? false;
+    const echo = !this.windowsChanged && ours;
+    const news = kinds.filter((kind) => kind !== 'presence');
+    if (news.some((kind) => CHANGE_SOURCES[kind]) && !(ours && news.every((kind) => kind === 'hub'))) this.#preemptFull();
+    for (const kind of kinds) {
+      for (const source of CHANGE_SOURCES[kind] || []) {
+        if (source !== 'windows') this.pendingSources.add(source);
+        else if (kind === 'presence') this.#requestWindows(Math.max(this.windowsGapMs, this.presenceGapMs), { preempt: false });
+        else this.#requestWindows(kind === 'hub' && echo ? Math.max(this.windowsGapMs, this.echoGapMs) : this.windowsGapMs);
+      }
+    }
+    if (!this.pendingSources.size || this.debounceTimer) return;
+    this.debounceTimer = setTimeout(() => {
+      this.debounceTimer = null;
+      this.#drain();
+    }, this.debounceMs);
+    this.debounceTimer.unref?.();
+  }
+
+  #requestFull() {
+    this.pendingFull = true;
+    this.#drain();
+  }
+
+  /**
+   * A change never waits behind a full pass: one that is running stops at its next read or batch boundary and runs
+   * again after the change passes. Once the last complete full pass is maxFullDelayMs old, it runs to the end, so
+   * constant change cannot starve the reconcile.
+   */
+  #fullPreemptible() {
+    return this.now() - this.lastFullCompletedAt < this.maxFullDelayMs;
+  }
+
+  #preemptFull() {
+    if (this.fullController && !this.fullController.signal.aborted && this.#fullPreemptible()) this.fullController.abort();
+  }
+
+  /**
+   * The four graph windows are the hub's slowest read, so change events refresh them at most once per gapMs (normally
+   * windowsGapMs), counted from the end of the last windows pass so the hub is never asked for them back to back.
+   * A request with a shorter gap replaces a pending one with a longer gap.
+   */
+  #requestWindows(gapMs = this.windowsGapMs, { preempt = true } = {}) {
+    if (this.windowsTimer && this.windowsTimerGapMs <= gapMs) return;
+    clearTimeout(this.windowsTimer);
+    this.windowsTimerGapMs = gapMs;
+    const waitMs = Math.max(this.debounceMs, this.lastWindowsAt + gapMs - this.now());
+    this.windowsTimer = setTimeout(() => {
+      this.windowsTimer = null;
+      if (this.windowsRunning) {
+        this.windowsDeferredGapMs = Math.min(this.windowsDeferredGapMs ?? Infinity, gapMs);
+        if (preempt) this.#preemptFull();
+        return;
+      }
+      if (this.now() < this.lastWindowsAt + gapMs) {
+        this.#requestWindows(gapMs, { preempt });
+        return;
+      }
+      this.pendingSources.add('windows');
+      this.#drain();
+    }, waitMs);
+    this.windowsTimer.unref?.();
+  }
+
+  /**
+   * Two lanes. Windows refreshes (the hub's slowest read; edges and bus ride on their 168 h read) run one at a time in a
+   * lane of their own, so a channels, deliveries or inbox change never waits behind one: it runs at once in the change
+   * lane, beside it. The lanes write disjoint item families. Full passes run in the change lane and read the windows
+   * too, so a full pass and a windows pass never run at once.
+   */
+  #drain({ retry = false } = {}) {
+    if (!this.controller || this.controller.signal.aborted) return;
+    if (this.blocked && !retry) {
+      this.pendingFull = false;
+      this.pendingSources.clear();
+      return;
+    }
+    if (this.running) {
+      this.#drainWindows();
+      return;
+    }
+    const changes = new Set([...this.pendingSources].filter((source) => source !== 'windows'));
+    // Pending changes go ahead of a pending full pass, unless the reconcile is overdue (see #fullPreemptible).
+    if (this.pendingFull && !(changes.size && this.#fullPreemptible())) {
+      if (this.windowsPass) return;
+      this.pendingFull = false;
+      this.pendingSources.clear();
+      this.#startChanges({ full: true, sources: changes });
+      return;
+    }
+    this.#drainWindows();
+    if (!changes.size) return;
+    for (const source of changes) this.pendingSources.delete(source);
+    this.#startChanges({ full: false, sources: changes });
+  }
+
+  // Between slow refreshes only the recent windows are read; the 72 h and 168 h ones, edges and bus wait for the next.
+  #drainWindows() {
+    if (this.windowsPass || this.fullController || !this.pendingSources.has('windows')) return;
+    this.pendingSources.delete('windows');
+    const slow = this.now() >= this.lastSlowWindowsAt + this.slowWindowsGapMs;
+    const sources = new Set(slow ? ['graph', 'windows'] : ['recent']);
+    this.windowsRunning = true;
+    this.windowsPass = this.#pass({ full: false, sources, lane: 'windows' }).finally(() => {
+      this.windowsPass = null;
+      this.lastWindowsAt = this.now();
+      this.windowsRunning = false;
+      if (slow) {
+        this.lastSlowWindowsAt = this.lastWindowsAt;
+      } else if (this.controller && !this.controller.signal.aborted) {
+        // A change the long windows have not seen yet reaches them once their gap is up, change events or not.
+        this.#requestWindows(Math.max(this.windowsGapMs, this.lastSlowWindowsAt + this.slowWindowsGapMs - this.lastWindowsAt));
+      }
+      // The briefs of the tasks this pass saw move follow in a pass of their own, after the windows are up.
+      this.pendingSources.add('briefs');
+      this.#requestDeferredWindows();
+      if (this.debounceTimer === null) this.#drain();
+    });
+  }
+
+  #startChanges({ full, sources }) {
+    this.fullController = full ? new AbortController() : null;
+    if (full) this.windowsRunning = true;
+    this.running = this.#pass({ full, sources, lane: 'changes', signal: this.fullController?.signal }).then((preempted) => {
+      this.running = null;
+      this.fullController = null;
+      if (!full) {
+        if (this.debounceTimer === null) this.#drain();
+        return;
+      }
+      this.windowsRunning = false;
+      if (preempted) {
+        // The windows it read never reached the cloud: a windows refresh owed meanwhile runs at once.
+        if (this.windowsDeferredGapMs !== null || (this.windowsTimer && this.windowsTimerGapMs <= this.windowsGapMs)) {
+          clearTimeout(this.windowsTimer);
+          this.windowsTimer = null;
+          this.windowsDeferredGapMs = null;
+          this.pendingSources.add('windows');
+        }
+        this.#drain();
+        return;
+      }
+      // The next reconcile counts from the end of this one, so a slow hub never gets full passes back to back.
+      if (this.controller && !this.controller.signal.aborted) {
+        clearTimeout(this.reconcileTimer);
+        this.reconcileTimer = setTimeout(() => this.#requestFull(), this.reconcileMs);
+        this.reconcileTimer.unref?.();
+      }
+      this.lastWindowsAt = this.now();
+      this.lastSlowWindowsAt = this.lastWindowsAt;
+      this.#requestDeferredWindows();
+      if (this.debounceTimer === null) this.#drain();
+    });
+  }
+
+  #requestDeferredWindows() {
+    if (this.windowsDeferredGapMs === null) return;
+    const gapMs = this.windowsDeferredGapMs;
+    this.windowsDeferredGapMs = null;
+    this.#requestWindows(gapMs);
+  }
+
+  /** One mirror pass; resolves to whether a change preempted it, and never rejects. */
+  #pass({ full, sources, lane, signal }) {
+    const pass = this.onPassStart({ lane });
+    let preempted = false;
+    return this.sync.run(full ? { full, sources, signal } : { full, sources })
+      .then((result) => {
+        this.blocked = null;
+        this.refusals = 0;
+        if (full) this.lastFullCompletedAt = this.now();
+        if (result.briefsPending) this.pendingSources.add('briefs');
+        if (lane === 'windows') this.windowsChanged = result.pushed + result.tombstoned > 0;
+        this.log('mirror', { full, pushed: result.pushed, tombstoned: result.tombstoned, rejected: result.rejected.length });
+      })
+      .catch((error) => {
+        const code = error.code || 'HUB_MIRROR_FAILED';
+        if (code === 'HUB_MIRROR_PREEMPTED') {
+          preempted = true;
+          this.pendingFull = true;
+          this.log('mirror_preempted', { full });
+          return undefined;
+        }
+        if (isMirrorRefusal(code)) return this.#block(error, code);
+        this.blocked = null;
+        this.log('mirror_failed', { full, code });
+      })
+      .then(() => {
+        this.onPassEnd(pass);
+        return preempted;
+      });
+  }
+
+  // A refused push never succeeds by repeating it, so change events and
+  // reconciles are ignored until one full retry after a capped backoff.
+  async #block(error, code) {
+    const baseMs = Math.min(this.refusedCapMs, this.refusedBaseMs * 2 ** this.refusals);
+    const waitMs = Math.round(baseMs * (1 + this.random() * 0.1));
+    this.refusals += 1;
+    this.blocked = { code, retryAt: this.now() + waitMs };
+    this.log('mirror_blocked', { code, retryInMs: waitMs });
+    await this.sync.recordError?.(error, { retryAt: this.blocked.retryAt }).catch(() => {});
+    if (!this.controller || this.controller.signal.aborted) return;
+    clearTimeout(this.retryTimer);
+    this.retryTimer = setTimeout(() => {
+      this.retryTimer = null;
+      this.pendingFull = true;
+      this.#drain({ retry: true });
+    }, waitMs);
+    this.retryTimer.unref?.();
+  }
+
+  async #listen(signal) {
+    let attempt = 0;
+    let openedBefore = false;
+    while (!signal.aborted) {
+      try {
+        await this.reader.subscribe({
+          signal,
+          onOpen: () => {
+            attempt = 0;
+            this.connected = true;
+            if (openedBefore) this.#requestFull();
+            openedBefore = true;
+          },
+          onChange: (kinds) => this.onChange(kinds),
+        });
+      } catch (error) {
+        if (signal.aborted) break;
+        this.log('mirror_stream_failed', { code: error.code || error.name || 'HUB_LOCAL_HUB_STREAM_FAILED' });
+      }
+      this.connected = false;
+      if (signal.aborted) break;
+      const wait = fullJitterDelay(attempt, { baseMs: this.reconnectBaseMs, capMs: this.reconnectCapMs, random: this.random });
+      attempt += 1;
+      await new Promise((resolve) => {
+        const timer = setTimeout(resolve, wait);
+        signal.addEventListener('abort', () => {
+          clearTimeout(timer);
+          resolve();
+        }, { once: true });
+      });
+    }
+  }
+}