@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.2

package/src/runner/bridge.js

@@ -0,0 +1,289 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { fullJitterDelay } from '../lib/backoff.js';
+import { scopeBlock } from '../lib/errors.js';
+import { OUTBOX_MAX_PENDING } from '../lib/runnerStore.js';
+import { OutboxFlusher } from '../lib/outboxFlusher.js';
+
+const ONLINE_MS = 35_000;
+const OFFLINE_MS = 60_000;
+const LAG_EVENTS = 50;
+const WAKE_GAP_MS = 30_000;
+const FLUSH_EVERY_MS = 1_000;
+const HEALTH_EVERY_MS = 5_000;
+const BATCHES_PER_FLUSH = 30;
+/** The Phase 2 scanner's file sources (channel.md, personas.json, state/chat) and presence, as the shadow ran them. */
+export const FILE_SOURCES = Object.freeze(['personas', 'chat', 'channels', 'presence']);
+const FILE_SCAN_MS = 10_000;
+/** The manager lease (Q9): renewed well inside its ttl; this machine stops counting on it `LEASE_MARGIN_MS` early. */
+export const LEASE_TTL_MS = 30_000;
+const LEASE_RENEW_MS = 10_000;
+const LEASE_MARGIN_MS = 5_000;
+/** runner.db meta key of the cloud's presence keyset cursor (S3), kept beside the event cursor and sent on every sync. */
+const PRESENCE_CURSOR = 'presence_cursor';
+/** This many sync failures in a row (no page applied in between) and health says so, ahead of anything milder. */
+export const SYNC_FAILING_AFTER = 3;
+/** SCOPE-02: while blocked for a scope or grant, the sync is retried this often with the sign-in read again, so a re-login clears it. */
+export const BLOCK_RETRY_MS = 60_000;
+const ERROR_CODES = new Set(['HUB_LOGIN_REQUIRED', 'HUB_SCOPE_DENIED', 'capability_denied', 'cloud_first_off', 'HUB_MACHINE_REVOKED']);
+
+/**
+ * The daemon's half of the runner (design §6.3): it holds the cloud credentials, so it long-polls `/connector/sync`
+ * and hands each page to the runner process to apply, and it flushes runner.db's outbox to `/connector/commands`.
+ * Outbound only. The cursor it asks from is the one the runner committed with the page it applied (I2), read back
+ * from runner.db, so a crash anywhere re-asks for the same page. Health (§9.4) is measured here and sent to the
+ * runner for the facade, `pulse status` and the cloud.
+ */
+export class RunnerBridge {
+  constructor({
+    runner, transport, store, machineId = null, now = Date.now, monotonic = () => performance.now(), random = Math.random, waitSeconds = 20, log = () => {},
+    lease = { ttlMs: LEASE_TTL_MS, renewMs: LEASE_RENEW_MS, marginMs: LEASE_MARGIN_MS }, fileScan = null, fileScanMs = FILE_SCAN_MS, blockRetryMs = BLOCK_RETRY_MS,
+    flusher = new OutboxFlusher({ store, transport, random }),
+  }) {
+    this.runner = runner;
+    this.transport = transport;
+    this.store = store;
+    this.now = now;
+    this.monotonic = monotonic;
+    this.random = random;
+    this.waitSeconds = waitSeconds;
+    this.log = log;
+    this.flusher = flusher;
+    this.machineId = machineId;
+    this.leaseConfig = lease;
+    this.leaseDeadline = 0;
+    this.fileScan = fileScan;
+    this.fileScanMs = fileScanMs;
+    this.blockRetryMs = blockRetryMs;
+    this.running = false;
+    this.kicked = null;
+    this.state = { lastSyncAt: null, lastContactAt: null, failures: 0, syncFailure: null, error: null, resync: null, head: 0, sleptAt: null, flushError: null };
+  }
+
+  start() {
+    if (this.running) return;
+    this.running = true;
+    this.runner.on('wrote', () => this.kick());
+    this.syncing = this.#syncLoop();
+    this.flushing = this.#flushLoop();
+    if (this.leaseConfig) this.leasing = this.#leaseLoop();
+    if (this.fileScan) this.scanning = this.#fileLoop();
+    this.healthTimer = setInterval(() => this.#pushHealth(), HEALTH_EVERY_MS);
+    this.healthTimer.unref?.();
+    this.#pushHealth();
+  }
+
+  async stop() {
+    this.running = false;
+    clearInterval(this.healthTimer);
+    this.kick();
+    this.leaseWake?.();
+    this.fileWake?.();
+    await Promise.allSettled([this.syncing, this.flushing, this.leasing, this.scanning]);
+    if (this.leaseConfig && this.leaseDeadline > this.monotonic()) {
+      await this.#claimLease({ release: true }).catch(() => {});
+      this.runner.send({ type: 'lease', held: false });
+    }
+  }
+
+  /** A local write landed: flush now instead of at the next tick. */
+  kick() {
+    const wake = this.kicked;
+    this.kicked = null;
+    wake?.();
+  }
+
+  applied() {
+    return Number(this.store.db.prepare('SELECT applied_seq FROM pulse_sync WHERE one = 1').get()?.applied_seq || 0);
+  }
+
+  async #syncLoop() {
+    let lastWall = this.now();
+    let lastMono = this.monotonic();
+    let lagging = true;
+    while (this.running) {
+      const wallGap = this.now() - lastWall;
+      const monoGap = this.monotonic() - lastMono;
+      if (wallGap - monoGap > WAKE_GAP_MS) {
+        this.state.sleptAt = this.now();
+        this.log('runner_woke', { asleepMs: Math.round(wallGap - monoGap) });
+        lagging = true;
+      }
+      lastWall = this.now();
+      lastMono = this.monotonic();
+      const blocked = this.state.error?.block ? this.state.error : null;
+      if (this.state.resync || (this.state.error && !blocked)) { await sleep(5_000); continue; }
+      if (blocked) {
+        const left = blocked.at + this.blockRetryMs - this.now();
+        if (left > 0) { await sleep(Math.min(5_000, left)); continue; }
+        this.transport.auth?.invalidate?.();
+      }
+      try {
+        const since = this.applied();
+        const presence = this.store.meta?.(PRESENCE_CURSOR) ?? null;
+        const page = await this.transport.sync({ since, wait: lagging ? 0 : this.waitSeconds, ...(presence ? { presence } : {}) });
+        this.state.lastContactAt = this.now();
+        const result = await this.runner.request({ type: 'apply', page });
+        this.state.lastSyncAt = this.now();
+        if (blocked) {
+          this.state.error = null;
+          this.log('sync_unblocked', { code: blocked.code });
+          this.#pushHealth();
+        }
+        if (typeof page.presenceNext === 'string' && page.presenceNext && page.presenceNext !== presence) this.store.setMeta?.(PRESENCE_CURSOR, page.presenceNext);
+        this.state.failures = 0;
+        this.state.syncFailure = null;
+        this.state.head = Number(page.head) || 0;
+        lagging = Number(page.head) > Number(result.next);
+        if (result.quarantined) this.log('sync_quarantined', { count: result.quarantined, next: result.next });
+      } catch (error) {
+        if (!this.running) break;
+        if (error.code === 'runner_exited') { await sleep(1_000); continue; }
+        if (error.status === 410 || error.code === 'resync_required') {
+          this.state.resync = { at: this.now(), floor: error.details?.floor ?? null };
+          this.log('sync_resync_required', { since: this.applied() });
+          continue;
+        }
+        // A cursor the cloud refuses (after a downgrade, say) would fail every sync: drop it and start presence over.
+        if (error.code === 'invalid_presence' && this.store.meta?.(PRESENCE_CURSOR)) {
+          this.store.setMeta?.(PRESENCE_CURSOR, null);
+          this.log('presence_cursor_reset', { code: error.code });
+          continue;
+        }
+        if (ERROR_CODES.has(error.code)) {
+          this.state.error = { code: error.code, at: this.now(), block: scopeBlock(error) };
+          this.log('sync_refused', { code: error.code });
+          this.#pushHealth();
+          continue;
+        }
+        this.state.failures += 1;
+        this.state.syncFailure = { code: error.code || 'error', at: this.now() };
+        this.log('sync_failed', { code: error.code || 'error', failures: this.state.failures });
+        await sleep(Math.max(1_000, fullJitterDelay(this.state.failures, { capMs: 30_000, random: this.random })));
+      }
+    }
+  }
+
+  async #flushLoop() {
+    let failures = 0;
+    while (this.running) {
+      try {
+        const tally = await this.flusher.drain({ maxBatches: BATCHES_PER_FLUSH });
+        this.state.lastContactAt = tally.batches ? this.now() : this.state.lastContactAt;
+        this.state.flushError = null;
+        failures = 0;
+        if (tally.written + tally.duplicate > 0) await this.runner.request({ type: 'retryDeferred' }).catch(() => {});
+      } catch (error) {
+        failures += 1;
+        this.state.flushError = { code: error.code || 'error', at: this.now() };
+        if (ERROR_CODES.has(error.code)) this.state.error = { code: error.code, at: this.now(), block: scopeBlock(error) };
+        await sleep(Math.max(1_000, fullJitterDelay(failures, { capMs: 30_000, random: this.random })));
+        continue;
+      }
+      await Promise.race([sleep(FLUSH_EVERY_MS), new Promise((resolve) => { this.kicked = resolve; })]);
+    }
+  }
+
+  /**
+   * After cutover the hub.db sources are captured in the runner; the file sources stay with the Phase 2 scanner, now
+   * reading runner.db: channel.md posts agents append, personas, project chat, and presence beats (not log events).
+   */
+  async #fileLoop() {
+    while (this.running) {
+      try {
+        const report = await this.fileScan.run({ mode: 'shadow', sources: FILE_SOURCES });
+        for (let start = 0; start < report.presence.length; start += 100) {
+          const chunk = report.presence.slice(start, start + 100);
+          const answer = await this.transport.commands({ commands: chunk });
+          chunk.forEach((beat, index) => {
+            if (!answer?.results?.[index]?.error) this.store.setPresenceSent(beat.payload.actorId, beat.payload.at);
+          });
+        }
+        if (report.totals.planned) this.kick();
+      } catch (error) {
+        this.log('file_scan_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) });
+      }
+      await Promise.race([sleep(this.fileScanMs), new Promise((resolve) => { this.fileWake = resolve; })]);
+    }
+  }
+
+  /** One lease call through /commands (no seq). Resolves with the lease, or throws. */
+  async #claimLease({ release = false } = {}) {
+    const payload = release ? { role: 'manager', release: true } : { role: 'manager', ttlMs: this.leaseConfig.ttlMs };
+    const answer = await this.transport.commands({ commands: [{ type: 'runner.lease', idempotencyKey: `lease:${this.machineId || 'runner'}:${this.now()}`, payload }] });
+    const result = answer?.results?.[0];
+    if (!result || result.error) throw Object.assign(new Error(result?.error?.message || 'no lease result'), { code: result?.error?.code || 'lease_failed' });
+    return result.lease;
+  }
+
+  /**
+   * Claims or renews the manager lease every renewMs. The deadline is the monotonic time the winning request was sent,
+   * plus the ttl, minus a margin: never the cloud's `until` against this machine's clock (H4). A failed renew keeps the
+   * deadline it had; when it passes, the runner is told at once and stops its manager before the cloud lets anyone else in.
+   */
+  async #leaseLoop() {
+    const { ttlMs, renewMs, marginMs } = this.leaseConfig;
+    let lapse = null;
+    const report = (lease) => {
+      const remainingMs = Math.max(0, this.leaseDeadline - this.monotonic());
+      this.runner.send({ type: 'lease', held: remainingMs > 0, remainingMs, holder: lease?.holder ?? null, fence: lease?.fence ?? 0 });
+    };
+    while (this.running) {
+      const sentAt = this.monotonic();
+      try {
+        const lease = await this.#claimLease();
+        this.lease = lease;
+        this.leaseDeadline = lease.here ? sentAt + ttlMs - marginMs : 0;
+        report(lease);
+      } catch (error) {
+        this.log('lease_failed', { code: error.code || 'error' });
+        report(this.lease);
+      }
+      clearTimeout(lapse);
+      const remaining = this.leaseDeadline - this.monotonic();
+      if (remaining > 0) {
+        lapse = setTimeout(() => report(this.lease), remaining);
+        lapse.unref?.();
+      }
+      await Promise.race([sleep(renewMs), new Promise((resolve) => { this.leaseWake = resolve; })]);
+    }
+    clearTimeout(lapse);
+  }
+
+  /** The measured state (§9.4): never "running" alone. */
+  health() {
+    const now = this.now();
+    const pending = this.store.pendingCount();
+    const dead = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM dead').get().n);
+    const quarantined = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM pulse_quarantine').get()?.n || 0);
+    const applied = this.applied();
+    const lag = Math.max(0, this.state.head - applied);
+    const outbox = { pending, dead, capacity: OUTBOX_MAX_PENDING };
+    const base = { lastSyncAt: this.state.lastSyncAt, applied, head: this.state.head, lag, outbox, quarantined,
+      staleSeconds: this.state.lastSyncAt ? (now - this.state.lastSyncAt) / 1000 : null };
+    const { block } = this.state.error || {};
+    if (block) {
+      const missing = block.missingScopes.length ? `missing ${block.missingScopes.join(', ')}` : 'missing a scope or capability';
+      return { ...base, state: 'blocked', reason: `${block.code}: ${missing}`, code: block.code, missingScopes: block.missingScopes, fix: block.fix };
+    }
+    if (this.state.error) return { ...base, state: 'error', reason: this.state.error.code, fix: this.state.error.code === 'HUB_LOGIN_REQUIRED' ? 'agentpulse login' : null };
+    if (this.state.resync) return { ...base, state: 'error', reason: 'resync_required' };
+    if (this.state.failures >= SYNC_FAILING_AFTER && this.state.syncFailure) {
+      const { code } = this.state.syncFailure;
+      return { ...base, state: 'degraded', reason: `sync failing (${code})`, code, failures: this.state.failures };
+    }
+    if (this.state.sleptAt && (!this.state.lastSyncAt || this.state.lastSyncAt < this.state.sleptAt)) return { ...base, state: 'sleeping', reason: 'woke from sleep, catching up' };
+    if (!this.state.lastContactAt || now - this.state.lastContactAt > OFFLINE_MS) {
+      return { ...base, state: 'offline', reason: this.state.lastContactAt ? 'no contact with Pulse' : 'not connected yet' };
+    }
+    if (lag > LAG_EVENTS || (pending > 0 && this.state.flushError)) return { ...base, state: 'catching_up', reason: `${lag} events and ${pending} changes to go` };
+    if (dead || quarantined || pending > 0.8 * OUTBOX_MAX_PENDING) {
+      return { ...base, state: 'degraded', reason: `${dead} change${dead === 1 ? '' : 's'} and ${quarantined} event${quarantined === 1 ? '' : 's'} set aside` };
+    }
+    if (!this.state.lastSyncAt || now - this.state.lastSyncAt > ONLINE_MS) return { ...base, state: 'catching_up', reason: 'waiting for a sync' };
+    return { ...base, state: 'online', reason: 'in sync' };
+  }
+
+  #pushHealth() {
+    try { this.runner.send({ type: 'health', health: this.health() }); } catch (error) { this.log('health_failed', { code: error.code || 'error' }); }
+  }
+}