@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.2

package/src/runner/runnerSchema.js

@@ -0,0 +1,51 @@
+/**
+ * The runner's own tables in runner.db, beside the hub ledger and the Phase 2 outbox (design §6.2).
+ * - pulse_sync: the applied cursor, advanced only in the transaction that applies its events (I2).
+ * - pulse_events: what sync applied, skipped or set aside, kept 7 days.
+ * - pulse_quarantine: events that could not be applied; the cursor still moves past them (I1).
+ * - pulse_deferred: foreign changes to an entity whose own newer version is still in the outbox (M1).
+ * - pulse_spool: spool lines already imported, by content hash.
+ * - pulse_registered: when each address last registered through the facade, so an older spooled register yields.
+ * - pulse_rendered: remote posts appended to a channel.md: pending until the file and the ingest record both have it.
+ * - pulse_jobs_local: local effects (runs, manager turns) journaled before they start.
+ * - pulse_fence: deliveries held for an old-code listener still running at cutover.
+ * - Pulse 1.1.1: the bus identity tables (busIdentity.js) and the read, skipped and bounce records (deliveryStates.js).
+ * - Pulse 1.1.2: the posts each session wrote, open permission prompts and stale episodes (channelDiscipline.js).
+ */
+import { installBusIdentity } from './busIdentity.js';
+import { installChannelDiscipline } from './channelDiscipline.js';
+import { installDeliveryStates } from './deliveryStates.js';
+
+const SCHEMA = [
+  `CREATE TABLE IF NOT EXISTS pulse_sync (
+     one INTEGER PRIMARY KEY CHECK (one = 1), applied_seq INTEGER NOT NULL DEFAULT 0, head INTEGER NOT NULL DEFAULT 0,
+     floor INTEGER NOT NULL DEFAULT 0, presence_stamp INTEGER NOT NULL DEFAULT 0, last_ok_at INTEGER, last_error TEXT,
+     resync TEXT)`,
+  'INSERT OR IGNORE INTO pulse_sync (one) VALUES (1)',
+  `CREATE TABLE IF NOT EXISTS pulse_events (
+     seq INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE, type TEXT NOT NULL, key TEXT NOT NULL, origin TEXT,
+     outcome TEXT NOT NULL, applied_at INTEGER NOT NULL)`,
+  'CREATE INDEX IF NOT EXISTS pulse_events_at ON pulse_events (applied_at)',
+  `CREATE TABLE IF NOT EXISTS pulse_quarantine (
+     seq INTEGER PRIMARY KEY, id TEXT, type TEXT, reason TEXT NOT NULL, payload TEXT, at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS pulse_deferred (
+     seq INTEGER PRIMARY KEY, grp TEXT NOT NULL, event TEXT NOT NULL, at INTEGER NOT NULL)`,
+  'CREATE INDEX IF NOT EXISTS pulse_deferred_grp ON pulse_deferred (grp, seq)',
+  'CREATE TABLE IF NOT EXISTS pulse_registered (id TEXT PRIMARY KEY, at INTEGER NOT NULL)',
+  'CREATE TABLE IF NOT EXISTS pulse_spool (sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, line INTEGER NOT NULL, outcome TEXT NOT NULL, imported_at INTEGER NOT NULL)',
+  `CREATE TABLE IF NOT EXISTS pulse_rendered (
+     post_key TEXT PRIMARY KEY, channel TEXT NOT NULL, file TEXT, text TEXT NOT NULL, state TEXT NOT NULL, at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS pulse_jobs_local (
+     job_id TEXT PRIMARY KEY, kind TEXT NOT NULL, fence INTEGER, effect_started_at INTEGER, outcome TEXT,
+     settle_attempts INTEGER NOT NULL DEFAULT 0, settle_failed INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS pulse_fence (
+     message_id TEXT NOT NULL, recipient TEXT NOT NULL, holder TEXT NOT NULL, since INTEGER NOT NULL,
+     PRIMARY KEY (message_id, recipient))`,
+];
+
+export function installRunnerSchema(db) {
+  for (const sql of SCHEMA) db.prepare(sql).run();
+  installBusIdentity(db);
+  installDeliveryStates(db);
+  installChannelDiscipline(db);
+}

package/src/runner/runnerService.js

@@ -0,0 +1,140 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { HubDbReader } from '../lib/hubDbReader.js';
+import { HubScan } from '../lib/hubScan.js';
+import { pairedStatus } from '../lib/pairedConfig.js';
+import { RUNNER_DB_FILE, RunnerStore } from '../lib/runnerStore.js';
+import { channelRoots } from '../lib/shadowService.js';
+import { UploadRedactor } from '../lib/uploadRedactor.js';
+import { RunnerBridge } from './bridge.js';
+import { forkRunner } from './runnerChild.js';
+
+export const RUNNER_CONFIG_FILE = 'runner.json';
+const RESTART_MIN_MS = 1_000;
+const RESTART_MAX_MS = 30_000;
+const SECRETS_EVERY_MS = 60_000;
+
+/** Whether this machine has cut over: the daemon runs the hub (the runner) and serves the local API (design §11). */
+export const runnerMode = (config) => config?.cloudFirst?.mode === 'runner';
+
+/**
+ * The daemon's runner (cloud-first Phase 3): after cutover it starts the runner process (the hub core over runner.db
+ * and the facade on 127.0.0.1:8890), keeps it running, and bridges it to Pulse with the daemon's own credentials.
+ * `update(config)` is called with every heartbeat's config, as the mirror and the shadow are.
+ */
+export class RunnerService {
+  constructor({
+    directory, transportFactory, home = os.homedir(), fork = forkRunner, openStore = (dir) => RunnerStore.open(dir),
+    createBridge = (options) => new RunnerBridge(options), log = () => {}, now = Date.now,
+  }) {
+    this.directory = directory;
+    this.transportFactory = transportFactory;
+    this.home = home;
+    this.fork = fork;
+    this.openStore = openStore;
+    this.createBridge = createBridge;
+    this.log = log;
+    this.now = now;
+    this.key = null;
+    this.current = null;
+    this.restarts = 0;
+    this.state = { state: 'off' };
+  }
+
+  /** Start, keep or stop the runner for this config. Never waits on the runner. */
+  update(config) {
+    const on = runnerMode(config) && pairedStatus(config).ok;
+    const key = on ? JSON.stringify({ c: config.cloudFirst, m: config.paired.machineId, v: config.paired.policyVersion }) : null;
+    if (key === this.key) return this.state;
+    this.key = key;
+    this.starting = (this.starting || Promise.resolve()).then(() => this.#stopCurrent()).then(() => (on ? this.#start(config) : null))
+      .catch((error) => {
+        this.state = { state: 'error', code: error.code || 'runner_start_failed', message: String(error.message).slice(0, 200) };
+        this.log('runner_start_failed', { code: this.state.code });
+      });
+    if (!on) this.state = { state: 'off' };
+    return this.state;
+  }
+
+  async stop() {
+    this.key = null;
+    await this.starting;
+    await this.#stopCurrent();
+    this.state = { state: 'off' };
+  }
+
+  health() {
+    const bridge = this.current?.bridge;
+    return { ...this.state, ...(bridge ? { runner: bridge.health(), lease: bridge.lease ?? null } : {}) };
+  }
+
+  async #start(config) {
+    const { cloudFirst, paired } = config;
+    const hubHome = path.resolve(cloudFirst.hubHome);
+    const dbFile = path.join(this.directory, RUNNER_DB_FILE);
+    const roots = channelRoots(config, hubHome);
+    const approvedRoots = (config.localRoots || []).map((root) => root.canonicalRoot).filter(Boolean).sort();
+    const configFile = path.join(this.directory, RUNNER_CONFIG_FILE);
+    await writeFile(configFile, `${JSON.stringify({
+      port: cloudFirst.port ?? 8890, host: '127.0.0.1', mapping: 'hub', machineId: paired.machineId, cutoverSeq: Number(cloudFirst.cutoverSeq) || 0,
+      channelRoots: roots, approvedRoots, pulseUrl: cloudFirst.pulseUrl ?? null, enableCollector: true, enableManager: true,
+      seedFrom: cloudFirst.seedFrom ?? null, seedSha256: cloudFirst.seedSha256 ?? null,
+    }, null, 2)}\n`, { mode: 0o600 });
+    const store = await this.openStore(this.directory);
+    const transport = await this.transportFactory(config);
+    const held = () => transport.auth?.heldSecrets?.() ?? [];
+    const current = { store, transport, stopping: false };
+    this.current = current;
+    const launch = async () => {
+      const runner = this.fork({ hubHome, dbFile, configFile });
+      runner.child.stderr?.on('data', (chunk) => this.log('runner_stderr', { line: String(chunk).slice(0, 300) }));
+      const ready = await runner.ready;
+      const hubToken = (await readFile(path.join(hubHome, 'state', 'hub.token'), 'utf8').catch(() => '')).trim() || null;
+      const fileScan = new HubScan({
+        store, reader: new HubDbReader({ hubDir: hubHome, file: dbFile }), hubDir: hubHome, roots,
+        redaction: new UploadRedactor({ workspace: path.dirname(hubHome), home: this.home, approvedRoots }),
+        secrets: () => [hubToken, ...held()],
+      });
+      const bridge = this.createBridge({ runner, transport, store, machineId: paired.machineId, fileScan, log: this.log });
+      bridge.start();
+      runner.send({ type: 'secrets', values: held() });
+      const secrets = setInterval(() => runner.send({ type: 'secrets', values: held() }), SECRETS_EVERY_MS);
+      secrets.unref?.();
+      current.runner = runner;
+      current.bridge = bridge;
+      current.secrets = secrets;
+      this.restarts = 0;
+      this.state = { state: 'running', port: ready.port, pid: ready.pid, since: new Date(this.now()).toISOString() };
+      this.log('runner_started', { port: ready.port, pid: ready.pid });
+      runner.child.once('exit', (code, signal) => {
+        clearInterval(secrets);
+        void bridge.stop();
+        if (current.stopping || this.current !== current) return;
+        this.restarts += 1;
+        const waitMs = Math.min(RESTART_MAX_MS, RESTART_MIN_MS * 2 ** Math.min(5, this.restarts - 1));
+        this.state = { state: 'restarting', code: 'runner_exited', exit: signal || code, restarts: this.restarts, retryInMs: waitMs };
+        this.log('runner_exited', { exit: signal || code, restarts: this.restarts });
+        void sleep(waitMs).then(() => (current.stopping || this.current !== current ? null : launch()))
+          .catch((error) => { this.state = { state: 'error', code: error.code || 'runner_start_failed', message: String(error.message).slice(0, 200) }; });
+      });
+    };
+    await launch();
+  }
+
+  async #stopCurrent() {
+    const current = this.current;
+    if (!current) return;
+    current.stopping = true;
+    this.current = null;
+    clearInterval(current.secrets);
+    await current.bridge?.stop();
+    if (current.runner?.child && current.runner.child.exitCode === null) {
+      const exited = new Promise((resolve) => current.runner.child.once('exit', resolve));
+      current.runner.child.kill('SIGTERM');
+      await Promise.race([exited, sleep(5_000)]);
+    }
+    current.store?.close();
+  }
+}

package/src/runner/spool.js

@@ -0,0 +1,86 @@
+import { createHash } from 'node:crypto';
+import { readFileSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+
+const SPOOL_FILE = /^\d{8}\.jsonl$/;
+
+/** When the facade last registered this address (runner.db only; a restored hub.db has no such record). */
+function registeredOnline(db, id) {
+  const known = db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type = 'table' AND name = 'pulse_registered'").get();
+  return known ? db.prepare('SELECT at FROM pulse_registered WHERE id = ?').get(id) : null;
+}
+
+/**
+ * Applies one spooled write to a hub, the way the facade route would have, under the id the client printed: a
+ * publish keeps its edge id (published again it returns the stored envelope), an ack is idempotent and gives the
+ * deterministic reply id, a register is an upsert. The same function runs in the runner and in the undo down-sync.
+ */
+export function applySpooled(hub, { op, body, at = 0 }) {
+  const b = body && typeof body === 'object' ? body : {};
+  switch (op) {
+    case 'publish':
+      return hub.messaging.publish({
+        id: b.id, from: b.from, fromName: b.fromName, topic: b.topic, type: b.type || 'MESSAGE', body: b.body, needsAck: b.needsAck,
+        ttlMs: b.ttlMs, replyTo: b.replyTo,
+      }, hub.registry.directory());
+    case 'ack':
+      return hub.messaging.ack(String(b.as || ''), String(b.id || ''), { reply: b.reply || b.body, fromName: b.fromName, reaction: b.reaction }, hub.registry.directory());
+    case 'register': {
+      // A registration made online after this line was spooled is newer; the line doesn't put back older topics.
+      const online = registeredOnline(hub.db, String(b.id || ''));
+      if (online && Number(online.at) > Number(at)) return { superseded: true };
+      return hub.registry.register({ ...b, id: String(b.id || '') });
+    }
+    default:
+      return { error: `unknown spool op ${String(op).slice(0, 40)}` };
+  }
+}
+
+/** Complete lines only: a torn last line (no newline yet) waits for the next pass. */
+function completeLines(text) {
+  const end = text.lastIndexOf('\n');
+  return end < 0 ? [] : text.slice(0, end).split('\n');
+}
+
+/**
+ * Imports every spool line not imported before, oldest file first, each in its own capture transaction with its
+ * pulse_spool row, so a crash between lines imports the rest exactly once on the next pass.
+ */
+export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () => {} }) {
+  let files;
+  try {
+    files = readdirSync(spoolDir).filter((f) => SPOOL_FILE.test(f)).sort();
+  } catch (error) {
+    if (error.code === 'ENOENT') return { imported: 0, refused: 0 };
+    throw error;
+  }
+  const seen = hub.db.prepare('SELECT 1 AS found FROM pulse_spool WHERE sha256 = ?');
+  const record = hub.db.prepare('INSERT INTO pulse_spool (sha256, file, line, outcome, imported_at) VALUES (?, ?, ?, ?, ?)');
+  let imported = 0;
+  let refused = 0;
+  for (const file of files) {
+    const lines = completeLines(readFileSync(path.join(spoolDir, file), 'utf8'));
+    lines.forEach((line, index) => {
+      if (!line.trim()) return;
+      const sha = createHash('sha256').update(line).digest('hex');
+      if (seen.get(sha)) return;
+      capture.tx(() => {
+        let outcome;
+        try {
+          const entry = JSON.parse(line);
+          const result = entry && entry.v === 1 ? applySpooled(hub, entry) : { error: 'unknown spool version' };
+          outcome = result && result.error ? `refused: ${String(result.error).slice(0, 80)}` : result && result.superseded ? 'superseded' : 'applied';
+        } catch {
+          outcome = 'refused: unreadable line';
+        }
+        record.run(sha, file, index + 1, outcome, now());
+        if (outcome === 'applied') imported += 1;
+        else {
+          refused += 1;
+          log('spool_line_refused', { file, line: index + 1, outcome });
+        }
+      }, { label: 'spool import' });
+    });
+  }
+  return { imported, refused };
+}

package/src/runner/spoolInto.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+// Rollback's last step before the old hub starts again (ADR-4, A3): every write the thin clients spooled is applied to
+// the restored hub.db by its edge id, with the ported 6b2fb0b core, so nothing an agent was told was sent goes
+// missing. Run with PULSE_HUB_HOME set: `node spoolInto.js <hub.db>`. Prints `{ imported, refused }`.
+import path from 'node:path';
+import { withTx } from '../hub/core/db.mjs';
+import { openHub } from '../hub/core/hub.mjs';
+import { hubHome } from '../hub/home.mjs';
+import { importSpool } from './spool.js';
+
+const file = process.argv[2];
+if (!file) {
+  process.stderr.write('usage: spoolInto.js <hub.db>\n');
+  process.exit(2);
+}
+const stateDir = path.join(hubHome(), 'state');
+const hub = openHub(stateDir, path.resolve(file));
+hub.db.prepare('CREATE TABLE IF NOT EXISTS pulse_spool (sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, line INTEGER NOT NULL, outcome TEXT NOT NULL, imported_at INTEGER NOT NULL)').run();
+const capture = { tx: (fn, { label = 'spool into hub.db' } = {}) => withTx(hub.db, fn, label) };
+const result = importSpool({ hub, capture, spoolDir: path.join(stateDir, 'spool') });
+hub.db.close();
+process.stdout.write(`${JSON.stringify(result)}\n`);

package/src/runner/statusPage.js

@@ -0,0 +1,30 @@
+const escape = (value) => String(value ?? '')
+  .replaceAll('&', '&')
+  .replaceAll('<', '&lt;')
+  .replaceAll('>', '&gt;')
+  .replaceAll('"', '&quot;');
+
+const row = (label, value) => `<tr><th>${escape(label)}</th><td>${escape(value)}</td></tr>`;
+
+/** What 127.0.0.1:8890 shows while Pulse is out of reach (D11): the measured health and what is held here. */
+export function statusPage({ health, pulseUrl }) {
+  const h = health || {};
+  const queue = h.outbox || {};
+  return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><title>Pulse runner</title>
+<style>body{font:15px/1.5 system-ui,sans-serif;margin:3rem auto;max-width:36rem;color:#1d2433}th{text-align:left;padding-right:1.5rem;font-weight:600}td,th{padding:.2rem 0}</style>
+</head><body>
+<h1>Pulse runner</h1>
+<p>${escape(h.summary || 'The local Agent Hub moved into Pulse. Agents on this machine keep working; changes wait here until Pulse is reachable.')}</p>
+<table>
+${row('State', h.state || 'unknown')}
+${row('Reason', h.reason || '')}
+${h.fix ? row('Fix', h.fix) : ''}
+${row('Last sync', h.lastSyncAt ? new Date(h.lastSyncAt).toISOString() : 'never')}
+${row('Waiting to send', queue.pending ?? 0)}
+${row('Set aside', (queue.dead ?? 0) + (h.quarantined ?? 0))}
+</table>
+${pulseUrl ? `<p><a href="${escape(pulseUrl)}">Open Pulse</a></p>` : ''}
+</body></html>
+`;
+}