@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.3

package/src/lib/cloudFirstCommands.js

@@ -0,0 +1,293 @@
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { HubError, scopeBlock } from './errors.js';
+import { HubDbReader } from './hubDbReader.js';
+import { HubScan } from './hubScan.js';
+import { discoverLocalHub, LocalHubReader } from './localHub.js';
+import { assertNodeFloor } from './nodeFloor.js';
+import { pairedStatus } from './pairedConfig.js';
+import { reconcilePasses } from './reconcile.js';
+import { REDACTION_ROOTS_META, RunnerStore } from './runnerStore.js';
+import {
+  channelRoots, IMPORT_REQUEST_FILE, importEnabled, PARITY_LOG_FILE, PARITY_REQUEST_FILE, RECONCILE_REQUEST_FILE, shadowEnabled,
+} from './shadowService.js';
+import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
+import { UploadRedactor } from './uploadRedactor.js';
+
+const POLL_MS = 1_000;
+
+/** Per-source rows of a scan report, for the dry-run table. */
+export function sourceRows(report) {
+  return Object.entries(report.sources).map(([source, stats]) => ({
+    source,
+    read: stats.read,
+    planned: stats.planned,
+    unchanged: stats.unchanged,
+    skipped: stats.skipped,
+    refused: stats.refused,
+    events: stats.events,
+    redactions: stats.redactions,
+    cut: stats.cut,
+    vanished: stats.vanished,
+  }));
+}
+
+function delta(before = {}, after = {}) {
+  const out = { written: 0, duplicate: 0, dead: 0 };
+  for (const source of new Set([...Object.keys(before), ...Object.keys(after)])) {
+    const was = before[source] || { written: 0, duplicate: 0, dead: {} };
+    const now = after[source] || { written: 0, duplicate: 0, dead: {} };
+    out.written += now.written - was.written;
+    out.duplicate += now.duplicate - was.duplicate;
+    out.dead += Object.values(now.dead).reduce((n, v) => n + v, 0) - Object.values(was.dead).reduce((n, v) => n + v, 0);
+  }
+  return out;
+}
+
+/**
+ * SCOPE-02: the daemon is blocked by a missing scope or grant since this request: say so now, with the fix, instead of
+ * waiting out the timeout while it retries.
+ */
+function blockedForScopes(state, requestedAt) {
+  if (state?.state !== 'blocked' || !(state.at >= requestedAt) || !state.fix) return;
+  const block = scopeBlock({ code: state.code, details: { missingScopes: state.missingScopes } });
+  if (!block) return;
+  const missing = block.missingScopes.length ? `lacks ${block.missingScopes.join(', ')}` : `was refused (${block.code})`;
+  throw new HubError(block.code, `Pulse is blocked: this machine's sign-in or grant ${missing}. Run \`${block.fix}\`, then try again.`, {
+    details: { missingScopes: block.missingScopes, fix: block.fix },
+  });
+}
+
+/**
+ * The `pulse import`, `pulse shadow` and `pulse parity` commands. Only the dry run reads the hub here; a real import,
+ * its reconcile, the shadow and parity checks run in the daemon on its own connector session, asked for through
+ * request files in the config directory and followed through runner.db.
+ */
+export class CloudFirstCommands {
+  constructor({ configDir, now = Date.now, wait = sleep, home = os.homedir(), discover = discoverLocalHub, tmpDir = os.tmpdir() }) {
+    this.configDir = configDir;
+    this.configFile = path.join(configDir, 'config.json');
+    this.now = now;
+    this.wait = wait;
+    this.home = home;
+    this.discover = discover;
+    this.tmpDir = tmpDir;
+  }
+
+  async #config() {
+    const config = await readOwnerOnlyJson(this.configFile);
+    if (!config) throw new HubError('HUB_NOT_INSTALLED', 'Pulse is not installed here; run `agentpulse pulse install`');
+    return config;
+  }
+
+  /**
+   * Scan every source as an import would, against a copy of runner.db (or an empty store), and report what it would
+   * send. Reads the hub only; writes nothing to it, to runner.db or to Pulse.
+   */
+  async dryRun() {
+    assertNodeFloor();
+    const config = await this.#config();
+    const hub = await this.discover({ config });
+    const scratch = await mkdtemp(path.join(this.tmpDir, 'pulse-import-dry-'));
+    try {
+      const copied = RunnerStore.copy(this.configDir, path.join(scratch, 'runner.db'));
+      const store = await RunnerStore.open(scratch);
+      try {
+        const token = await new LocalHubReader({ dir: hub.dir, port: hub.port }).tokenValue().catch(() => null);
+        const redaction = new UploadRedactor({
+          workspace: path.dirname(hub.dir), home: this.home,
+          approvedRoots: (config.localRoots || []).map((root) => root.canonicalRoot).filter(Boolean).sort(),
+        });
+        redaction.seed(store.meta(REDACTION_ROOTS_META));
+        const scan = new HubScan({
+          store,
+          reader: new HubDbReader({ hubDir: hub.dir }),
+          hubDir: hub.dir,
+          roots: channelRoots(config, hub.dir),
+          redaction,
+          secrets: () => [token],
+          now: this.now,
+        });
+        const report = await scan.run({ mode: 'import' });
+        return {
+          command: 'import',
+          dryRun: true,
+          againstExistingStore: copied,
+          totals: report.totals,
+          sources: sourceRows(report),
+          presenceBeats: report.presence.length,
+          message: `Dry run: ${report.totals.read} rows read, ${report.totals.planned} would send ${report.totals.events} events `
+            + `(${report.totals.unchanged} unchanged, ${report.totals.skipped} skipped, ${report.totals.refused} refused, `
+            + `${report.totals.redactions} redactions). ${report.totals.balanced ? 'Every row is accounted for.' : 'ROWS DO NOT ADD UP.'}`,
+        };
+      } finally {
+        store.close();
+      }
+    } finally {
+      await rm(scratch, { recursive: true, force: true });
+    }
+  }
+
+  async #importConfig() {
+    const config = await this.#config();
+    if (!pairedStatus(config).ok) throw new HubError('HUB_NOT_PAIRED', 'Pair this machine first: `agentpulse pulse pair`');
+    if (!importEnabled(config)) {
+      await writeOwnerOnlyJson(this.configFile, { ...config, cloudFirst: { ...(config.cloudFirst || {}), import: true } });
+    }
+  }
+
+  /**
+   * Ask the daemon for an import and follow it until it has fully drained (or only queue it, with `wait: false`). The
+   * daemon snapshots the sources first; `reuseSnapshot` imports the last snapshot again (a re-run that must write 0).
+   */
+  async import({ wait = true, reuseSnapshot = false, timeoutMs = 60 * 60_000, onProgress = () => {} } = {}) {
+    assertNodeFloor();
+    await this.#importConfig();
+    const requestedAt = this.now();
+    await writeOwnerOnlyJson(path.join(this.configDir, IMPORT_REQUEST_FILE), { requestedAt, reuseSnapshot });
+    if (!wait) return { command: 'import', requestedAt, state: 'requested', message: 'Import requested; the daemon runs it on its next pass.' };
+    const deadline = this.now() + timeoutMs;
+    for (;;) {
+      const store = RunnerStore.openReadOnly(this.configDir);
+      const job = store?.meta('import');
+      const state = store?.meta('shadow_state');
+      store?.close();
+      if (job?.requestedAt === requestedAt && job.phase === 'done') {
+        const sent = delta(job.before, job.after);
+        return {
+          command: 'import',
+          requestedAt,
+          state: 'done',
+          snapshot: job.snapshot ? { takenAt: new Date(job.snapshot.takenAt).toISOString(), files: job.snapshot.files, reused: job.reusedSnapshot } : null,
+          scan: job.scan?.totals,
+          cloud: sent,
+          message: `Import done${job.reusedSnapshot ? ' (same snapshot)' : ''}: ${job.scan?.totals?.planned ?? 0} rows planned, `
+            + `${sent.written} events written, ${sent.duplicate} already in Pulse, ${sent.dead} set aside.`,
+        };
+      }
+      blockedForScopes(state, requestedAt);
+      if (state?.state === 'blocked' || state?.state === 'error') onProgress({ state: state.state, code: state.code });
+      else onProgress({ state: job?.requestedAt === requestedAt ? job.phase : 'waiting for the daemon', pending: state?.pending ?? null });
+      if (this.now() >= deadline) throw new HubError('HUB_IMPORT_TIMEOUT', 'The import did not finish in time; it keeps running in the daemon. See `agentpulse pulse shadow status`.');
+      await this.wait(POLL_MS);
+    }
+  }
+
+  /**
+   * ADR-4: ask the daemon to rescan the last import's snapshot and compare it with the cloud's view of this machine's
+   * log; returns the report, with `exitCode` 1 unless every family, channel and conversation is equal or differs only
+   * by retired channels and redacted items (CUT-02).
+   */
+  async reconcile({ timeoutMs = 10 * 60_000, onProgress = () => {} } = {}) {
+    assertNodeFloor();
+    await this.#importConfig();
+    const requestedAt = this.now();
+    await writeOwnerOnlyJson(path.join(this.configDir, RECONCILE_REQUEST_FILE), { requestedAt });
+    const deadline = this.now() + timeoutMs;
+    for (;;) {
+      const store = RunnerStore.openReadOnly(this.configDir);
+      const last = store?.meta('reconcile_last');
+      const state = store?.meta('shadow_state');
+      store?.close();
+      if (last?.requestedAt === requestedAt) {
+        return { command: 'import', reconcile: true, ...last, exitCode: reconcilePasses(last.status) ? 0 : 1, message: reconcileLine(last) };
+      }
+      blockedForScopes(state, requestedAt);
+      onProgress({ state: state?.state ?? 'waiting for the daemon', code: state?.code ?? null });
+      if (this.now() >= deadline) throw new HubError('HUB_RECONCILE_TIMEOUT', 'The daemon did not finish the reconcile in time; see `agentpulse pulse shadow status`.');
+      await this.wait(POLL_MS);
+    }
+  }
+
+  async shadow({ action }) {
+    const config = await this.#config();
+    switch (action) {
+      case 'on':
+      case 'off': {
+        if (action === 'on') assertNodeFloor();
+        const enable = action === 'on';
+        await writeOwnerOnlyJson(this.configFile, { ...config, cloudFirst: { ...(config.cloudFirst || {}), shadow: enable } });
+        return {
+          command: 'shadow',
+          enabled: enable,
+          message: enable
+            ? 'Shadow feed on: once your team has Pulse cloud-first on, the daemon imports on `agentpulse pulse import` and then tails the local hub into Pulse.'
+            : 'Shadow feed off: nothing more goes to the Pulse log from this machine. The mirror is unchanged.',
+        };
+      }
+      case 'status': {
+        const store = RunnerStore.openReadOnly(this.configDir);
+        const state = store?.meta('shadow_state') ?? { state: shadowEnabled(config) ? 'starting' : 'off' };
+        const out = {
+          command: 'shadow',
+          enabled: shadowEnabled(config),
+          state,
+          importedAt: store?.meta('imported_at') ?? null,
+          import: store?.meta('import') ?? null,
+          pending: store ? store.pendingCount() : 0,
+          counts: store ? store.counts() : {},
+          setAside: store ? store.deadRows({ limit: 10 }) : [],
+          parity: store?.meta('parity_last') ?? null,
+          reconcile: store?.meta('reconcile_last') ?? null,
+        };
+        store?.close();
+        out.message = `Shadow feed is ${out.enabled ? state.state : 'off'}${state.code ? ` (${state.code})` : ''}; ${out.pending} commands waiting.`;
+        return out;
+      }
+      default:
+        throw new HubError('HUB_SHADOW_ACTION_INVALID', `Unknown shadow action: ${action}`);
+    }
+  }
+
+  /** The last parity check, its history, or a fresh one (`now`), which the daemon runs and records. */
+  async parity({ hours = 6, now = false, timeoutMs = 120_000 } = {}) {
+    if (now) {
+      const requestedAt = this.now();
+      await writeOwnerOnlyJson(path.join(this.configDir, PARITY_REQUEST_FILE), { requestedAt, hours });
+      const deadline = this.now() + timeoutMs;
+      for (;;) {
+        const store = RunnerStore.openReadOnly(this.configDir);
+        const last = store?.meta('parity_last');
+        store?.close();
+        if (last?.requestedAt === requestedAt) return { command: 'parity', ...last, message: parityLine(last) };
+        if (this.now() >= deadline) throw new HubError('HUB_PARITY_TIMEOUT', 'The daemon did not run the parity check in time; is the shadow feed running?');
+        await this.wait(POLL_MS);
+      }
+    }
+    const store = RunnerStore.openReadOnly(this.configDir);
+    const last = store?.meta('parity_last') ?? null;
+    store?.close();
+    let history = [];
+    try {
+      history = (await readFile(path.join(this.configDir, PARITY_LOG_FILE), 'utf8')).split('\n').filter(Boolean).map((line) => JSON.parse(line))
+        .filter((line) => line.at >= this.now() - hours * 3_600_000);
+    } catch {
+      history = [];
+    }
+    return {
+      command: 'parity',
+      last,
+      history,
+      message: last ? `${parityLine(last)} ${history.length} checks in the last ${hours} h, ${history.filter((line) => line.status === 'pass').length} at 100 %.` : 'No parity check has run yet.',
+    };
+  }
+}
+
+const VERDICT = Object.freeze({ equal: 'EQUAL', explained: 'EQUAL (retired and redacted listed)', mismatch: 'MISMATCH' });
+
+export function reconcileLine(report) {
+  if (report.status === 'no_import') return report.message;
+  const families = Object.values(report.families || {});
+  const equal = families.filter((row) => row.equal).length;
+  const classes = report.classes ? Object.entries(report.classes).map(([name, n]) => `${name} ${n}`).join(', ') : null;
+  return `Reconcile ${VERDICT[report.status] ?? 'MISMATCH'}: ${equal} of ${families.length} families equal, `
+    + `${report.mismatches.length} mismatches${classes ? ` (by class: ${classes})` : ''}; ${report.withheld.length} withheld and ${report.cut.length} cut, by key in the report; `
+    + `Pulse head ${report.head}, ${report.otherWriters?.entries ?? 0} entries by other writers not compared.`;
+}
+
+export function parityLine(report) {
+  if (!report) return 'No parity report.';
+  return `Parity ${report.status} at ${report.percent ?? '—'} % over ${report.hours} h (${new Date(report.at).toISOString()}).`;
+}

package/src/lib/contract.sha256

@@ -1,2 +1,2 @@
-88e39539d7d2b1775b4970a8aeed416ae6d7203c53df143093ea54d2014cdd31  connector-contract.json
+ce1142e1e133c8aff45a999d0d4b67faa6177f420ddcd1b449af81f8e806d8d0  connector-contract.json
 7b7a59c08797c100b46436123dfcb09b517730357fa90cd0890da9b80308faf3  mirror-contract.json