@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.3

package/src/lib/uploadRedactor.js

@@ -0,0 +1,102 @@
+import path from 'node:path';
+import { redactSecretSubtree, secretKeyReplacement } from './mirrorContract.js';
+import { createPathRedactor } from './pathRedaction.js';
+import { redactSecrets } from './redaction.js';
+
+const SEEN_MAX = 100_000;
+// The line net can fire again on text a first pass redacted (a marker earlier on the line); a stable redaction repeats
+// the passes until nothing changes, so the cloud's own pass over it changes nothing either.
+const STABLE_PASSES = 4;
+
+/**
+ * What leaves this machine for Pulse passes through here, for the mirror and the Pulse log alike: the contract's
+ * secret passes first, then path redaction, on every string and key, so a secret inside a path is never cut down to
+ * a last segment that still holds it, and nothing is projected, truncated or hashed before both have run. Repository
+ * roots are learned from the deliveries the hub reports; the redacted form of a value depends on those roots, the
+ * workspace, the home folder and the held secrets, and on nothing else.
+ */
+export class UploadRedactor {
+  constructor({ workspace, home, approvedRoots = [] }) {
+    this.workspace = workspace;
+    this.home = home;
+    this.repoRoots = new Map(approvedRoots.map((root) => [root, path.basename(root)]));
+    this.worktreeRoots = new Map();
+  }
+
+  learnRepos(deliveries) {
+    for (const stream of deliveries.flatMap((delivery) => delivery.streams || [])) {
+      const git = stream.git || {};
+      if (typeof git.mainRepo === 'string' && path.isAbsolute(git.mainRepo)) this.repoRoots.set(git.mainRepo, path.basename(git.mainRepo));
+      for (const extra of [git.worktreePath, git.channelCheckout]) {
+        if (typeof extra === 'string' && path.isAbsolute(extra) && !this.repoRoots.has(extra)) this.worktreeRoots.set(extra, path.basename(extra));
+      }
+    }
+  }
+
+  /**
+   * The learned roots, to persist: a worktree whose delivery is gone must keep redacting as it did, or a rescan of the
+   * same text gives another body than the one already sent (CUT-02).
+   */
+  roots() {
+    return { repo: [...this.repoRoots], worktree: [...this.worktreeRoots] };
+  }
+
+  /** Roots persisted by an earlier scan, added to what this one learns. */
+  seed(roots) {
+    for (const [root, name] of Array.isArray(roots?.repo) ? roots.repo : []) {
+      if (typeof root === 'string' && path.isAbsolute(root) && typeof name === 'string') this.repoRoots.set(root, name);
+    }
+    for (const [root, name] of Array.isArray(roots?.worktree) ? roots.worktree : []) {
+      if (typeof root === 'string' && path.isAbsolute(root) && typeof name === 'string' && !this.repoRoots.has(root)) this.worktreeRoots.set(root, name);
+    }
+  }
+
+  /** Everything the redacted form depends on besides the input, as one string (for memo keys and digests). */
+  basis(secrets) {
+    return JSON.stringify([[...this.repoRoots], [...this.worktreeRoots], this.workspace, this.home, secrets]);
+  }
+
+  /** `stable` (the Pulse log) repeats the secret passes until a pass changes nothing; the mirror keeps one pass. */
+  redactor(secrets, { stable = false } = {}) {
+    const repos = [...this.repoRoots, ...this.worktreeRoots].map(([root, name]) => ({ root, name }));
+    const paths = createPathRedactor({ repos, workspace: this.workspace, home: this.home });
+    const seen = new Map();
+    let redactions = 0;
+    const redactString = (input) => {
+      const original = String(input ?? '');
+      let done = seen.get(original);
+      if (done === undefined) {
+        let secret = redactSecrets(original, { secrets });
+        let count = secret.redactions;
+        for (let pass = 1; stable && pass < STABLE_PASSES; pass += 1) {
+          const again = redactSecrets(secret.text, { secrets });
+          if (again.text === secret.text) break;
+          count += again.redactions;
+          secret = again;
+        }
+        done = { text: paths.redactString(secret.text), redactions: count };
+        if (seen.size >= SEEN_MAX) seen.clear();
+        seen.set(original, done);
+      }
+      redactions += done.redactions;
+      return done.text;
+    };
+    const entry = (key, item) => {
+      const replaced = secretKeyReplacement(key, item);
+      if (replaced === undefined || replaced === null) return redactSecretSubtree(key, item, redactString);
+      redactions += 1;
+      return replaced;
+    };
+    const redactValue = (value, { keep = [] } = {}) => paths.redactValue(value, { keep, redactString, entry });
+    return {
+      redactString,
+      redactValue,
+      /** Secret replacements made since the last call (memoised strings count each time they are used). */
+      takeRedactions: () => {
+        const count = redactions;
+        redactions = 0;
+        return count;
+      },
+    };
+  }
+}

package/src/runner/applier.js

@@ -0,0 +1,297 @@
+import { acceptDelivery } from '../hub/lib/store.mjs';
+import { applyRemotePresence } from './busIdentity.js';
+
+/** How far a recipient row has got; a cloud receipt moves a local row only forward. `leased` stays on the machine. */
+const RANK = { queued: 0, leased: 0, delivered: 1, acked: 2, expired: 3 };
+const RECEIPT_STATE = { 'receipt.delivered': 'delivered', 'receipt.acked': 'acked', 'receipt.expired': 'expired' };
+
+/** Cloud types this machine keeps no local copy of; recorded as seen, never quarantined. */
+const NOT_LOCAL = new Set([
+  'project.created', 'project.updated', 'project.removed', 'project.member_added', 'project.role_changed', 'project.member_removed',
+  'rules.pack_enabled', 'rules.pack_disabled', 'rules.rule_toggled', 'team.flag_set', 'persona.updated', 'channel.created',
+  'post.edited', 'post.removed', 'decision.recorded',
+]);
+
+const json = (value) => (value === undefined || value === null ? null : typeof value === 'string' ? value : JSON.stringify(value));
+
+/** The ordering group a local change to this event's entity is queued under (Phase 2's grp), or null. */
+function entityGroup(event) {
+  const p = event.payload || {};
+  switch (event.type) {
+    case 'actor.registered':
+    case 'actor.updated':
+      return `actor:${p.id}`;
+    case 'actor.row_updated':
+      return `fact:agent-row:${p.id}`;
+    case 'delivery.updated':
+      return `fact:delivery:${p.id}`;
+    case 'stream.updated':
+      return `fact:stream:${p.key}`;
+    case 'request.created':
+      return `fact:request:${p.id}`;
+    case 'delivery.accepted':
+      return `fact:accepted:${p.id}`;
+    case 'status.overridden':
+      return `fact:status_override:${p.key}`;
+    case 'settings.changed':
+      return `fact:settings:${p.key}`;
+    default:
+      return null;
+  }
+}
+
+/**
+ * Applies `/connector/sync` pages to runner.db (design §4.5, §6.3): one transaction per page, each event in its own
+ * savepoint, with capture off so nothing applied here is sent back, and the cursor advanced in the same transaction.
+ *
+ * - An event is skipped as this machine's own when its key is already known (sent, pending or dead here), or when it
+ *   came from this machine at or below the cutover seq, whose content the seed already holds (N3).
+ * - A change to an entity whose own newer version is still in the outbox is deferred until that version is acked,
+ *   then applied only if it is later (M1).
+ * - An event that cannot be applied goes to pulse_quarantine with its reason; the cursor still moves past it (I1).
+ */
+export function createApplier({ runner, machineId, cutoverSeq = () => 0, now = Date.now, log = () => {}, checkpoint = () => {} }) {
+  /**
+   * Every machine id this hub's content was sent up under: today's, and those the import and the shadow ran as before a
+   * re-pair (`origin_machines`, recorded at cutover). Up to the cutover seq, their events are already in the seed.
+   */
+  const ownMachines = () => new Set([machineId, ...(runner.store.meta('origin_machines') || [])].filter(Boolean));
+  const { hub, store, capture, mapping } = runner;
+  const { db } = hub;
+  const q = {
+    seen: db.prepare('SELECT 1 AS found FROM pulse_events WHERE seq = ? OR id = ?'),
+    record: db.prepare('INSERT OR IGNORE INTO pulse_events (seq, id, type, key, origin, outcome, applied_at) VALUES (?, ?, ?, ?, ?, ?, ?)'),
+    quarantine: db.prepare('INSERT OR IGNORE INTO pulse_quarantine (seq, id, type, reason, payload, at) VALUES (?, ?, ?, ?, ?, ?)'),
+    defer: db.prepare('INSERT OR IGNORE INTO pulse_deferred (seq, grp, event, at) VALUES (?, ?, ?, ?)'),
+    deferred: db.prepare('SELECT seq, grp, event FROM pulse_deferred ORDER BY seq'),
+    undefer: db.prepare('DELETE FROM pulse_deferred WHERE seq = ?'),
+    pending: db.prepare('SELECT 1 AS found FROM outbox WHERE grp = ? LIMIT 1'),
+    ackedSeq: db.prepare("SELECT MAX(CAST(s.seq AS INTEGER)) AS seq FROM sent s JOIN version v ON v.key = s.key WHERE v.entity = ?"),
+    sent: db.prepare("INSERT INTO sent (key, source, type, seq, duplicate, at) VALUES (?, 'sync', ?, ?, 0, ?) ON CONFLICT (key) DO NOTHING"),
+    cursor: db.prepare('UPDATE pulse_sync SET applied_seq = ?, head = ?, floor = ?, last_ok_at = ?, last_error = NULL WHERE one = 1'),
+    message: db.prepare(`INSERT OR IGNORE INTO messages (id, sender, topic, type, body, needs_reply, expires_at, reply_to, reaction, from_name, created_at, meta)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
+    delivery: db.prepare("INSERT OR IGNORE INTO deliveries (message_id, recipient, state, queued_at, attempts) VALUES (?, ?, 'queued', ?, 0)"),
+    receipt: db.prepare('SELECT state FROM deliveries WHERE message_id = ? AND recipient = ?'),
+    messageExists: db.prepare('SELECT 1 AS found FROM messages WHERE id = ?'),
+    fact: db.prepare(`INSERT INTO facts (kind, key, payload, updated_at) VALUES (?, ?, ?, ?)
+      ON CONFLICT (kind, key) DO UPDATE SET payload = excluded.payload, updated_at = excluded.updated_at`),
+    factOf: db.prepare('SELECT payload FROM facts WHERE kind = ? AND key = ?'),
+    dropFact: db.prepare('DELETE FROM facts WHERE kind = ? AND key = ?'),
+    event: db.prepare('INSERT INTO events (at, kind, actor, payload) VALUES (?, ?, ?, ?)'),
+  };
+  const touched = [];
+  const adopt = (change) => touched.push(change);
+
+  function setFact(kind, key, payload, at) {
+    q.fact.run(kind, key, json(payload), Number(at) || now());
+    adopt({ tbl: 'facts', k1: kind, k2: key });
+  }
+
+  function receipt(event) {
+    const p = event.payload;
+    const target = RECEIPT_STATE[event.type];
+    const current = q.receipt.get(p.messageId, p.recipient);
+    if (!current) {
+      if (!q.messageExists.get(p.messageId)) throw Object.assign(new Error(`receipt for a message this machine never had: ${p.messageId}`), { reason: 'message_missing' });
+      q.delivery.run(p.messageId, p.recipient, Number(p.at) || now());
+    } else if (RANK[current.state] >= RANK[target] || current.state === 'expired') {
+      return 'not_forward';
+    }
+    const at = Number(p.at) || now();
+    if (target === 'delivered') {
+      db.prepare(`UPDATE deliveries SET state = 'delivered', delivered_at = COALESCE(delivered_at, ?), delivered_via = COALESCE(?, delivered_via),
+        lease_token = NULL, lease_until = NULL WHERE message_id = ? AND recipient = ?`).run(at, p.via ?? null, p.messageId, p.recipient);
+    } else if (target === 'acked') {
+      db.prepare(`UPDATE deliveries SET state = 'acked', acked_at = ?, delivered_at = COALESCE(delivered_at, ?), awaiting_reply = 0,
+        lease_token = NULL, lease_until = NULL WHERE message_id = ? AND recipient = ?`).run(at, at, p.messageId, p.recipient);
+    } else {
+      db.prepare("UPDATE deliveries SET state = 'expired', lease_token = NULL, lease_until = NULL WHERE message_id = ? AND recipient = ?").run(p.messageId, p.recipient);
+    }
+    adopt({ tbl: 'deliveries', k1: p.messageId, k2: p.recipient });
+    return 'applied';
+  }
+
+  function upsertActor(actor) {
+    db.prepare(`INSERT INTO actors (id, node_type, role, parent_id, delivery_id, stream_key, cursor_title, prompt_title, persona_name,
+        persona_first, persona_username, persona_seed, persona_json, source, topics, started_at, last_at, tokens_est, msgs_in, msgs_out,
+        repo, channel, synthetic, extra)
+      VALUES (@id, @node_type, @role, @parent_id, @delivery_id, @stream_key, @cursor_title, @prompt_title, @persona_name, @persona_first,
+        @persona_username, @persona_seed, @persona_json, @source, @topics, @started_at, @last_at, @tokens_est, @msgs_in, @msgs_out, @repo,
+        @channel, @synthetic, @extra)
+      ON CONFLICT (id) DO UPDATE SET node_type = excluded.node_type, role = excluded.role, parent_id = excluded.parent_id,
+        delivery_id = excluded.delivery_id, stream_key = excluded.stream_key, cursor_title = excluded.cursor_title,
+        prompt_title = excluded.prompt_title, persona_name = excluded.persona_name, persona_first = excluded.persona_first,
+        persona_username = excluded.persona_username, persona_seed = excluded.persona_seed, persona_json = excluded.persona_json,
+        source = excluded.source, topics = excluded.topics, started_at = excluded.started_at,
+        last_at = MAX(COALESCE(actors.last_at, 0), COALESCE(excluded.last_at, 0)), tokens_est = excluded.tokens_est,
+        msgs_in = excluded.msgs_in, msgs_out = excluded.msgs_out, repo = excluded.repo, channel = excluded.channel,
+        synthetic = excluded.synthetic, extra = excluded.extra`).run({
+      id: actor.id, node_type: actor.node_type, role: actor.role ?? null, parent_id: actor.parent_id ?? null, delivery_id: actor.delivery_id ?? null,
+      stream_key: actor.stream_key ?? null, cursor_title: actor.cursor_title ?? null, prompt_title: actor.prompt_title ?? null,
+      persona_name: actor.persona_name ?? null, persona_first: actor.persona_first ?? null, persona_username: actor.persona_username ?? null,
+      persona_seed: actor.persona_seed ?? null, persona_json: json(actor.persona), source: actor.source || 'registered',
+      topics: json(actor.topics ?? []), started_at: actor.started_at ?? null, last_at: actor.last_at ?? null, tokens_est: actor.tokens_est ?? null,
+      msgs_in: actor.msgs_in ?? 0, msgs_out: actor.msgs_out ?? 0, repo: actor.repo ?? null, channel: actor.channel ?? null,
+      synthetic: actor.synthetic ? 1 : 0, extra: json(actor.extra),
+    });
+    adopt({ tbl: 'actors', k1: actor.id });
+  }
+
+  /** The delivery as the local collector stores it: its streams carried inside it, from their own stream facts. */
+  function deliveryFact(p) {
+    const prior = JSON.parse(q.factOf.get('delivery', p.id)?.payload || 'null');
+    const { streamKeys = [], ...delivery } = p.delivery || {};
+    const byKey = new Map((prior?.streams || []).map((s) => [s.key, s]));
+    const streams = streamKeys.map((key) => {
+      const own = q.factOf.get('stream', key);
+      return own ? { ...JSON.parse(own.payload), key } : byKey.get(key) || { key };
+    });
+    return { ...delivery, streams };
+  }
+
+  function applyOne(event) {
+    const p = event.payload || {};
+    switch (event.type) {
+      case 'message.published': {
+        q.message.run(p.id, p.from, p.topic, p.type, p.body, p.needsAck ? 1 : 0, Number(p.expiresAt), p.replyTo ?? null, p.reaction ?? null,
+          p.fromName ?? null, Number(p.at) || now(), json(p.meta ?? {}));
+        for (const to of p.to || []) q.delivery.run(p.id, to, Number(p.at) || now());
+        return 'applied';
+      }
+      case 'receipt.delivered':
+      case 'receipt.acked':
+      case 'receipt.expired':
+        return receipt(event);
+      case 'message.retracted':
+        db.prepare('DELETE FROM deliveries WHERE message_id = ?').run(p.id);
+        db.prepare('DELETE FROM messages WHERE id = ?').run(p.id);
+        return 'applied';
+      case 'actor.registered':
+      case 'actor.updated':
+        upsertActor({ ...(p.actor || {}), id: p.id });
+        return 'applied';
+      case 'actor.retired':
+        return 'not_local';
+      case 'actor.row_updated':
+        setFact('agent-row', p.id, p.row, p.at);
+        return 'applied';
+      case 'delivery.updated':
+      case 'delivery.created':
+        setFact('delivery', p.id, deliveryFact(p), p.at);
+        return 'applied';
+      case 'stream.updated': {
+        setFact('stream', p.key, p.stream, p.at);
+        const owner = JSON.parse(q.factOf.get('delivery', p.deliveryId)?.payload || 'null');
+        if (owner) setFact('delivery', p.deliveryId, { ...owner, streams: (owner.streams || []).map((s) => (s.key === p.key ? { ...p.stream, key: p.key } : s)) }, p.at);
+        return 'applied';
+      }
+      case 'request.created':
+        setFact('request', p.id, p.request, p.at);
+        return 'applied';
+      case 'delivery.accepted':
+        setFact('accepted', p.id, { at: Number(p.at) || now() }, p.at);
+        acceptDelivery(p.id);
+        return 'applied';
+      case 'settings.changed':
+        setFact('settings', p.key, p.value, p.at);
+        return 'applied';
+      case 'status.overridden':
+        setFact('status_override', p.key, p.override, p.at);
+        return 'applied';
+      case 'fact.removed':
+        q.dropFact.run(p.kind, p.key);
+        return 'applied';
+      case 'audit.recorded':
+        q.event.run(Number(p.at) || now(), p.kind, p.actor ?? null, json(p.data ?? {}));
+        return 'applied';
+      case 'post.created':
+        // Appended to channel.md by the channel bridge after this page commits (render.js), once.
+        return 'applied';
+      default:
+        if (NOT_LOCAL.has(event.type)) return 'not_local';
+        throw Object.assign(new Error(`no local meaning for ${event.type}`), { reason: 'unknown_type' });
+    }
+  }
+
+  const ownEvent = (event, own) => store.known(event.key) || (own.has(event.machineId) && Number(event.seq) <= Number(cutoverSeq()));
+
+  function step(event, { retry = false, own = ownMachines() } = {}) {
+    if (!retry && q.seen.get(event.seq, event.id)) return 'replayed';
+    if (ownEvent(event, own)) return 'own';
+    const group = entityGroup(event);
+    if (group && q.pending.get(group)) {
+      q.defer.run(event.seq, group, JSON.stringify(event), now());
+      return 'deferred';
+    }
+    if (retry && group) {
+      const entity = group;
+      const acked = Number(q.ackedSeq.get(entity)?.seq || 0);
+      if (acked > Number(event.seq)) return 'superseded';
+    }
+    db.prepare('SAVEPOINT pulse_event').run();
+    try {
+      const outcome = applyOne(event);
+      db.prepare('RELEASE pulse_event').run();
+      return outcome;
+    } catch (error) {
+      db.prepare('ROLLBACK TO pulse_event').run();
+      db.prepare('RELEASE pulse_event').run();
+      touched.length = 0;
+      q.quarantine.run(event.seq, event.id ?? null, event.type ?? null, String(error.reason || error.message).slice(0, 300), JSON.stringify(event.payload ?? null).slice(0, 32_000), now());
+      return 'quarantined';
+    }
+  }
+
+  function settle(event, outcome) {
+    q.record.run(event.seq, event.id || `seq:${event.seq}`, event.type || 'unknown', event.key || '', event.machineId ?? null, outcome, now());
+    if (outcome === 'applied' && event.key) q.sent.run(event.key, event.type, String(event.seq), now());
+    for (const change of touched.splice(0)) mapping.adopt?.(change, db);
+  }
+
+  return {
+    /** One page, in one transaction. Returns counts and the cursor it moved to. */
+    apply(page) {
+      const counts = { applied: 0, skipped: 0, deferred: 0, quarantined: 0, notLocal: 0, posts: [] };
+      const own = ownMachines();
+      capture.applying(() => {
+        for (const event of page.events || []) {
+          const outcome = step(event, { own });
+          if (outcome === 'replayed') continue;
+          if (outcome === 'applied') counts.applied += 1;
+          else if (outcome === 'own' || outcome === 'not_forward') counts.skipped += 1;
+          else if (outcome === 'deferred') counts.deferred += 1;
+          else if (outcome === 'quarantined') counts.quarantined += 1;
+          else if (outcome === 'not_local') counts.notLocal += 1;
+          if (outcome !== 'deferred') settle(event, outcome);
+          else q.record.run(event.seq, event.id || `seq:${event.seq}`, event.type, event.key || '', event.machineId ?? null, 'deferred', now());
+          if (outcome === 'applied' && event.type === 'post.created') counts.posts.push(event);
+          checkpoint('after-event');
+        }
+        q.cursor.run(Number(page.next) || 0, Number(page.head) || 0, Number(page.floor) || 0, now());
+        // Other machines' beats: whose aliases and sessions are live there (Pulse 1.1.1 F1, bus identity).
+        counts.presence = applyRemotePresence(db, page.presence, machineId);
+      });
+      return { ...counts, next: Number(page.next) || 0 };
+    },
+
+    /** Deferred foreign changes whose entity has nothing pending any more: applied now if they are the later write. */
+    retryDeferred() {
+      const counts = { applied: 0, superseded: 0, waiting: 0 };
+      capture.applying(() => {
+        for (const row of q.deferred.all()) {
+          if (q.pending.get(row.grp)) { counts.waiting += 1; continue; }
+          const event = JSON.parse(row.event);
+          const outcome = step(event, { retry: true });
+          if (outcome === 'deferred') { counts.waiting += 1; continue; }
+          q.undefer.run(row.seq);
+          db.prepare('UPDATE pulse_events SET outcome = ? WHERE seq = ?').run(outcome, event.seq);
+          if (outcome === 'applied') { counts.applied += 1; settle(event, outcome); }
+          if (outcome === 'superseded') counts.superseded += 1;
+        }
+      });
+      if (counts.applied) log('deferred_applied', counts);
+      return counts;
+    },
+  };
+}