@agentpulselabs/cli-plugin-pulse 0.2.0 → 1.0.0

package/src/lib/mirrorSource.js

@@ -0,0 +1,455 @@
+import { createHash } from 'node:crypto';
+import os from 'node:os';
+import path from 'node:path';
+import { GRAPH_HOURS as WINDOW_HOURS, mergeWindows, RowMemo, windowRows } from './graphWindows.js';
+import { HubError } from './errors.js';
+import { MIRROR_CONTRACT, redactSecretSubtree, sanitizeItem, secretKeyReplacement } from './mirrorContract.js';
+import { createPathRedactor } from './pathRedaction.js';
+import { redactSecrets } from './redaction.js';
+import { readRulePacks } from './rulePacks.js';
+
+export const MIRROR_SOURCES = Object.freeze(['deliveries', 'graph', 'channels', 'inbox', 'rules', 'windows', 'briefs']);
+/**
+ * The window a `recent` read refreshes on every change: the one Pulse opens on. The others (each a whole hub collect,
+ * run one at a time in its worker) come from the last `windows` read.
+ */
+export const RECENT_HOURS = Object.freeze([24]);
+
+const SOURCE_KINDS = Object.freeze({
+  recent: ['graph_window'],
+  deliveries: ['delivery', 'stream', 'agent'],
+  graph: ['edge', 'bus'],
+  channels: ['channel', 'message'],
+  inbox: ['inbox'],
+  rules: ['rule_pack'],
+  windows: ['graph_window'],
+  briefs: ['brief'],
+});
+const BRIEF_READS_IN_FLIGHT = 4;
+const GRAPH_HOURS = 168;
+const SECRET_SEEN_MAX = 100_000;
+const TASK_KEY = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
+const BRIEF = MIRROR_CONTRACT.kinds.brief.properties;
+const longer = (text, max) => typeof text === 'string' && Array.from(text).length > max;
+
+/** A redacted brief with `truncated` set when the contract limits will cut any of it, so a cut is never silent. */
+function markTruncated(brief) {
+  const { maxItems, items } = BRIEF.sections;
+  const sections = Array.isArray(brief.sections) ? brief.sections : [];
+  const truncated = sections.length > maxItems || longer(brief.title, BRIEF.title.maxLength)
+    || sections.some((s) => longer(s?.heading, items.properties.heading.maxLength) || longer(s?.body, items.properties.body.maxLength));
+  return { ...brief, truncated };
+}
+
+/** Every task key in the windows, with its latest activity, so a brief is re-read only after its task moved. */
+function taskKeys(windows) {
+  const keys = new Map();
+  for (const { body } of windows) {
+    for (const stream of (Array.isArray(body.deliveries) ? body.deliveries : []).flatMap((d) => (Array.isArray(d.streams) ? d.streams : []))) {
+      if (typeof stream.key !== 'string' || !TASK_KEY.test(stream.key)) continue;
+      keys.set(stream.key, Math.max(keys.get(stream.key) || 0, Number(stream.lastAt) || 0));
+    }
+  }
+  return keys;
+}
+const MAX_EXTERNAL_ID = 300;
+const ACK_PRECEDENCE = ['expired', 'unacked', 'queued', 'delivered', 'acked'];
+const MAX_MEMBERS = 20;
+
+function ackState(statuses) {
+  const states = new Set((Array.isArray(statuses) ? statuses : []).map((status) => status?.state));
+  return ACK_PRECEDENCE.find((state) => states.has(state)) ?? null;
+}
+
+function busThreads(messages) {
+  const channels = new Map();
+  const items = [];
+  for (const message of messages) {
+    const topic = String(message.topic || '');
+    const pair = topic.startsWith('agent/') ? [String(message.from), topic.slice('agent/'.length)].sort() : null;
+    const id = pair ? `bus:${pair.join('|')}` : `bus:${topic}`;
+    const channel = channels.get(id) || { id, kind: 'bus', name: null, topic: pair ? null : topic, members: new Set(), lastAt: null };
+    for (const member of pair || [message.from, ...(Array.isArray(message.to) ? message.to : [])]) {
+      if (typeof member === 'string' && member && channel.members.size < MAX_MEMBERS) channel.members.add(member);
+    }
+    if (Number.isFinite(message.at) && !(channel.lastAt >= message.at)) channel.lastAt = message.at;
+    channels.set(id, channel);
+    const statuses = Array.isArray(message.status) ? message.status : [];
+    items.push({
+      channelId: id,
+      data: {
+        channelId: id,
+        id: message.id,
+        at: message.at,
+        from: message.from,
+        fromId: message.from ?? null,
+        fromName: message.fromName ?? null,
+        to: Array.isArray(message.to) ? message.to : [],
+        topic: topic || null,
+        type: message.type ?? null,
+        body: String(message.body ?? message.text ?? ''),
+        replyTo: message.replyTo ?? null,
+        via: statuses.find((status) => status?.via)?.via ?? null,
+        needsAck: typeof message.needsAck === 'boolean' ? message.needsAck : null,
+        ackState: ackState(statuses),
+      },
+    });
+  }
+  return {
+    channels: [...channels.values()].map((channel) => ({ ...channel, members: [...channel.members].sort() })),
+    messages: items,
+  };
+}
+
+function externalIdFor(value) {
+  if (value.length <= MAX_EXTERNAL_ID) return value;
+  return `${value.slice(0, 200)}~${createHash('sha256').update(value).digest('hex').slice(0, 32)}`;
+}
+
+function firstTime(...values) {
+  return values.find((value) => Number.isFinite(value) && value > 0);
+}
+
+function persona(value) {
+  if (!value || typeof value !== 'object') return null;
+  const out = {};
+  for (const key of ['name', 'first', 'username', 'seed']) if (value[key] !== undefined) out[key] = value[key];
+  return out;
+}
+
+function channelEntries(body) {
+  const projects = (body.projects || []).map((entry) => ({
+    id: entry.id, kind: 'project', name: entry.name ?? null, project: entry.project ?? null, lastAt: entry.lastAt ?? null,
+  }));
+  const tasks = (body.tasks || []).map((entry) => ({
+    id: entry.id, kind: 'task', name: entry.name ?? null, key: entry.key ?? null, repo: entry.repo ?? null,
+    project: entry.project ?? null, title: entry.title ?? null, lastAt: entry.lastAt ?? null,
+  }));
+  const dms = (body.dms || []).map((entry) => ({ id: `dm:${entry.id}`, kind: 'dm', name: entry.id, lastAt: entry.lastAt ?? null }));
+  return [...projects, ...tasks, ...dms].filter((entry) => typeof entry.id === 'string');
+}
+
+export class MirrorSource {
+  constructor({
+    reader,
+    workspace,
+    home = os.homedir(),
+    approvedRoots = [],
+    readRules = readRulePacks,
+    now = Date.now,
+    maxMessagesPerChannel = 200,
+    briefBudgetMs = 3_000,
+    heldSecrets = () => [],
+  }) {
+    this.heldSecrets = heldSecrets;
+    this.reader = reader;
+    this.workspace = workspace;
+    this.home = home;
+    this.approvedRoots = approvedRoots;
+    this.readRules = readRules;
+    this.now = now;
+    this.maxMessagesPerChannel = maxMessagesPerChannel;
+    this.briefBudgetMs = briefBudgetMs;
+    this.repoRoots = new Map(approvedRoots.map((root) => [root, path.basename(root)]));
+    this.windowCache = new Map();
+    this.rowMemo = new RowMemo();
+    this.worktreeRoots = new Map();
+    this.briefs = new Map();
+    this.taskKeys = null;
+  }
+
+  /**
+   * Briefs for these task keys: cached ones whose task has not moved, fresh reads for the rest, a few at a time and
+   * for at most briefBudgetMs so a slow hub never holds the next windows pass back. A brief not reached keeps its
+   * cached answer and is read on a later pass. A 404 is "no brief". True when every brief was answered.
+   */
+  async #readBriefs(keys, errors, absent, signal) {
+    let failed = false;
+    const next = new Map();
+    const stale = [];
+    for (const [key, lastAt] of keys) {
+      const cached = this.briefs.get(key);
+      if (cached && cached.lastAt === lastAt) next.set(key, cached);
+      else stale.push([key, lastAt, cached]);
+    }
+    const deadline = this.now() + this.briefBudgetMs;
+    const readOne = async ([key, lastAt, cached]) => {
+      try {
+        next.set(key, { lastAt, body: await this.reader.get('/api/task/brief', { key }) });
+      } catch (error) {
+        if (error.code === 'HUB_LOCAL_HUB_HTTP_404') {
+          next.set(key, { lastAt, body: null });
+          absent.briefs = (absent.briefs || 0) + 1;
+          return;
+        }
+        failed = true;
+        errors.push({ source: 'briefs', code: error.code || 'HUB_MIRROR_READ_FAILED' });
+        if (cached) next.set(key, cached);
+      }
+    };
+    let index = 0;
+    const lane = async () => {
+      while (index < stale.length && this.now() < deadline && !signal?.aborted) await readOne(stale[index++]);
+    };
+    await Promise.all(Array.from({ length: Math.min(BRIEF_READS_IN_FLIGHT, stale.length) }, lane));
+    for (const [key, , cached] of stale.slice(index)) if (cached) next.set(key, cached);
+    this.briefs = new Map([...keys.keys()].filter((key) => next.has(key)).map((key) => [key, next.get(key)]));
+    return !failed && index === stale.length;
+  }
+
+  #learnRepos(deliveries) {
+    for (const stream of deliveries.flatMap((delivery) => delivery.streams || [])) {
+      const git = stream.git || {};
+      if (typeof git.mainRepo === 'string' && path.isAbsolute(git.mainRepo)) this.repoRoots.set(git.mainRepo, path.basename(git.mainRepo));
+      for (const extra of [git.worktreePath, git.channelCheckout]) {
+        if (typeof extra === 'string' && path.isAbsolute(extra) && !this.repoRoots.has(extra)) this.worktreeRoots.set(extra, path.basename(extra));
+      }
+    }
+  }
+
+  /**
+   * Secrets first, then paths, on every string and key: a secret inside a path is never cut down to a last segment
+   * that still holds it, and nothing is projected, truncated or hashed before both have run.
+   */
+  #redactor(secrets) {
+    const repos = [...this.repoRoots, ...this.worktreeRoots].map(([root, name]) => ({ root, name }));
+    const paths = createPathRedactor({ repos, workspace: this.workspace, home: this.home });
+    const seen = new Map();
+    const redactString = (input) => {
+      const original = String(input ?? '');
+      let text = seen.get(original);
+      if (text === undefined) {
+        text = paths.redactString(redactSecrets(original, { secrets }).text);
+        if (seen.size >= SECRET_SEEN_MAX) seen.clear();
+        seen.set(original, text);
+      }
+      return text;
+    };
+    const entry = (key, item) => secretKeyReplacement(key, item) ?? redactSecretSubtree(key, item, redactString);
+    const redactValue = (value, { keep = [] } = {}) => paths.redactValue(value, { keep, redactString, entry });
+    return { redactString, redactValue };
+  }
+
+  async collect({ full = false, sources = new Set(MIRROR_SOURCES), channelLastAt = new Map(), signal } = {}) {
+    const wanted = full ? new Set(MIRROR_SOURCES) : sources;
+    const checkpoint = () => {
+      if (signal?.aborted) throw new HubError('HUB_MIRROR_PREEMPTED', 'A change preempted this mirror pass');
+    };
+    const raw = [];
+    const complete = new Set();
+    const errors = [];
+    // Optional per-item reads that answered 404: the item does not exist, which never makes a pass incomplete.
+    const absent = {};
+    const withheld = [];
+    const nextChannelLastAt = new Map(channelLastAt);
+    const refetchedChannels = [];
+    let listedChannels = null;
+    const read = async (source, run) => {
+      checkpoint();
+      try {
+        await run();
+        for (const kind of SOURCE_KINDS[source]) complete.add(kind);
+      } catch (error) {
+        if (error.code === 'HUB_MIRROR_PREEMPTED') throw error;
+        errors.push({ source, code: error.code || 'HUB_MIRROR_READ_FAILED' });
+      }
+    };
+    const fallback = this.now();
+    const pushDeliveries = (body) => {
+      const deliveries = Array.isArray(body.deliveries) ? body.deliveries : [];
+      this.#learnRepos(deliveries);
+      if (!wanted.has('deliveries')) return;
+      for (const delivery of deliveries) {
+        raw.push({ kind: 'delivery', externalId: delivery.id, updatedAt: firstTime(delivery.updated_at, delivery.lastAt, fallback), data: delivery });
+        for (const stream of delivery.streams || []) {
+          raw.push({
+            kind: 'stream',
+            externalId: stream.key,
+            updatedAt: firstTime(stream.lastAt, fallback),
+            data: { ...stream, deliveryId: delivery.id, branch: stream.git?.branch ?? null, isWorktree: stream.git?.isWorktree ?? null },
+          });
+        }
+      }
+      for (const agent of Array.isArray(body.agents) ? body.agents : []) {
+        const data = agent.persona === undefined ? agent : { ...agent, persona: persona(agent.persona) };
+        raw.push({ kind: 'agent', externalId: agent.id, updatedAt: firstTime(agent.updated_at, agent.lastAt, fallback), data });
+      }
+    };
+    const readDeliveries = async () => {
+      await read('deliveries', async () => pushDeliveries(await this.reader.get('/api/deliveries')));
+      if (!wanted.has('deliveries')) for (const kind of SOURCE_KINDS.deliveries) complete.delete(kind);
+    };
+
+    // The hub's /api/deliveries is the window it collected last (HUB-PERF-01), so a full pass, which tombstones, takes
+    // them from its own 168 h read instead: the widest set, which every narrower answer is inside.
+    if (!full && (wanted.has('deliveries') || wanted.has('rules'))) await readDeliveries();
+
+    // `recent` reads the short windows and takes the others as last read, rows already redacted and checked.
+    const windows = [];
+    const freshHours = wanted.has('windows')
+      ? WINDOW_HOURS
+      : wanted.has('recent') ? WINDOW_HOURS.filter((hours) => RECENT_HOURS.includes(hours) || !this.windowCache.has(hours)) : [];
+    if (freshHours.length) {
+      await read(wanted.has('windows') ? 'windows' : 'recent', async () => {
+        const bodies = await Promise.all(freshHours.map((hours) => this.reader.get('/api/graph', { hours })));
+        const fresh = new Map(freshHours.map((hours, index) => [hours, bodies[index]]));
+        for (const hours of WINDOW_HOURS) {
+          const body = fresh.get(hours);
+          if (body) this.#learnRepos(Array.isArray(body.deliveries) ? body.deliveries : []);
+          windows.push({ hours, body: body ?? this.windowCache.get(hours).body, fresh: Boolean(body) });
+        }
+      });
+      if (windows.length === WINDOW_HOURS.length) this.taskKeys = taskKeys(windows);
+    }
+
+    if (full) {
+      const widest = windows.find(({ hours }) => hours === GRAPH_HOURS)?.body;
+      if (widest) await read('deliveries', async () => pushDeliveries(widest));
+      else {
+        await readDeliveries();
+        for (const kind of SOURCE_KINDS.deliveries) complete.delete(kind);
+      }
+    }
+
+    if (wanted.has('graph')) {
+      await read('graph', async () => {
+        const body = windows.find(({ hours }) => hours === GRAPH_HOURS)?.body
+          ?? await this.reader.get('/api/graph', { hours: GRAPH_HOURS });
+        for (const link of Array.isArray(body.links) ? body.links : []) {
+          raw.push({
+            kind: 'edge',
+            externalId: `${link.source}>${link.target}:${link.kind}`,
+            updatedAt: firstTime(body.generatedAt, fallback),
+            data: link,
+          });
+        }
+        const bus = busThreads(Array.isArray(body.bus?.messages) ? body.bus.messages : []);
+        for (const channel of bus.channels) {
+          raw.push({ kind: 'channel', externalId: channel.id, updatedAt: firstTime(channel.lastAt, fallback), data: channel });
+        }
+        for (const message of bus.messages) {
+          raw.push({
+            kind: 'message',
+            externalId: externalIdFor(`${message.channelId}#${message.data.id}`),
+            updatedAt: firstTime(message.data.at, fallback),
+            data: message.data,
+          });
+        }
+      });
+    }
+
+    if (wanted.has('channels')) {
+      await read('channels', async () => {
+        const entries = channelEntries(await this.reader.get('/api/chat/channels'));
+        for (const entry of entries) {
+          raw.push({ kind: 'channel', externalId: entry.id, updatedAt: firstTime(entry.lastAt, fallback), data: entry });
+        }
+        const listed = new Set(entries.map((entry) => entry.id));
+        for (const id of nextChannelLastAt.keys()) if (!listed.has(id)) nextChannelLastAt.delete(id);
+        for (const entry of entries) {
+          if (channelLastAt.has(entry.id) && channelLastAt.get(entry.id) === entry.lastAt) continue;
+          checkpoint();
+          let body;
+          try {
+            body = await this.reader.get('/api/chat/messages', { ch: entry.id });
+          } catch (error) {
+            // A listed channel the hub has no messages for (a DM whose agent is gone) is empty, not a failed read.
+            if (error.code !== 'HUB_LOCAL_HUB_HTTP_404') {
+              errors.push({ source: 'messages', code: error.code || 'HUB_MIRROR_READ_FAILED' });
+              continue;
+            }
+            absent.messages = (absent.messages || 0) + 1;
+            body = { messages: [] };
+          }
+          const messages = (Array.isArray(body.messages) ? body.messages : []).slice(-this.maxMessagesPerChannel);
+          for (const message of messages) {
+            raw.push({
+              kind: 'message',
+              externalId: externalIdFor(`${entry.id}#${message.id}`),
+              updatedAt: firstTime(message.at, fallback),
+              data: { ...message, channelId: entry.id, body: String(message.body ?? ''), ackState: message.delivery?.busState ?? null },
+            });
+          }
+          nextChannelLastAt.set(entry.id, entry.lastAt);
+          refetchedChannels.push(entry.id);
+        }
+        listedChannels = entries.map((entry) => entry.id);
+      });
+      if (listedChannels && refetchedChannels.length !== listedChannels.length) complete.delete('message');
+    }
+
+    if (wanted.has('inbox')) {
+      await read('inbox', async () => {
+        const body = await this.reader.get('/api/inbox');
+        for (const message of Array.isArray(body.messages) ? body.messages : []) {
+          raw.push({ kind: 'inbox', externalId: message.id, updatedAt: firstTime(message.at, fallback), data: { ...message, body: String(message.body ?? '') } });
+        }
+      });
+    }
+
+    // Briefs are read in a pass of their own after a windows pass, so the windows never wait on them; the task keys
+    // and their activity come from the last complete windows read.
+    let briefsPending = false;
+    if (wanted.has('briefs') && this.taskKeys) {
+      const failures = errors.length;
+      const answered = await this.#readBriefs(this.taskKeys, errors, absent, signal);
+      checkpoint();
+      if (answered) complete.add('brief');
+      else briefsPending = errors.length === failures;
+      for (const [key, { lastAt, body }] of this.briefs) {
+        if (!body || !Array.isArray(body.sections) || !body.sections.length) continue;
+        raw.push({ kind: 'brief', externalId: key, updatedAt: firstTime(lastAt, fallback), data: { ...body, key } });
+      }
+    }
+
+    if (wanted.has('rules')) {
+      await read('rules', async () => {
+        const repos = [...this.repoRoots].map(([root, name]) => ({ root, name }));
+        const { packs } = await this.readRules({ workspace: this.workspace, home: this.home, repos });
+        for (const pack of packs) raw.push({ kind: 'rule_pack', externalId: pack.id, updatedAt: fallback, data: pack });
+      });
+    }
+
+    checkpoint();
+    const secrets = [await this.reader.tokenValue().catch(() => null), ...this.heldSecrets()].filter(Boolean);
+    const redactor = this.#redactor(secrets);
+    const items = new Map();
+    for (const entry of raw) {
+      if (typeof entry.externalId !== 'string' || !entry.externalId) continue;
+      const externalId = externalIdFor(entry.externalId);
+      const redacted = redactor.redactValue(entry.data, { keep: entry.kind === 'rule_pack' ? ['file'] : [] });
+      const result = sanitizeItem(entry.kind, entry.kind === 'brief' ? markTruncated(redacted) : redacted, { secrets });
+      if (result.error) {
+        withheld.push({ kind: entry.kind, externalId, error: result.error });
+        continue;
+      }
+      items.set(`${entry.kind}\0${externalId}`, { kind: entry.kind, externalId, updatedAt: entry.updatedAt, data: result.data });
+    }
+
+    // Rows are shared across windows, so a pass that missed a window uploads none of them; the last counts stand.
+    const graphDropped = {};
+    if (windows.length === WINDOW_HOURS.length) {
+      // A row's redacted, checked form depends on the roots the redactor knows and the secrets, and on nothing else.
+      this.rowMemo.begin(JSON.stringify([[...this.repoRoots], [...this.worktreeRoots], this.workspace, this.home, secrets]));
+      const options = { redact: (value) => redactor.redactValue(value), secrets, memo: this.rowMemo };
+      const built = mergeWindows(windows.map(({ hours, body, fresh }) => {
+        if (!fresh) return this.windowCache.get(hours).rows;
+        const rows = windowRows(hours, body, options);
+        this.windowCache.set(hours, { body, rows });
+        return rows;
+      }));
+      Object.assign(graphDropped, built.dropped);
+      const updatedAt = firstTime(Math.max(...windows.map(({ body }) => Number(body.generatedAt) || 0)), fallback);
+      for (const row of built.items) {
+        const externalId = externalIdFor(row.externalId);
+        const result = this.rowMemo.take(`item\0${JSON.stringify(row.data)}`, () => sanitizeItem('graph_window', row.data, { secrets }));
+        if (result.error) {
+          withheld.push({ kind: 'graph_window', externalId, error: result.error });
+          continue;
+        }
+        items.set(`graph_window\0${externalId}`, { kind: 'graph_window', externalId, updatedAt, data: result.data });
+      }
+    }
+    return { items, complete, errors, absent, withheld, graphDropped, channelLastAt: nextChannelLastAt, refetchedChannels, listedChannels, briefsPending };
+  }
+}