@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.2

package/src/lib/hubScan.js

@@ -0,0 +1,611 @@
+import { createHash } from 'node:crypto';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { setImmediate as yieldToLoop } from 'node:timers/promises';
+import { channelFiles, splitPosts } from './channelScanner.js';
+import {
+  actorPayload, contentSha, eventPayload, factDescriptors, keyPart, messagePayload, personaPayload, postKey, postPayload, receiptEvents, without,
+} from './hubMapping.js';
+import { contentWithheld, leaksPath, pathRejected, SECRET_RULES } from './mirrorContract.js';
+import { canonicalJson } from './planContract.js';
+
+export const SCAN_SOURCES = Object.freeze(['facts', 'actors', 'messages', 'receipts', 'events', 'personas', 'chat', 'channels', 'presence']);
+/** Fact kinds the cloud log carries, and the ones it does not, with the reason. */
+export const FACT_KINDS = Object.freeze(['delivery', 'agent-row', 'request', 'accepted', 'settings', 'status_override']);
+export const SKIPPED_FACT_KINDS = Object.freeze({
+  stream: 'carried_in_delivery',
+  run: 'machine_runtime_state',
+  wake: 'machine_runtime_state',
+  manager: 'machine_runtime_state',
+  totals: 'phase1_deviation_totals',
+});
+export const REMOVABLE_KINDS = Object.freeze(['delivery', 'stream', 'agent-row', 'request', 'status_override']);
+export const ALL_FACT_KINDS = Object.freeze([...FACT_KINDS, ...Object.keys(SKIPPED_FACT_KINDS)]);
+
+// Same as the cloud's idempotency key rule (bll/pulse/PulseCommandService.js KEY).
+const KEY = /^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,319}$/;
+const ABSOLUTE_PATH = /(?:^|[\s"'(=:])(?:\/(?:Users|home|Volumes|private|var|tmp|opt)\/|[A-Za-z]:\\)/;
+const MAX_PAYLOAD_BYTES = 32 * 1024;
+const POST_BODY_MAX = 16_000;
+// The hub stores a bus body of at most 4,000 characters; a redaction that lengthened one is cut back to the cloud's bound.
+const MESSAGE_BODY_MAX = 4_000;
+// Fields of an actor row that move on their own while it works; a change in only these is sent at most once a minute.
+export const ACTOR_TIME_FIELDS = Object.freeze(['last_at', 'msgs_in', 'msgs_out', 'tokens_est']);
+export const MESSAGE_LOOKBACK_MS = 120_000;
+export const RECEIPT_WINDOW_MS = 8 * 86_400_000;
+export const RETRACT_WINDOW_MS = 48 * 3_600_000;
+export const ACTOR_COALESCE_MS = 60_000;
+export const PRESENCE_COALESCE_MS = 30_000;
+export const CHANNEL_QUIET_MS = 2_000;
+const YIELD_EVERY = 200;
+
+const sha = (text) => createHash('sha256').update(text).digest('hex');
+const short = (text) => sha(text).slice(0, 12);
+const clean = (value) => JSON.parse(JSON.stringify(value ?? null));
+
+function newSource() {
+  return { read: 0, planned: 0, unchanged: 0, skipped: {}, refused: {}, events: {}, redactions: 0, cut: 0, vanished: 0 };
+}
+
+/**
+ * One pass over the local hub's sources (design §11 table, PLAN v2): every row becomes Pulse commands queued in the
+ * runner store, or is counted as unchanged, skipped (with a reason) or refused (with a code), so for each source
+ * read = planned + unchanged + skipped + refused. `import` reads every source from the start; `shadow` reads from the
+ * saved cursors with lookbacks and coalesces fields that move on their own. hub.db is read once, in one short read
+ * transaction, before anything is redacted or queued; each source's commands and its cursor commit together.
+ */
+export class HubScan {
+  constructor({ store, reader, hubDir, roots, redaction, secrets = () => [], now = Date.now }) {
+    this.store = store;
+    this.reader = reader;
+    this.hubDir = hubDir;
+    this.roots = roots;
+    this.redaction = redaction;
+    this.secrets = secrets;
+    this.now = now;
+  }
+
+  /**
+   * What one pass (or one capture transaction) needs: its counters, its time, whether it coalesces as the shadow
+   * does, and a stable redactor that knows the repos named by these delivery facts.
+   */
+  context({ now = this.now(), shadow = true, deliveries = [] } = {}) {
+    const report = Object.fromEntries(SCAN_SOURCES.map((source) => [source, newSource()]));
+    const secrets = this.secrets().filter(Boolean);
+    this.redaction.learnRepos(deliveries);
+    const redactor = this.redaction.redactor(secrets, { stable: true });
+    return { report, now, shadow, redactor, secrets };
+  }
+
+  /**
+   * `only` ({ channels, topics }: sets of channel keys and bus topics) limits the channels and messages scanned, for a
+   * reconcile's rescan of a few items; nothing else is scanned with it.
+   */
+  async run({ mode, sources = SCAN_SOURCES, only = null }) {
+    if (mode !== 'import' && mode !== 'shadow') throw new TypeError(`HubScan mode must be import or shadow, not ${mode}`);
+    if (only) sources = sources.filter((source) => source === 'channels' || source === 'messages');
+    const now = this.now();
+    const shadow = mode === 'shadow';
+    const want = new Set(sources);
+    const cursor = (source, fallback) => (shadow ? this.store.cursor(source) ?? fallback : fallback);
+    const messagesCursor = cursor('messages', { createdAt: 0 });
+    const eventsCursor = cursor('events', { id: 0 });
+    const hub = this.reader.read((snapshot) => {
+      const messages = snapshot.messagesSince(shadow ? Math.max(0, messagesCursor.createdAt - MESSAGE_LOOKBACK_MS) : 0);
+      const open = shadow ? this.store.openReceipts() : [];
+      const recent = shadow ? this.store.recentMessages(now - RETRACT_WINDOW_MS) : [];
+      return {
+        facts: snapshot.facts(ALL_FACT_KINDS),
+        actors: snapshot.actors(),
+        presence: snapshot.presence(),
+        messages,
+        recipients: snapshot.recipients(messages.map((message) => message.id)),
+        open: open.map((entry) => ({ ...entry, row: snapshot.receipt(entry.messageId, entry.recipient) })),
+        gone: recent.filter((entry) => !snapshot.messageExists(entry.id)),
+        events: snapshot.eventsAfter(eventsCursor.id),
+      };
+    });
+    const deliveries = hub.facts.filter((fact) => fact.kind === 'delivery').map((fact) => parseFact(fact)).filter(Boolean);
+    const context = this.context({ now, shadow, deliveries });
+    const { report } = context;
+
+    if (want.has('facts')) await this.#facts(context, hub.facts);
+    if (want.has('actors')) await this.#actors(context, hub.actors);
+    if (want.has('messages')) await this.#messages(context, only ? hub.messages.filter((message) => only.topics.has(message.topic)) : hub.messages, hub.recipients, messagesCursor);
+    if (shadow && want.has('receipts')) await this.#receiptTransitions(context, hub.open, hub.gone);
+    if (want.has('events')) await this.#events(context, hub.events, eventsCursor);
+    if (want.has('personas')) await this.#personas(context);
+    if (want.has('chat')) await this.#chat(context, hub.messages);
+    if (want.has('channels')) await this.#channels(context, only);
+    const presence = want.has('presence') ? this.presenceBeats(context, hub.presence) : [];
+    return { mode, at: now, sources: report, presence, totals: totals(report) };
+  }
+
+  /**
+   * Redact, check and key one command. `fixedKey` for append-only items (their key comes from the source id or raw
+   * content); otherwise the key is content-addressed per entity from the redacted payload without `at`.
+   */
+  prepare(context, source, { type, group, payload, fixedKey = null, entity = null, prefix = null, separator = ':' }) {
+    const stats = context.report[source];
+    const redacted = clean(context.redactor.redactValue(payload));
+    stats.redactions += context.redactor.takeRedactions();
+    if (type === 'message.published' && redacted.body.length > MESSAGE_BODY_MAX) redacted.body = `${redacted.body.slice(0, MESSAGE_BODY_MAX - 1)}…`;
+    let key = fixedKey;
+    let version = null;
+    if (!key) {
+      const content = contentSha(redacted);
+      if (this.store.version(entity)?.sha === content) return { unchanged: true };
+      key = this.store.versionKey(entity, prefix, content, separator);
+      version = { entity, sha: content, key };
+    }
+    const entry = { key, group, type, payload: redacted, source };
+    const unsafe = (text) => !text || ABSOLUTE_PATH.test(text) || pathRejected(text) || SECRET_RULES.some((rule) => rule.probe.test(text));
+    if (!KEY.test(key) || unsafe(key) || unsafe(group)) return { refused: 'unsafe_key', entry: { ...entry, key: `unsafe:${sha(key).slice(0, 40)}`, group: 'unsafe' } };
+    if (contentWithheld(redacted, context.secrets)) return { refused: 'content_withheld', entry };
+    if (leaksPath(redacted)) return { refused: 'path_not_redacted', entry };
+    if (Buffer.byteLength(JSON.stringify(redacted), 'utf8') > MAX_PAYLOAD_BYTES) return { refused: 'payload_too_large', entry };
+    return { entry, version };
+  }
+
+  /** Queue the ready commands of one row; returns whether anything new was queued. */
+  queue(context, source, prepared) {
+    const stats = context.report[source];
+    let added = 0;
+    for (const item of prepared) {
+      if (!item.entry || item.refused) continue;
+      if (this.store.known(item.entry.key)) continue;
+      added += this.store.enqueue([item.entry]);
+      if (item.version) this.store.setVersion(item.version.entity, { ...item.version, rawSha: item.rawSha ?? null, staticSha: item.staticSha ?? null });
+      stats.events[item.entry.type] = (stats.events[item.entry.type] || 0) + 1;
+    }
+    return added > 0;
+  }
+
+  /**
+   * Count one source row by the outcome of its commands: planned when anything new was queued, refused (recorded with
+   * its code in the store's dead letters, so it is visible and never sent) when any command was, else unchanged.
+   */
+  count(context, source, prepared, queued) {
+    const stats = context.report[source];
+    stats.read += 1;
+    const refused = prepared.find((item) => item.refused);
+    if (refused) {
+      stats.refused[refused.refused] = (stats.refused[refused.refused] || 0) + 1;
+      if (refused.entry) this.store.refuse(refused.entry, { code: refused.refused });
+    } else if (queued) stats.planned += 1;
+    else stats.unchanged += 1;
+  }
+
+  skip(context, source, reason) {
+    const stats = context.report[source];
+    stats.read += 1;
+    stats.skipped[reason] = (stats.skipped[reason] || 0) + 1;
+  }
+
+  async #facts(context, rows) {
+    const present = new Map(REMOVABLE_KINDS.map((kind) => [kind, new Set()]));
+    await this.#chunked(rows, (row) => {
+      const fact = this.factRow(context, row);
+      if (!fact) return;
+      present.get(row.kind)?.add(row.key);
+      if (row.kind === 'delivery') for (const stream of fact.streams || []) present.get('stream').add(stream.key);
+    });
+    // A fact the hub no longer has (keepFacts, dropFact) is removed in Pulse too, never inferred there from absence.
+    const removed = REMOVABLE_KINDS.flatMap((kind) => this.store.entities(`fact:${kind}:`)
+      .map((entity) => ({ kind, entity, key: entity.slice(`fact:${kind}:`.length) }))
+      .filter((entry) => !present.get(entry.kind).has(entry.key)));
+    await this.#chunked(removed, ({ kind, key }) => this.factRemoved(context, kind, key));
+  }
+
+  /** One fact row: its commands queued (or counted unchanged, skipped or refused). Returns the parsed fact, or null. */
+  factRow(context, row) {
+    if (SKIPPED_FACT_KINDS[row.kind]) {
+      this.skip(context, 'facts', SKIPPED_FACT_KINDS[row.kind]);
+      return null;
+    }
+    const fact = parseFact(row);
+    if (!fact) {
+      this.skip(context, 'facts', 'unreadable_payload');
+      return null;
+    }
+    const rawSha = sha(row.payload);
+    const entity = `fact:${row.kind}:${row.key}`;
+    const known = this.store.version(entity);
+    if (known && known.rawSha === rawSha) {
+      context.report.facts.read += 1;
+      context.report.facts.unchanged += 1;
+      return fact;
+    }
+    const prepared = this.factCommands(context, row, fact);
+    prepared[0] = { ...prepared[0], rawSha };
+    const queued = this.queue(context, 'facts', prepared);
+    if (!queued && known && !prepared.some((item) => item.refused)) this.store.touchVersion(entity, rawSha);
+    this.count(context, 'facts', prepared, queued);
+    return fact;
+  }
+
+  /** A fact Pulse has that the hub dropped: removed in Pulse, keyed by the version it had. No-op if never sent. */
+  factRemoved(context, kind, key) {
+    const entity = `fact:${kind}:${key}`;
+    const version = this.store.version(entity);
+    if (!version) return;
+    const prepared = [this.prepare(context, 'facts', {
+      type: 'fact.removed', group: entity, payload: { kind, key, at: context.now },
+      fixedKey: `${keyPart(`rm:${kind}`, key)}:${version.sha}`,
+    })];
+    if (this.queue(context, 'facts', prepared)) this.store.dropVersion(entity);
+  }
+
+  /** Run `fn` over rows in store transactions of YIELD_EVERY rows, giving the event loop a turn between them. */
+  async #chunked(rows, fn) {
+    for (let start = 0; start < rows.length; start += YIELD_EVERY) {
+      this.store.transaction(() => {
+        for (const row of rows.slice(start, start + YIELD_EVERY)) fn(row);
+      });
+      await yieldToLoop();
+    }
+  }
+
+  factCommands(context, row, fact) {
+    return factDescriptors(row, fact).map(({ type, entity, prefix, group, payload }) => this.prepare(context, 'facts', { type, group, payload, entity, prefix }));
+  }
+
+  async #actors(context, rows) {
+    await this.#chunked(rows, (actor) => this.actorRow(context, actor));
+  }
+
+  /** One actor row in the registry's shape: registered first, updated after, coalesced as the shadow does. */
+  actorRow(context, actor) {
+    const entity = `actor:${actor.id}`;
+    const rawSha = sha(JSON.stringify(actor));
+    const known = this.store.version(entity);
+    if (known && known.rawSha === rawSha) {
+      context.report.actors.read += 1;
+      context.report.actors.unchanged += 1;
+      return;
+    }
+    // The first version a store sees is the registration (`actor:<id>:<sha>`); later ones are updates
+    // (`actor:<id>@<sha>`), so the same content never travels under one key as two types.
+    const registered = !known;
+    const prepared = this.prepare(context, 'actors', {
+      type: registered ? 'actor.registered' : 'actor.updated', group: entity, entity, prefix: keyPart('actor', actor.id),
+      separator: registered ? ':' : '@', payload: actorPayload(actor),
+    });
+    const staticSha = prepared.entry ? short(canonicalJson(without(prepared.entry.payload.actor, ...ACTOR_TIME_FIELDS))) : null;
+    if (context.shadow && known && prepared.entry && known.staticSha === staticSha && context.now - known.at < ACTOR_COALESCE_MS) {
+      this.skip(context, 'actors', 'coalesced');
+      return;
+    }
+    const item = { ...prepared, rawSha, staticSha };
+    const queued = this.queue(context, 'actors', [item]);
+    if (!queued && prepared.unchanged) this.store.touchVersion(entity, rawSha);
+    this.count(context, 'actors', [item], queued);
+  }
+
+  async #messages(context, rows, recipients, cursor) {
+    let newest = cursor.createdAt;
+    await this.#chunked(rows, (message) => {
+      newest = Math.max(newest, Number(message.created_at));
+      this.messageRow(context, message, recipients.get(message.id) || []);
+    });
+    // The cursor moves only after every chunk it covers is durably queued; a crash before this rescans and dedupes.
+    this.store.transaction(() => {
+      this.store.setCursor('messages', { createdAt: newest });
+      this.store.pruneRecentMessages(context.now - RETRACT_WINDOW_MS);
+    });
+  }
+
+  /** One bus message with its recipient rows: published once, then each receipt its rows' states imply. */
+  messageRow(context, message, to) {
+    const group = `bus:${message.topic}`;
+    const refusedBefore = this.store.deadCode(message.id);
+    if (refusedBefore) {
+      this.count(context, 'messages', [{ refused: refusedBefore }], false);
+      return;
+    }
+    const published = this.store.known(message.id) ? null : this.prepare(context, 'messages', {
+      type: 'message.published', group, fixedKey: message.id, payload: messagePayload(message, to),
+    });
+    // A message that cannot go up takes its receipts with it: they would wait for it forever.
+    if (published?.refused) {
+      this.count(context, 'messages', [published], false);
+      return;
+    }
+    const prepared = published ? [published] : [];
+    const states = [];
+    for (const row of to) {
+      const { state, events } = receiptEvents(row, message.topic, message.expires_at);
+      states.push([row.recipient, state]);
+      for (const event of events) {
+        if (!this.store.known(event.key)) prepared.push(this.prepare(context, 'messages', { type: event.type, group: event.group, fixedKey: event.key, payload: event.payload }));
+      }
+    }
+    const queued = this.queue(context, 'messages', prepared);
+    for (const [recipient, state] of states) this.store.setReceiptState(message.id, recipient, state);
+    if (Number(message.created_at) >= context.now - RETRACT_WINDOW_MS) this.store.rememberMessage(message.id, Number(message.created_at), group);
+    this.count(context, 'messages', prepared, queued);
+  }
+
+  async #receiptTransitions(context, open, gone) {
+    await this.#chunked(open, (entry) => this.receiptRow(context, entry));
+    await this.#chunked(gone, (message) => this.messageGone(context, message));
+  }
+
+  /** A recipient row whose state may have moved since it was last sent: `{ messageId, recipient, state, row }`. */
+  receiptRow(context, entry) {
+    if (!entry.row) {
+      this.skip(context, 'receipts', 'message_deleted');
+      return;
+    }
+    const { state, events } = receiptEvents(entry.row, entry.row.topic, entry.row.expires_at);
+    if (state === entry.state) {
+      context.report.receipts.read += 1;
+      context.report.receipts.unchanged += 1;
+      return;
+    }
+    const prepared = events.filter((event) => !this.store.known(event.key))
+      .map((event) => this.prepare(context, 'receipts', { type: event.type, group: event.group, fixedKey: event.key, payload: event.payload }));
+    const queued = this.queue(context, 'receipts', prepared);
+    this.store.setReceiptState(entry.messageId, entry.recipient, state);
+    this.count(context, 'receipts', prepared, queued);
+  }
+
+  /** A sent message the hub deleted (`{ id, group }`): retracted in Pulse. */
+  messageGone(context, message) {
+    const prepared = [this.prepare(context, 'receipts', {
+      type: 'message.retracted', group: message.group, fixedKey: `retract:${message.id}`, payload: { id: message.id, at: context.now },
+    })];
+    const queued = this.queue(context, 'receipts', prepared);
+    if (queued || prepared[0].refused || this.store.known(`retract:${message.id}`)) {
+      this.store.forgetMessage(message.id);
+      this.store.dropReceipts(message.id);
+    }
+    this.count(context, 'receipts', prepared, queued);
+  }
+
+  async #events(context, rows, cursor) {
+    let last = cursor.id;
+    await this.#chunked(rows, (event) => {
+      last = Math.max(last, Number(event.id));
+      this.eventRow(context, event);
+    });
+    this.store.setCursor('events', { id: last });
+  }
+
+  eventRow(context, event) {
+    const prepared = [this.prepare(context, 'events', {
+      type: 'audit.recorded', group: 'audit', fixedKey: `evt:${event.id}`,
+      payload: eventPayload(event),
+    })];
+    this.count(context, 'events', prepared, this.queue(context, 'events', prepared));
+  }
+
+  async #personas(context) {
+    const file = path.join(this.hubDir, 'state', 'personas.json');
+    let info;
+    try {
+      info = await stat(file);
+    } catch {
+      return;
+    }
+    const position = { mtimeMs: info.mtimeMs, size: info.size };
+    const last = context.shadow ? this.store.cursor('personas') : null;
+    if (last && last.mtimeMs === position.mtimeMs && last.size === position.size) return;
+    let personas;
+    try {
+      personas = JSON.parse(await readFile(file, 'utf8'));
+    } catch {
+      return;
+    }
+    let index = 0;
+    for (const [id, persona] of Object.entries(personas && typeof personas === 'object' ? personas : {})) {
+      if ((index += 1) % YIELD_EVERY === 0) await yieldToLoop();
+      this.store.transaction(() => {
+        const entity = `persona:${id}`;
+        const prepared = [this.prepare(context, 'personas', { type: 'persona.updated', group: entity, entity, prefix: keyPart('persona', id), payload: personaPayload(id, persona) })];
+        this.count(context, 'personas', prepared, this.queue(context, 'personas', prepared));
+      });
+    }
+    this.store.setCursor('personas', position);
+  }
+
+  async #chat(context, messages) {
+    const dir = path.join(this.hubDir, 'state', 'chat');
+    let names = [];
+    try {
+      names = (await readdir(dir)).filter((name) => name.endsWith('.jsonl')).sort();
+    } catch {
+      return;
+    }
+    const busIds = new Set(messages.map((message) => message.id));
+    for (const name of names) {
+      const base = name.slice(0, -'.jsonl'.length);
+      const lines = (await readFile(path.join(dir, name), 'utf8')).split('\n').filter(Boolean);
+      const done = context.shadow ? this.store.cursor(`chat:${base}`)?.lines ?? 0 : 0;
+      this.store.transaction(() => {
+        for (const [index, line] of lines.entries()) {
+          if (index < done) continue;
+          let entry;
+          try {
+            entry = JSON.parse(line);
+          } catch {
+            this.skip(context, 'chat', 'unreadable_line');
+            continue;
+          }
+          if (!Number.isFinite(entry?.at) || typeof entry.text !== 'string' || typeof entry.from !== 'string') {
+            this.skip(context, 'chat', 'unreadable_line');
+            continue;
+          }
+          if (base.startsWith('dm-')) {
+            const id = `m-${base}-${entry.at}`;
+            if (busIds.has(id) || this.store.known(id)) this.skip(context, 'chat', 'already_bus_message');
+            else this.count(context, 'chat', [{ refused: 'dm_not_on_bus' }], false);
+            continue;
+          }
+          if (!base.startsWith('project-')) {
+            this.skip(context, 'chat', 'unknown_chat_file');
+            continue;
+          }
+          const project = base.slice('project-'.length);
+          const prepared = [this.prepare(context, 'chat', {
+            type: 'post.created', group: `conv:project:${project}`, fixedKey: `${keyPart('chat', base)}:${index}:${short(line)}`,
+            payload: { channel: `project:${project}`, author: entry.from, role: entry.from, type: 'MESSAGE', body: entry.text, at: entry.at },
+          })];
+          this.count(context, 'chat', prepared, this.queue(context, 'chat', prepared));
+        }
+        this.store.setCursor(`chat:${base}`, { lines: lines.length });
+      });
+    }
+  }
+
+  async #channels(context, only = null) {
+    const { files, collisions, unnamed } = await channelFiles(this.roots);
+    for (const _ of collisions) this.skip(context, 'channels', 'shadowed_by_first_root');
+    for (const _ of unnamed) this.skip(context, 'channels', 'name_outside_key_charset');
+    for (const file of files) {
+      if (only && !only.channels.has(`task:${file.key}`)) continue;
+      const position = { mtimeMs: file.mtimeMs, size: file.size };
+      const last = this.store.cursor(`channel:${file.key}`);
+      if (context.shadow && last && last.mtimeMs === position.mtimeMs && last.size === position.size) continue;
+      if (context.shadow && context.now - file.mtimeMs < CHANNEL_QUIET_MS) continue;
+      let text;
+      try {
+        text = await readFile(file.file, 'utf8');
+      } catch {
+        continue;
+      }
+      await yieldToLoop();
+      this.store.transaction(() => {
+        this.#ingestChannel(context, file, text);
+        this.store.setCursor(`channel:${file.key}`, position);
+      });
+    }
+  }
+
+  /**
+   * Diff one channel file against what was ingested from it: a post is identified by the digest of its own block and
+   * its occurrence among identical blocks, so an edit or insert elsewhere leaves it alone. A post whose block changed
+   * in place (its old digest gone, a new one at the same position) is an edit of its first key; a post that vanished
+   * stays in Pulse and is recorded once. `at` is fixed at first ingestion.
+   */
+  #ingestChannel(context, file, text) {
+    const channel = `task:${file.key}`;
+    const group = `conv:${channel}`;
+    const posts = splitPosts(text, file.mtimeMs);
+    const before = this.store.ingested(channel);
+    const seen = new Map();
+    const current = posts.map((post, ordinal) => {
+      const digest = sha(post.text).slice(0, 16);
+      const n = seen.get(digest) || 0;
+      seen.set(digest, n + 1);
+      return { post, ordinal, id: `${digest}:${n}` };
+    });
+    const ids = new Set(current.map((entry) => entry.id));
+    const byId = new Map(before.map((row) => [row.sha, row]));
+    const byOrdinal = new Map(before.map((row) => [row.ordinal, row]));
+    const prepared = [];
+    if (!this.store.known(`channel:${keyPart('task', file.key)}`)) {
+      prepared.push(this.prepare(context, 'channels', { type: 'channel.created', group, fixedKey: `channel:${keyPart('task', file.key)}`, payload: { key: channel } }));
+    }
+    this.queue(context, 'channels', prepared);
+    const kept = new Set();
+    const rows = [];
+    for (const { post, ordinal, id } of current) {
+      const known = byId.get(id);
+      if (known) {
+        kept.add(known.sha);
+        rows.push({ ordinal, sha: id, postKey: known.postKey, at: known.at });
+        context.report.channels.read += 1;
+        context.report.channels.unchanged += 1;
+        continue;
+      }
+      const prior = byOrdinal.get(ordinal);
+      const edit = prior && !ids.has(prior.sha) && !kept.has(prior.sha);
+      const { body, truncated } = this.#postBody(context, post.text);
+      const item = edit
+        ? this.prepare(context, 'channels', { type: 'post.edited', group, fixedKey: `edit:${prior.postKey.slice(5)}:${id.split(':')[0]}`, payload: { channel, postKey: prior.postKey, body, at: context.now } })
+        : this.prepare(context, 'channels', {
+          type: 'post.created', group, fixedKey: postKey(file.key, id), payload: postPayload(channel, post, body, truncated),
+        });
+      const queued = this.queue(context, 'channels', [item]);
+      this.count(context, 'channels', [item], queued);
+      if (queued || this.store.known(item.entry?.key ?? '')) {
+        if (edit) kept.add(prior.sha);
+        rows.push({ ordinal, sha: id, postKey: edit ? prior.postKey : item.entry.key, at: edit ? prior.at : post.at });
+      }
+    }
+    for (const row of before) {
+      if (kept.has(row.sha) || ids.has(row.sha)) continue;
+      const item = this.prepare(context, 'channels', {
+        type: 'audit.recorded', group, fixedKey: `vanished:${row.postKey.slice(5)}`,
+        payload: { kind: 'channel.post_vanished', actor: null, data: { channel, postKey: row.postKey }, at: context.now },
+      });
+      this.queue(context, 'channels', [item]);
+      context.report.channels.vanished = (context.report.channels.vanished || 0) + 1;
+    }
+    this.store.replaceIngested(channel, rows);
+  }
+
+  /**
+   * The block as it will be stored: a long one is redacted first and then cut visibly at the cloud's 16,000
+   * characters, with its full length and digest; a short one is redacted with the rest of its payload.
+   */
+  #postBody(context, text) {
+    if (text.length <= POST_BODY_MAX - 200) return { body: text, truncated: null };
+    const redacted = context.redactor.redactString(text);
+    context.report.channels.redactions += context.redactor.takeRedactions();
+    if (redacted.length <= POST_BODY_MAX) return { body: redacted, truncated: null };
+    context.report.channels.cut += 1;
+    const marker = `\n\n[… ${redacted.length - POST_BODY_MAX + 80} more characters in channel.md]`;
+    const body = `${redacted.slice(0, POST_BODY_MAX - 80)}${marker}`.slice(0, POST_BODY_MAX);
+    return { body, truncated: { chars: redacted.length, sha256: sha(redacted) } };
+  }
+
+  /** Presence rows due a heartbeat (not log events, D10): the caller sends them and marks them sent. */
+  presenceBeats(context, rows) {
+    const beats = [];
+    for (const row of rows) {
+      const last = this.store.presenceSent(row.actor_id);
+      const lastSeen = Number(row.last_seen);
+      const stopped = row.last_event === 'stop';
+      const due = !last || (lastSeen > last.lastSeen && (!context.shadow || context.now - last.sentAt >= PRESENCE_COALESCE_MS || stopped));
+      context.report.presence.read += 1;
+      if (!due) {
+        context.report.presence.unchanged += 1;
+        continue;
+      }
+      context.report.presence.planned += 1;
+      beats.push({ type: 'presence.heartbeat', idempotencyKey: `presence:${short(row.actor_id)}:${lastSeen}`, payload: { actorId: row.actor_id, event: row.last_event ?? null, at: lastSeen } });
+    }
+    return beats;
+  }
+}
+
+function parseFact(row) {
+  try {
+    const value = JSON.parse(row.payload);
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Sums over every source; presence beats are not events, so they are counted as `beats`, not as planned. */
+function totals(report) {
+  const out = { read: 0, planned: 0, unchanged: 0, skipped: 0, refused: 0, events: 0, beats: 0, redactions: 0, balanced: true };
+  for (const [source, stats] of Object.entries(report)) {
+    const skipped = Object.values(stats.skipped).reduce((sum, n) => sum + n, 0);
+    const refused = Object.values(stats.refused).reduce((sum, n) => sum + n, 0);
+    out.read += stats.read;
+    if (source === 'presence') out.beats += stats.planned;
+    else out.planned += stats.planned;
+    out.unchanged += stats.unchanged;
+    out.skipped += skipped;
+    out.refused += refused;
+    out.events += Object.values(stats.events).reduce((sum, n) => sum + n, 0);
+    out.redactions += stats.redactions;
+    if (stats.read !== stats.planned + stats.unchanged + skipped + refused) out.balanced = false;
+  }
+  return out;
+}
+

package/src/lib/hubSnapshot.js

@@ -0,0 +1,64 @@
+import { copyFile, mkdir, readdir, rm, stat, utimes, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { channelFiles } from './channelScanner.js';
+import { openDatabase } from './sqlite.js';
+
+export const SNAPSHOT_DIR = 'import-snapshot';
+const MANIFEST = 'manifest.json';
+
+async function copyKeepingTime(from, to) {
+  await mkdir(path.dirname(to), { recursive: true });
+  await copyFile(from, to);
+  const info = await stat(from);
+  await utimes(to, info.atime, info.mtime);
+}
+
+async function listing(dir) {
+  try {
+    return await readdir(dir);
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+}
+
+/**
+ * A frozen copy of everything an import reads (cloud-first ADR-4): hub.db (VACUUM INTO from a read-only connection,
+ * one consistent read), state/personas.json, state/chat/*.jsonl and every channel file under each channel root, with
+ * their mtimes. The import and its `--reconcile` scan this copy, so both read the same sources while the hub moves on.
+ * Laid out as a hub directory plus one folder per root, so the scanner sees the same names, keys and collisions.
+ */
+export async function takeSnapshot({ hubDir, roots, into, now = Date.now }) {
+  await rm(into, { recursive: true, force: true });
+  const snapshotHubDir = path.join(into, 'agents-hub');
+  const state = path.join(hubDir, 'state');
+  await mkdir(path.join(snapshotHubDir, 'state'), { recursive: true, mode: 0o700 });
+  const db = openDatabase(path.join(state, 'hub.db'), { readOnly: true });
+  try {
+    db.prepare('PRAGMA busy_timeout = 5000').run();
+    db.prepare('VACUUM INTO ?').run(path.join(snapshotHubDir, 'state', 'hub.db'));
+  } finally {
+    db.close();
+  }
+  let files = 0;
+  if ((await listing(state)).includes('personas.json')) {
+    await copyKeepingTime(path.join(state, 'personas.json'), path.join(snapshotHubDir, 'state', 'personas.json'));
+    files += 1;
+  }
+  for (const name of (await listing(path.join(state, 'chat'))).filter((entry) => entry.endsWith('.jsonl'))) {
+    await copyKeepingTime(path.join(state, 'chat', name), path.join(snapshotHubDir, 'state', 'chat', name));
+    files += 1;
+  }
+  const snapshotRoots = roots.map((_root, index) => path.join(into, 'roots', String(index)));
+  for (const [index, root] of roots.entries()) {
+    await mkdir(snapshotRoots[index], { recursive: true });
+    const { files: kept, collisions, unnamed } = await channelFiles([root]);
+    for (const entry of [...kept, ...collisions, ...unnamed]) {
+      await copyKeepingTime(entry.file, path.join(snapshotRoots[index], path.relative(root, entry.file)));
+      files += 1;
+    }
+  }
+  const manifest = { takenAt: now(), hubDir, roots, files, snapshotHubDir, snapshotRoots };
+  await writeFile(path.join(into, MANIFEST), `${JSON.stringify(manifest)}\n`, { mode: 0o600 });
+  return { ...manifest, dir: into };
+}