@agentpulselabs/cli-plugin-pulse 0.2.0 → 1.1.2

package/src/lib/reconcile.js

@@ -0,0 +1,316 @@
+import { lstat, readdir, readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { channelFiles, splitPosts } from './channelScanner.js';
+import { HubDbReader } from './hubDbReader.js';
+import { FACT_KINDS, SKIPPED_FACT_KINDS } from './hubScan.js';
+import { bodyDigest, RECONCILE_TYPES, reconcileEntry, reconcileFamilies } from './reconcileView.js';
+import { openDatabase } from './sqlite.js';
+
+const NAMED = 50;
+const SOURCE_TABLES = Object.freeze({ actors: 'actors', messages: 'messages', events: 'events', presence: 'presence' });
+
+/**
+ * How many rows each source holds, counted straight from the snapshot without the scanner, so a scanner that drops
+ * a row is caught even though the import and the reconcile run the same scanner.
+ */
+export async function sourceCounts({ hubDir, roots }) {
+  const counts = {};
+  const db = openDatabase(path.join(hubDir, 'state', 'hub.db'), { readOnly: true });
+  try {
+    for (const [source, table] of Object.entries(SOURCE_TABLES)) counts[source] = Number(db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n);
+    const kinds = [...FACT_KINDS, ...Object.keys(SKIPPED_FACT_KINDS)];
+    counts.facts = Number(db.prepare(`SELECT COUNT(*) AS n FROM facts WHERE kind IN (${kinds.map(() => '?').join(', ')})`).get(...kinds).n);
+  } finally {
+    db.close();
+  }
+  try {
+    const personas = JSON.parse(await readFile(path.join(hubDir, 'state', 'personas.json'), 'utf8'));
+    counts.personas = Object.keys(personas && typeof personas === 'object' ? personas : {}).length;
+  } catch {
+    counts.personas = 0;
+  }
+  counts.chat = 0;
+  const chatDir = path.join(hubDir, 'state', 'chat');
+  for (const name of (await readdir(chatDir).catch(() => [])).filter((entry) => entry.endsWith('.jsonl'))) {
+    counts.chat += (await readFile(path.join(chatDir, name), 'utf8')).split('\n').filter(Boolean).length;
+  }
+  const { files, collisions, unnamed } = await channelFiles(roots);
+  counts.channels = collisions.length + unnamed.length;
+  for (const file of files) counts.channels += splitPosts(await readFile(file.file, 'utf8'), file.mtimeMs).length;
+  return counts;
+}
+
+/** Receipt states a recipient ends in under the hub's (and the cloud's) rules, applied to the commands in order. */
+function receiptStates(commands) {
+  const state = new Map();
+  const topicOf = new Map();
+  for (const command of commands) {
+    const p = command.payload;
+    switch (command.type) {
+      case 'message.published':
+        topicOf.set(p.id, p.topic);
+        for (const to of p.to) state.set(`${p.id}\0${to}`, 'queued');
+        break;
+      case 'receipt.delivered':
+      case 'receipt.acked':
+      case 'receipt.expired': {
+        const id = `${p.messageId}\0${p.recipient}`;
+        const was = state.get(id);
+        if (was === undefined) break;
+        if (command.type === 'receipt.acked') state.set(id, 'acked');
+        else if (command.type === 'receipt.delivered' && (was === 'queued' || was === 'delivered')) state.set(id, 'delivered');
+        else if (command.type === 'receipt.expired' && was === 'queued') state.set(id, 'expired');
+        break;
+      }
+      default:
+        break;
+    }
+  }
+  const byTopic = {};
+  for (const [id, value] of state) {
+    const topic = topicOf.get(id.slice(0, id.indexOf('\0')));
+    byTopic[topic] ??= {};
+    byTopic[topic][value] = (byTopic[topic][value] || 0) + 1;
+  }
+  return byTopic;
+}
+
+/** The local side: the reconcile view of a fresh scan's commands, its receipts, and what it withheld or cut. */
+export function localReconcile({ commands, dead }) {
+  const withBody = (command) => (['post.created', 'message.published'].includes(command.type) ? bodyDigest(command.payload.body) : null);
+  const { view, sets } = reconcileFamilies(commands.filter((command) => RECONCILE_TYPES.includes(command.type))
+    .map((command) => reconcileEntry(command.type, command.key, command.payload, withBody(command))));
+  const byTopic = new Map();
+  for (const [id, { topic }] of sets.messages) byTopic.set(topic, [...(byTopic.get(topic) || []), id]);
+  return {
+    view,
+    // The identities behind each channel and conversation, so a difference can be classified (CUT-02).
+    keys: { posts: new Map([...sets.posts].map(([channel, list]) => [channel, list.map((post) => post.key)])), messages: byTopic },
+    receipts: receiptStates(commands),
+    cut: commands.filter((command) => command.type === 'post.created' && command.payload.truncated).map((command) => command.key).sort(),
+    withheld: dead.map((row) => ({ key: row.key, type: row.type, code: row.code })),
+  };
+}
+
+/**
+ * A rescan keys a post by the digest of its block as it is now. A post edited in place after it was sent is known to
+ * Pulse by its first key, and the delta carried the edit as `post.edited` against that key. So each post takes the key
+ * the live store ingested for the same channel and block (`ingestedKey(channel, '<digest>:<n>')`), and a carried edit
+ * compares as the post it edited.
+ */
+export function resolvePostKeys(commands, ingestedKey) {
+  return commands.map((command) => {
+    if (command.type !== 'post.created' || !command.key.startsWith('post:')) return command;
+    const known = ingestedKey(command.payload.channel, command.key.split(':').slice(-2).join(':'));
+    return known && known !== command.key ? { ...command, key: known } : command;
+  });
+}
+
+const TASK_CHANNEL = /^task:([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)$/;
+const isDirectory = (dir) => stat(dir).then((info) => info.isDirectory(), () => false);
+
+/**
+ * Where a task channel's file is now: 'absent' only when every channel root is a readable directory and none holds
+ * <repo>/.agents/<task>/channel.md; 'present' when one does; 'unknown' otherwise (a root missing, an error other than
+ * not-found, a channel that is not a task channel). Only 'absent' makes a channel retired.
+ */
+export async function channelOnDisk(channel, roots) {
+  const parts = String(channel).match(TASK_CHANNEL)?.slice(1);
+  if (!parts || !roots.length || parts.some((part) => part === '.' || part === '..')) return 'unknown';
+  for (const root of roots) {
+    if (!(await isDirectory(root))) return 'unknown';
+    try {
+      await stat(path.join(root, parts[0], '.agents', parts[1], 'channel.md'));
+      return 'present';
+    } catch (error) {
+      if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') return 'unknown';
+    }
+  }
+  return 'absent';
+}
+
+/** Channels the cloud has posts in that the rescan has none in: the ones whose file must be looked for on disk. */
+export function cloudOnlyChannels(local, cloud) {
+  return Object.keys(cloud.view?.posts?.byChannel ?? {}).filter((channel) => !local.view.posts.byChannel[channel]).sort();
+}
+
+/** The id a classified channel or conversation goes by in the `redacted` map. */
+export const itemId = (family, item) => `${family}\0${item}`;
+
+/**
+ * One differing channel (`posts.byChannel`) or conversation (`messages.byTopic`), classified (CUT-02, the 12:44:58
+ * DECISION):
+ * - `retired`: the cloud has a channel whose file is gone from disk under every root. Listed and allowed.
+ * - `redacted`: same identities, and a rescan with more redaction roots gives exactly the cloud's sha (`redacted`
+ *   holds those items and their roots). The cloud keeps its more-redacted text; nothing is sent. Allowed.
+ * - `drift`: same identities, other content, not explained by redaction. Every local key is one the cloud acknowledged
+ *   (`runner.db` sent) and the counts are equal, so only bodies differ. Fails: sending local text could un-redact.
+ * - `missing`: a local key the cloud never acknowledged, or a channel or conversation it lacks. Fails.
+ * - `unexplained`: anything else (a cloud-only channel whose file is there or cannot be checked, extra cloud items in a
+ *   live channel, no record of what was sent). Fails.
+ */
+function classify({ family, item, local, cloud, keys, sent, onDisk, redacted }) {
+  const type = family === 'posts.byChannel' ? 'post.created' : 'message.published';
+  if (!cloud) return { class: 'missing', note: 'not in Pulse' };
+  if (!local) {
+    if (family === 'posts.byChannel' && onDisk[item] === 'absent') return { class: 'retired', note: 'its channel file is gone from disk' };
+    return { class: 'unexplained', note: family === 'posts.byChannel' ? `only in Pulse, and its channel file is ${onDisk[item] ?? 'unknown'} on disk` : 'only in Pulse' };
+  }
+  if (!sent) return { class: 'unexplained', note: 'no record of what this machine sent' };
+  const lost = keys.filter((key) => !sent.sentAs(key, type));
+  if (lost.length) return { class: 'missing', note: `${lost.length} local item(s) Pulse never acknowledged, e.g. ${lost[0]}` };
+  if (local.count !== cloud.count) return { class: 'unexplained', note: `${local.count} here, ${cloud.count} in Pulse, every local one acknowledged` };
+  const roots = redacted.get(itemId(family, item));
+  if (roots) return { class: 'redacted', roots, note: 'Pulse holds the same text, redacted with more roots' };
+  return { class: 'drift', note: 'same items, other content, and no redaction root explains it' };
+}
+
+const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+/**
+ * Redaction roots a drifted item's text may have been sent with (CUT-02): for each path under the workspace that the
+ * text names, its shallowest directory that no longer exists, a worktree or repo deleted since. Only a candidate;
+ * the proof is a rescan that gives exactly the cloud's sha.
+ */
+export async function redactionCandidates({ texts, workspace }) {
+  const base = workspace.replace(/\/+$/, '');
+  const named = new RegExp(`${escapeRegExp(base)}((?:/[^\\s"'\`<>()\\[\\]{},;/]+)+)`, 'g');
+  const found = new Set();
+  for (const text of texts) {
+    for (const match of String(text).matchAll(named)) {
+      let dir = base;
+      for (const part of match[1].split('/').filter(Boolean)) {
+        if (part === '.' || part === '..') break;
+        dir = path.join(dir, part);
+        const info = await lstat(dir).catch((error) => (error.code === 'ENOENT' ? null : error));
+        if (info === null) found.add(dir);
+        if (!info?.isDirectory?.()) break;
+      }
+    }
+  }
+  return [...found].sort();
+}
+
+/**
+ * The raw text behind each drifted item, from the import's snapshot: a task channel's posts as blocks, a
+ * conversation's message bodies. Keyed by `itemId`. A project channel (chat) has none here.
+ */
+export async function driftTexts({ snapshotHubDir, snapshotRoots }, drift) {
+  const texts = new Map();
+  const channels = new Set(drift.filter((entry) => entry.family === 'posts.byChannel').map((entry) => entry.item));
+  if (channels.size) {
+    const { files } = await channelFiles(snapshotRoots);
+    for (const file of files.filter((entry) => channels.has(`task:${entry.key}`))) {
+      texts.set(itemId('posts.byChannel', `task:${file.key}`), splitPosts(await readFile(file.file, 'utf8'), file.mtimeMs).map((post) => post.text));
+    }
+  }
+  const topics = new Set(drift.filter((entry) => entry.family === 'messages.byTopic').map((entry) => entry.item));
+  if (topics.size) {
+    for (const message of new HubDbReader({ hubDir: snapshotHubDir }).read((snapshot) => snapshot.messagesSince(0))) {
+      if (!topics.has(message.topic)) continue;
+      const id = itemId('messages.byTopic', message.topic);
+      texts.set(id, [...(texts.get(id) || []), message.body]);
+    }
+  }
+  return texts;
+}
+
+// Key order carries no meaning: the cloud answers receipt states and topics in its own order.
+const canonical = (value) => {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
+  return value ?? null;
+};
+const same = (a, b) => JSON.stringify(canonical(a)) === JSON.stringify(canonical(b));
+
+const differingKeys = (local = {}, cloud = {}) => [...new Set([...Object.keys(local), ...Object.keys(cloud)])].sort().filter((key) => !same(local[key], cloud[key]));
+
+function named(mismatches, family, items) {
+  mismatches.push(...items.slice(0, NAMED));
+  if (items.length > NAMED) mismatches.push({ family, item: `… ${items.length - NAMED} more`, local: null, cloud: null });
+}
+
+function compareMap(mismatches, family, local = {}, cloud = {}) {
+  named(mismatches, family, differingKeys(local, cloud).map((key) => ({ family, item: key, local: local[key] ?? null, cloud: cloud[key] ?? null })));
+}
+
+/**
+ * Local scan against the cloud's answer (GET /connector/parity?view=reconcile), and the scan's own rows against the
+ * sources counted without it. Every differing channel and conversation is classified (`classify`): `retired` and
+ * `redacted` ones are listed and allowed, `drift`, `missing` and `unexplained` ones are mismatches. The `channels`,
+ * `posts` and `messages` totals differ only by what is retired or redacted, so the counts must add up to exactly that.
+ * Status: `equal` (nothing differs), `explained` (only retired or redacted items do) or `mismatch`. Every other
+ * family, the receipt counts, the cut list, every logged item being in the cloud's state tables and every source row
+ * being accounted for are compared exactly, as before. `sent` (the live runner store: `sentAs(key, type)`) proves
+ * which items the cloud acknowledged; `onDisk` maps each cloud-only channel to `channelOnDisk`; `redacted` maps the
+ * `itemId` of each item a rescan proved to the roots it took.
+ */
+export function compareReconcile({ local, cloud, counts, report, sent = null, onDisk = {}, redacted = new Map() }) {
+  const mismatches = [];
+  const items = [];
+  for (const [family, localMap, cloudMap, keysOf] of [
+    ['posts.byChannel', local.view.posts.byChannel, cloud.view?.posts?.byChannel, local.keys?.posts],
+    ['messages.byTopic', local.view.messages.byTopic, cloud.view?.messages?.byTopic, local.keys?.messages],
+  ]) {
+    for (const item of differingKeys(localMap, cloudMap)) {
+      const [l, c] = [localMap[item] ?? null, cloudMap?.[item] ?? null];
+      items.push({ family, item, local: l, cloud: c, ...classify({ family, item, local: l, cloud: c, keys: keysOf?.get(item) ?? [], sent, onDisk, redacted }) });
+    }
+  }
+  const retired = items.filter((entry) => entry.class === 'retired');
+  const proven = items.filter((entry) => entry.class === 'redacted');
+  const drift = items.filter((entry) => entry.class === 'drift');
+  const failing = items.filter((entry) => ['drift', 'missing', 'unexplained'].includes(entry.class));
+  const failed = (family) => failing.some((entry) => entry.family === family);
+  const retiredPosts = retired.reduce((n, entry) => n + entry.cloud.count, 0);
+  const totals = {
+    channels: local.view.channels.count + retired.length === cloud.view?.channels?.count && !failed('posts.byChannel'),
+    posts: local.view.posts.count + retiredPosts === cloud.view?.posts?.count && !failed('posts.byChannel'),
+    messages: local.view.messages.count === cloud.view?.messages?.count && !failed('messages.byTopic'),
+  };
+  const sources = Object.fromEntries(Object.entries(report.sources).map(([source, stats]) => {
+    const accounted = stats.planned + stats.unchanged + Object.values(stats.skipped).reduce((n, v) => n + v, 0)
+      + Object.values(stats.refused).reduce((n, v) => n + v, 0);
+    return [source, { counted: counts[source] ?? null, read: stats.read, accounted }];
+  }));
+  for (const [source, row] of Object.entries(sources)) {
+    if (row.counted !== null && row.counted !== row.read) mismatches.push({ family: 'source', item: source, local: row.read, cloud: null, note: `${row.counted} rows in the source, ${row.read} read` });
+    if (row.read !== row.accounted) mismatches.push({ family: 'source', item: source, local: row.accounted, cloud: null, note: `${row.read} read, ${row.accounted} accounted for` });
+  }
+  for (const family of ['actors', 'agentRows', 'personas', 'statusOverrides']) {
+    if (!same(local.view[family], cloud.view?.[family])) mismatches.push({ family, local: local.view[family], cloud: cloud.view?.[family] ?? null });
+  }
+  for (const family of ['channels', 'posts', 'messages']) {
+    const { count, sha } = local.view[family];
+    const other = cloud.view?.[family] ?? {};
+    if ((count !== other.count || sha !== other.sha) && !totals[family]) {
+      const note = retired.length ? { note: `${retired.length} retired channel(s) do not explain the difference` } : {};
+      mismatches.push({ family, class: 'unexplained', local: { count, sha }, cloud: { count: other.count ?? null, sha: other.sha ?? null }, ...note });
+    }
+  }
+  for (const family of ['posts.byChannel', 'messages.byTopic']) named(mismatches, family, failing.filter((entry) => entry.family === family));
+  compareMap(mismatches, 'facts', local.view.facts, cloud.view?.facts);
+  compareMap(mismatches, 'receipts', local.receipts, cloud.receipts);
+  // A retired channel's cut posts stay in Pulse's cut list; the rescan cannot have them.
+  const retiredPrefixes = retired.map((entry) => `post:${entry.item.slice('task:'.length)}:`);
+  const cloudCut = (cloud.cut ?? []).filter((key) => !retiredPrefixes.some((prefix) => key.startsWith(prefix)));
+  if (!same(local.cut, cloudCut)) mismatches.push({ family: 'cut', local: local.cut, cloud: cloud.cut ?? null });
+  for (const [family, absent] of Object.entries(cloud.missing || {})) {
+    mismatches.push({ family: `missing.${family}`, local: null, cloud: absent, note: 'in the log but not in the state tables' });
+  }
+  const classes = { retired: retired.length, redacted: proven.length, drift: 0, missing: 0, unexplained: 0 };
+  for (const entry of failing) classes[entry.class] += 1;
+  const status = mismatches.length ? 'mismatch' : retired.length || proven.length ? 'explained' : 'equal';
+  return {
+    status,
+    mismatches,
+    sources,
+    classes,
+    retired: retired.map((entry) => ({ channel: entry.item, posts: entry.cloud.count })),
+    redacted: proven.map(({ family, item, local: l, roots }) => ({ family, item, count: l.count, roots })),
+    drift: drift.map(({ family, item, local: l, cloud: c }) => ({ family, item, count: l.count, local: l.sha, cloud: c.sha })),
+  };
+}
+
+/** A reconcile that the cutover may go on from: nothing differs, or only retired and redacted items do. */
+export const reconcilePasses = (status) => status === 'equal' || status === 'explained';

package/src/lib/reconcileView.js

@@ -0,0 +1,129 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * The import reconciliation view (cloud-first ADR-4): one machine's imported items as families of identities, each a
+ * count and a hash, so the connector's fresh scan and the Pulse log can be compared item for item. The connector keeps
+ * a byte-identical copy (agentpulse-cli-plugin-pulse src/lib/reconcileView.js, checked by its tests) and builds its
+ * side with it.
+ */
+export const RECONCILE_TYPES = Object.freeze([
+  'channel.created', 'post.created', 'message.published', 'message.retracted', 'actor.registered', 'actor.row_updated',
+  'persona.updated', 'delivery.updated', 'stream.updated', 'request.created', 'delivery.accepted', 'settings.changed',
+  'audit.recorded', 'status.overridden', 'fact.removed',
+]);
+
+const FACT_KIND = Object.freeze({
+  'delivery.updated': 'delivery', 'stream.updated': 'stream', 'request.created': 'request', 'delivery.accepted': 'accepted',
+  'settings.changed': 'settings', 'audit.recorded': 'event',
+});
+
+/** A count and a hash over the keys, in the order given (posts) or sorted (every other family). */
+export function digest(keys) {
+  return { count: keys.length, sha: createHash('sha256').update(keys.join('\n'), 'utf8').digest('hex').slice(0, 16) };
+}
+
+const sorted = (set) => [...set].sort();
+
+/**
+ * Entries `{ type, key, channel, id, topic, factKey, kind, actorId, digest }` in log order (the connector's in send
+ * order). A post or message hashes as `<key> <sha256 of its body>`, so a changed body is a mismatch too. Returns the
+ * serializable view and the identity sets behind it.
+ */
+export function reconcileFamilies(entries) {
+  const channels = new Set();
+  const posts = new Map();
+  const messages = new Map();
+  const retracted = new Set();
+  const actors = new Set();
+  const agentRows = new Set();
+  const personas = new Set();
+  const statusOverrides = new Set();
+  const facts = new Map(Object.values(FACT_KIND).map((kind) => [kind, new Set()]));
+  const removed = [];
+  for (const entry of entries) {
+    switch (entry.type) {
+      case 'channel.created':
+        channels.add(entry.factKey);
+        break;
+      case 'post.created':
+        channels.add(entry.channel);
+        if (!posts.has(entry.channel)) posts.set(entry.channel, []);
+        posts.get(entry.channel).push({ key: entry.key, digest: entry.digest });
+        break;
+      case 'message.published':
+        messages.set(entry.id, { topic: entry.topic, digest: entry.digest });
+        break;
+      case 'message.retracted':
+        retracted.add(entry.id);
+        break;
+      case 'actor.registered':
+        actors.add(entry.id);
+        break;
+      case 'actor.row_updated':
+        agentRows.add(entry.id);
+        break;
+      case 'persona.updated':
+        personas.add(entry.actorId);
+        break;
+      case 'delivery.updated':
+      case 'request.created':
+      case 'delivery.accepted':
+        facts.get(FACT_KIND[entry.type]).add(entry.id);
+        break;
+      case 'stream.updated':
+      case 'settings.changed':
+        facts.get(FACT_KIND[entry.type]).add(entry.factKey);
+        break;
+      case 'audit.recorded':
+        facts.get('event').add(entry.key);
+        break;
+      case 'status.overridden':
+        statusOverrides.add(entry.factKey);
+        break;
+      case 'fact.removed':
+        removed.push(entry);
+        break;
+      default:
+        throw new Error(`reconcile: unhandled type ${entry.type}`);
+    }
+  }
+  for (const id of retracted) messages.delete(id);
+  for (const entry of removed) {
+    if (entry.kind === 'agent-row') agentRows.delete(entry.factKey);
+    else if (entry.kind === 'status_override') statusOverrides.delete(entry.factKey);
+    else facts.get(entry.kind)?.delete(entry.factKey);
+  }
+  const byTopic = new Map();
+  for (const [id, { topic, digest: body }] of messages) {
+    if (!byTopic.has(topic)) byTopic.set(topic, []);
+    byTopic.get(topic).push(`${id} ${body}`);
+  }
+  const line = (post) => `${post.key} ${post.digest}`;
+  const allPosts = [...posts.values()].flat().map(line);
+  return {
+    sets: { channels, posts, messages, actors, agentRows, personas, statusOverrides, facts },
+    view: {
+      channels: digest(sorted(channels)),
+      posts: { ...digest(allPosts), byChannel: Object.fromEntries([...posts].sort(([a], [b]) => (a < b ? -1 : 1)).map(([key, list]) => [key, digest(list.map(line))])) },
+      messages: { ...digest([...messages].map(([id, { digest: body }]) => `${id} ${body}`).sort()), byTopic: Object.fromEntries([...byTopic].sort(([a], [b]) => (a < b ? -1 : 1)).map(([topic, ids]) => [topic, digest(ids.sort())])) },
+      actors: digest(sorted(actors)),
+      agentRows: digest(sorted(agentRows)),
+      personas: digest(sorted(personas)),
+      statusOverrides: digest(sorted(statusOverrides)),
+      facts: Object.fromEntries([...facts].map(([kind, keys]) => [kind, digest(sorted(keys))])),
+    },
+  };
+}
+
+/** A log row (or a connector command) as a reconcile entry; `digest` is the sha256 hex of a post or message body. */
+export function reconcileEntry(type, key, payload = {}, digest = null) {
+  return {
+    type, key, channel: payload.channel ?? null, id: payload.id ?? null, topic: payload.topic ?? null,
+    factKey: payload.key ?? null, kind: payload.kind ?? null, actorId: payload.actorId ?? null, digest,
+  };
+}
+
+/** The sha256 hex of a body, as the state tables store it (agent_hub_posts.body_digest). */
+export function bodyDigest(body) {
+  return createHash('sha256').update(String(body ?? ''), 'utf8').digest('hex');
+}

package/src/lib/redaction.js

@@ -0,0 +1,128 @@
+import {
+  CREDENTIAL_REPLACEMENT, SECRET_RULES, WITHHELD_MARKER, redactSecretBlocks, redactSecretLines, secretKeyReplacement, secretLeafReplacement,
+  secretSubtreeFamily, v1GateFires,
+} from './mirrorContract.js';
+
+function truncateUtf8(value, maxBytes) {
+  const bytes = Buffer.from(value, 'utf8');
+  if (bytes.length <= maxBytes) return { text: value, truncated: false };
+  if (maxBytes <= 0) return { text: '', truncated: true };
+  return {
+    text: bytes.subarray(0, maxBytes).toString('utf8').replace(/\uFFFD$/, ''),
+    truncated: true,
+  };
+}
+
+/**
+ * The contract's passes over one string, without the v1 gate: the block pass, the secret table, the line net, then each
+ * held credential by identity. Every upload path goes through redactSecrets, which adds the gate.
+ */
+export function redactSecretPasses(input, { secrets = [] } = {}) {
+  const blocks = redactSecretBlocks(String(input ?? ''));
+  let { text, redactions } = blocks;
+  for (const rule of SECRET_RULES) {
+    if (!rule.probe.test(text)) continue;
+    text = text.replace(rule.pattern, (...match) => {
+      redactions += 1;
+      const groups = typeof match.at(-1) === 'object' ? match.at(-1) : {};
+      return `${groups.keep ?? ''}${rule.replacement}${groups.tail ?? ''}`;
+    });
+  }
+  const lines = redactSecretLines(text);
+  text = lines.text;
+  redactions += lines.redactions;
+  for (const secret of secrets) {
+    if (!secret || !text.includes(secret)) continue;
+    redactions += text.split(secret).length - 1;
+    text = text.split(secret).join(CREDENTIAL_REPLACEMENT);
+  }
+  return blocks.withheld ? { text, redactions, withheld: true } : { text, redactions };
+}
+
+/**
+ * The contract's passes over one string (redactSecretPasses), then its v1 gate. Returns the text, how many matches were
+ * replaced, and `withheld` (only when true) when a block had no end or v1's pattern still matches with the markers
+ * deleted: the item holding the text is withheld, and a gated text is the withheld marker alone. No replacement matches
+ * a rule, so redacting twice changes nothing.
+ */
+export function redactSecrets(input, options = {}) {
+  const result = redactSecretPasses(input, options);
+  if (result.withheld || !v1GateFires(result.text)) return result;
+  return { text: WITHHELD_MARKER, redactions: result.redactions + 1, withheld: true };
+}
+
+/**
+ * Every string value and key of a JSON value through redactSecrets, the whole string value of an entry whose key names
+ * a secret through secretKeyReplacement, and every string leaf under such a key through secretLeafReplacement; other
+ * values are kept as they are. `withheld` says some string held a block with no end.
+ */
+export function redactSecretsDeep(value, options = {}) {
+  let redactions = 0;
+  let withheld = false;
+  const text = (entry) => {
+    const result = redactSecrets(entry, options);
+    redactions += result.redactions;
+    withheld ||= result.withheld === true;
+    return result.text;
+  };
+  const walk = (entry, family = null) => {
+    if (typeof entry === 'string') {
+      const replacement = family && secretLeafReplacement(family, entry);
+      if (!replacement) return text(entry);
+      redactions += 1;
+      return replacement;
+    }
+    if (Array.isArray(entry)) return entry.map((item) => walk(item, family));
+    if (entry && typeof entry === 'object') {
+      return Object.fromEntries(Object.entries(entry).map(([key, item]) => {
+        const replacement = secretKeyReplacement(key, item);
+        if (replacement) redactions += 1;
+        return [text(key), replacement ?? walk(item, family ?? secretSubtreeFamily(key, item))];
+      }));
+    }
+    return entry;
+  };
+  const redacted = walk(value);
+  return withheld ? { value: redacted, redactions, withheld } : { value: redacted, redactions };
+}
+
+export function redactText(input, { roots = [], maxBytes = 32 * 1024 } = {}) {
+  let { text, redactions } = redactSecrets(input);
+  for (const root of [...roots].filter(Boolean).sort((left, right) => right.length - left.length)) {
+    if (!text.includes(root)) continue;
+    redactions += text.split(root).length - 1;
+    text = text.split(root).join('<root>');
+  }
+  const bounded = truncateUtf8(text, maxBytes);
+  return { ...bounded, redactions };
+}
+
+export function redactObject(value, options = {}) {
+  const maxBytes = options.maxBytes ?? 32 * 1024;
+  if (!Number.isInteger(maxBytes) || maxBytes < 64) {
+    const error = new Error('Object redaction limit must be an integer of at least 64 bytes');
+    error.code = 'HUB_REDACTION_LIMIT_INVALID';
+    throw error;
+  }
+  const roots = [...(options.roots || [])].filter(Boolean).sort((left, right) => right.length - left.length);
+  let withheld = false;
+  const text = (entry) => {
+    const result = redactSecrets(entry);
+    withheld ||= result.withheld === true;
+    return roots.reduce((out, root) => out.split(root).join('<root>'), result.text);
+  };
+  const walk = (entry, family = null) => {
+    if (typeof entry === 'string') return (family && secretLeafReplacement(family, entry)) || text(entry);
+    if (Array.isArray(entry)) return entry.map((item) => walk(item, family));
+    if (entry && typeof entry === 'object') {
+      return Object.fromEntries(Object.entries(entry).map(([key, item]) => [
+        text(key),
+        secretKeyReplacement(key, item) ?? walk(item, family ?? secretSubtreeFamily(key, item)),
+      ]));
+    }
+    return entry;
+  };
+  const redacted = walk(value === undefined ? null : value);
+  if (!withheld && Buffer.byteLength(JSON.stringify(redacted), 'utf8') <= maxBytes) return redacted;
+  return { contentWithheld: true, redactionVersion: 1 };
+}