@volter/supercode-interchange 0.1.21 → 0.1.23

package/channels-read.mjs

@@ -0,0 +1,90 @@
+// Recorded channel inventory/status. Configured means credential presence;
+// connection health is always unknown. This reader creates no endpoint.
+import { isAbsolute } from 'node:path';
+import { resolveHarnessHomes } from './homes.mjs';
+import { profilePathJoin as join, profilePathParent as parent } from './profile-paths.mjs';
+import { readChannelFolderHome } from './hermes-read-home.mjs';
+import { readOpenclawChannelHome } from './openclaw-profile-home.mjs';
+import { channelSessionCounts } from './channel-session-counts.mjs';
+import { nativeDuplicateObjectKeys, parseNativeJson, stringifyNativeJsonMember, setNativeInteger } from './native-json.mjs';
+
+export const CHANNEL_HARNESSES = Object.freeze(['hermes', 'openclaw', 'orchestrator']);
+const object = value => value != null && typeof value === 'object' && !Array.isArray(value);
+const compare = (a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b));
+const refuse = (message, rpcCode = -32602) => { throw Object.assign(new Error(message), { rpcCode }); };
+const filesystem = (path, cwd) => !path.length || isAbsolute(path) || process.platform === 'win32' && /^[A-Za-z]:/u.test(path) ? path : join(cwd, path);
+function struct(value, fields, name) {
+  if (Array.isArray(value)) {
+    if (value.length > fields.length) refuse(`${name} has too many fields`);
+    return parseNativeJson(`{${value.map((_, at) => `${JSON.stringify(fields[at])}:${stringifyNativeJsonMember(value, String(at))}`).join(',')}}`);
+  }
+  if (!object(value)) refuse(`${name} must be a struct`);
+  if (nativeDuplicateObjectKeys(value).some(key => fields.includes(key))) refuse(`${name} has a duplicate field`);
+  return value;
+}
+function query(params, env) {
+  const selected = struct(params, ['harness', 'name', 'homes'], 'channels params');
+  for (const key of ['harness', 'name']) if (selected[key] != null && typeof selected[key] !== 'string') refuse(`${key} must be a string`);
+  const homes = resolveHarnessHomes(env);
+  if (Object.hasOwn(selected, 'homes')) {
+    const supplied = struct(selected.homes, Object.keys(homes), 'homes');
+    for (const key of Object.keys(homes)) if (Object.hasOwn(supplied, key)) {
+      if (typeof supplied[key] !== 'string') refuse(`homes.${key} must be a string`);
+      homes[key] = supplied[key];
+    }
+  }
+  return { harness: selected.harness ?? null, name: selected.name ?? null, homes };
+}
+const envCredentials = [
+  ['telegram', ['TELEGRAM_BOT_TOKEN']], ['discord', ['DISCORD_BOT_TOKEN']], ['slack', ['SLACK_BOT_TOKEN']],
+  ['whatsapp_cloud', ['WHATSAPP_CLOUD_PHONE_NUMBER_ID', 'WHATSAPP_CLOUD_ACCESS_TOKEN'], true], ['signal', ['SIGNAL_HTTP_URL']], ['mattermost', ['MATTERMOST_TOKEN']],
+  ['matrix', ['MATRIX_ACCESS_TOKEN', 'MATRIX_PASSWORD']], ['homeassistant', ['HASS_TOKEN']], ['email', ['EMAIL_ADDRESS', 'EMAIL_PASSWORD', 'EMAIL_IMAP_HOST', 'EMAIL_SMTP_HOST'], true],
+  ['sms', ['TWILIO_ACCOUNT_SID']], ['dingtalk', ['DINGTALK_CLIENT_ID', 'DINGTALK_CLIENT_SECRET'], true], ['feishu', ['FEISHU_APP_ID', 'FEISHU_APP_SECRET'], true],
+  ['wecom', ['WECOM_BOT_ID', 'WECOM_SECRET'], true], ['wecom_callback', ['WECOM_CALLBACK_CORP_ID', 'WECOM_CALLBACK_CORP_SECRET'], true],
+  ['weixin', ['WEIXIN_TOKEN', 'WEIXIN_ACCOUNT_ID']], ['bluebubbles', ['BLUEBUBBLES_SERVER_URL', 'BLUEBUBBLES_PASSWORD'], true],
+  ['qqbot', ['QQ_APP_ID', 'QQ_CLIENT_SECRET']], ['yuanbao', ['YUANBAO_APP_ID', 'YUANBAO_APP_SECRET'], true], ['relay', ['GATEWAY_RELAY_URL']], ['api_server', ['API_SERVER_KEY']],
+];
+const presence = (platform, env) => {
+  const entry = envCredentials.find(([name]) => name === platform);
+  return entry == null ? null : entry[2] ? entry[1].every(key => Object.hasOwn(env, key)) : entry[1].some(key => Object.hasOwn(env, key));
+};
+async function rows(selected, context) {
+  if (selected.harness != null && !CHANNEL_HARNESSES.includes(selected.harness)) refuse(`harness \`${selected.harness}\` has no channel concept (channels exist for: ${CHANNEL_HARNESSES.join(', ')})`, -32020);
+  const homes = Object.fromEntries(Object.entries(selected.homes).map(([key, path]) => [key, filesystem(path, context.cwd)])), result = [];
+  for (const harness of CHANNEL_HARNESSES) {
+    if (selected.harness != null && selected.harness !== harness) continue;
+    const counts = await channelSessionCounts(harness, homes);
+    let channels;
+    try {
+      if (harness === 'openclaw') channels = readOpenclawChannelHome(homes.openclaw);
+      else {
+        const profiles = readChannelFolderHome(harness === 'hermes' ? parent(homes.hermes) ?? '.' : homes.orchestrator, harness);
+        channels = [];
+        for (const name of [...profiles.keys()].filter(name => harness !== 'hermes' || name === 'default').sort((a, b) => Number(a !== 'default') - Number(b !== 'default') || compare(a, b))) {
+          const recorded = new Map(profiles.get(name).map(row => [row.name, row]));
+          if (harness === 'hermes') for (const [platform] of envCredentials) if (!recorded.has(platform) && presence(platform, context.env) === true) recorded.set(platform, { name: platform, account: null, enabled: true, configured: false });
+          for (const key of [...recorded.keys()].sort(compare)) {
+            const row = recorded.get(key), present = harness === 'hermes' ? presence(key, context.env) : null;
+            channels.push({ ...row, kind: key, configured: present === true || row.configured || present == null });
+          }
+        }
+      }
+    } catch { continue; }
+    for (const row of channels) {
+      const value = { name: row.name, harness, kind: row.kind, account: row.account, enabled: row.enabled, configured: row.configured, status: 'unknown', sessions: null };
+      setNativeInteger(value, 'sessions', counts.get(row.kind) ?? 0n); result.push(value);
+    }
+  }
+  return result;
+}
+export async function listChannels(params, { env = process.env, cwd = process.cwd() } = {}) {
+  return { schema: 'supercode.channels.v1', channels: await rows(query(params, env), { env, cwd }) };
+}
+export async function channelStatus(params, { env = process.env, cwd = process.cwd() } = {}) {
+  const selected = query(params, env);
+  if (selected.harness == null) refuse('`harness` is required');
+  if (selected.name == null) refuse('`name` is required');
+  const channel = (await rows(selected, { env, cwd })).find(row => row.name === selected.name);
+  if (channel == null) refuse(`\`${selected.harness}\` has no channel \`${selected.name}\``);
+  return { schema: 'supercode.channels.v1', channel };
+}

package/corpus-audit-records.mjs

@@ -0,0 +1,192 @@
+import {parseNativeJson, setNativeInteger, stringifyNativeJson} from './native-json.mjs';
+import {field, stringField} from './session-codec-utils.mjs';
+import {nativeTrim} from './text.mjs';
+import {opencodeImagePart} from './session-opencode-content.mjs';
+import {piImageShape} from './session-pi-content.mjs';
+import {auditClaudeRecord, auditCodexRecord} from './corpus-audit-schema.mjs';
+
+export const AUDIT_COVERAGE = Object.freeze(['normalized', 'retained', 'dropped', 'unmodeled']);
+export const auditOrder = (a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b));
+const sorted = map => [...map].sort(([a], [b]) => auditOrder(a, b));
+const next = count => ((count ?? 0n) + 1n) & ((1n << 64n) - 1n);
+const countValue = (count, coverage, unmodeled_fields) => {
+  const out = {coverage}; setNativeInteger(out, 'count', count);
+  if (unmodeled_fields !== undefined) out.unmodeled_fields = unmodeled_fields;
+  return out;
+};
+const ordered = entries => parseNativeJson(`{${entries.map(([key, value]) => `${JSON.stringify(key)}:${stringifyNativeJson(value)}`).join(',')}}`);
+
+/** Counter state is exact u64; the public value keeps ordinary JS numbers
+ * with the shared native-JSON facts used by the lossless serializer. */
+export class CorpusAuditReport {
+  constructor(corpus) {
+    this.corpus = corpus; this.files = 0n; this.lines = 0n; this.parse_errors = 0n;
+    this.records = new Map(); this.blocks = new Map(); this.tools = new Map(); this.notes = new Map();
+  }
+  bump(key, coverage, extra = []) {
+    const tally = this.records.get(key) ?? {coverage, count:0n, fields:new Map()};
+    tally.coverage = coverage; tally.count = next(tally.count);
+    for (const key of extra) tally.fields.set(key, next(tally.fields.get(key)));
+    this.records.set(key, tally);
+  }
+  block(tag, coverage) {
+    const buckets = this.blocks.get(tag) ?? new Map();
+    buckets.set(coverage, next(buckets.get(coverage))); this.blocks.set(tag, buckets);
+  }
+  tool(name) { this.tools.set(name, next(this.tools.get(name))); }
+  note(name) { this.notes.set(name, next(this.notes.get(name))); }
+  toJSON() {
+    const records = ordered(sorted(this.records).map(([key, tally]) => [key, countValue(tally.count, tally.coverage, [...tally.fields.keys()].sort(auditOrder))]));
+    const blocks = ordered(sorted(this.blocks).map(([key, buckets]) => [key, AUDIT_COVERAGE.filter(c => buckets.has(c)).map(c => countValue(buckets.get(c), c))]));
+    const tools = ordered(sorted(this.tools)), notes = ordered(sorted(this.notes));
+    const out = {corpus:this.corpus};
+    for (const key of ['files', 'lines', 'parse_errors']) setNativeInteger(out, key, this[key]);
+    Object.assign(out, {records, blocks, tools, notes});
+    return out;
+  }
+}
+
+// The CLI enables serde_json preserve_order: raw Grok scalar/object text
+// and constructed report objects retain their native insertion order.
+export const auditJsonText = stringifyNativeJson;
+function imageCoverage(source) {
+  const type = stringField(source, 'type');
+  if (type === 'base64') return stringField(source, 'media_type') && stringField(source, 'data') ? 'normalized' : 'dropped';
+  return type === 'url' && stringField(source, 'url') ? 'normalized' : 'dropped';
+}
+function auditBlocks(blocks, rawPayload, report) {
+  const raw = field(rawPayload, 'content');
+  for (const [at, block] of blocks.entries()) {
+    const tag = block.tag ?? stringField(Array.isArray(raw) ? raw[at] : null, 'type') ?? '<no-type>';
+    const coverage = block.tag == null ? 'unmodeled' : block.tag === 'image' ? imageCoverage(field(block.value, 'source'))
+      : ['thinking', 'redacted_thinking'].includes(block.tag) ? 'dropped' : 'normalized';
+    report.block(tag, coverage);
+    const nested = field(block.value, 'content');
+    if (block.tag === 'tool_result' && Array.isArray(nested)) for (const item of nested) {
+      if (stringField(item, 'type') === 'image') report.block('tool_result/image', imageCoverage(field(item, 'source')));
+    }
+  }
+}
+function claude(raw, report) {
+  const record = auditClaudeRecord(raw), {tag, extra} = record;
+  if (tag == null) { report.bump(`<line>/${stringField(raw, 'type') ?? '<no-type>'}`, 'unmodeled'); return; }
+  if (tag === 'user' || tag === 'assistant') {
+    report.bump(tag, 'normalized', extra);
+    if (record.sidechain) report.note('sidechain (subagent) lines — flattened, not separated');
+    if (typeof record.content === 'string') report.block('text', 'normalized');
+    else {
+      // This lane deliberately receives no raw block array, as the replaced
+      // reader did: unknown Claude block tags are tallied as <no-type>.
+      auditBlocks(record.content, null, report);
+      for (const block of record.content) if (block.tag === 'tool_use') report.tool(block.value.name);
+    }
+  } else if (tag === 'system') report.bump(`system/${record.subtype}`, ['scheduled_task_fire', 'local_command', 'away_summary'].includes(record.subtype) ? 'normalized' : 'dropped', extra);
+  else report.bump(tag, ['attachment', 'fork-context-ref'].includes(tag) ? 'normalized'
+    : ['permission-mode', 'mode', 'last-prompt', 'queue-operation', 'ai-title', 'pr-link', 'frame-link', 'agent-name', 'worktree-state'].includes(tag) ? 'dropped' : 'unmodeled', extra);
+}
+function codex(raw, report) {
+  const record = auditCodexRecord(raw), {tag, extra} = record;
+  if (tag == null) { report.bump(`<line>/${stringField(raw, 'type') ?? '<no-type>'}`, 'unmodeled'); return; }
+  if (tag === 'response_item') {
+    const item = record.item, name = item.tag ?? stringField(raw.payload, 'type') ?? '<no-type>';
+    const coverage = item.tag == null ? 'unmodeled' : name === 'reasoning' ? 'retained' : 'normalized';
+    report.bump(`response_item/${name}${name === 'message' ? `[${item.value.role}]` : ''}`, coverage, item.extra);
+    if (name === 'message') auditBlocks(item.content, raw.payload, report);
+    if (name === 'function_call' || name === 'custom_tool_call' && item.value.name != null) report.tool(item.value.name);
+  } else if (tag === 'event_msg') {
+    const sub = record.payload.type ?? '?';
+    report.bump(`event_msg/${sub}`, ['agent_message', 'thread_rolled_back', 'thread_goal_updated', 'entered_review_mode', 'exited_review_mode'].includes(sub) ? 'retained' : 'dropped', extra);
+  } else report.bump(tag, tag === 'compacted' ? 'normalized' : 'retained', extra);
+}
+function pi(raw, report) {
+  const type = stringField(raw, 'type');
+  if (type === 'message') {
+    const message = field(raw, 'message'), role = stringField(message, 'role'), content = field(message, 'content');
+    const unknownImage = ['user', 'toolResult', 'custom'].includes(role) && Array.isArray(content)
+      && content.some(item => stringField(item, 'type') === 'image' && piImageShape(item) == null);
+    report.bump(unknownImage ? 'message/UnknownImageShape' : ['user', 'assistant', 'toolResult', 'bashExecution', 'custom'].includes(role) ? `message/${role}` : `message/UnknownRole:${role ?? '<none>'}`,
+      unknownImage || !['user', 'assistant', 'toolResult', 'bashExecution', 'custom'].includes(role) ? 'unmodeled' : 'normalized');
+  } else if (['session', 'custom_message', 'compaction', 'branch_summary', 'model_change', 'session_info'].includes(type)) report.bump(type, 'normalized');
+  else if (['thinking_level_change', 'custom', 'label'].includes(type)) report.bump(type, 'dropped');
+  else report.bump(`<line>/${type ?? '<no-type>'}`, 'unmodeled');
+}
+function gemini(raw, report) {
+  const type = stringField(raw, 'type');
+  if (type == null) {
+    const key = stringField(raw, 'sessionId') != null ? 'session_header' : field(raw, '$set') !== undefined ? 'metadata_update' : '<line>/<no-type>';
+    report.bump(key, key.startsWith('<line>') ? 'unmodeled' : 'retained'); return;
+  }
+  if (type === 'user' || type === 'gemini') {
+    report.bump(type, 'normalized');
+    const content = field(raw, 'content');
+    if (Array.isArray(content)) for (const part of content) {
+      if (field(part, 'text') !== undefined) report.bump('content/text', 'normalized');
+      else if (field(part, 'inlineData') !== undefined) report.bump('content/inlineData', 'normalized');
+      else if (field(part, 'functionCall') !== undefined || field(part, 'functionResponse') !== undefined) {
+        const key = field(part, 'functionCall') !== undefined ? 'functionCall' : 'functionResponse';
+        report.bump(`content/${key}`, 'normalized'); const name = stringField(field(part, key), 'name'); if (name != null) report.tool(name);
+      } else report.bump('content/unknown', 'unmodeled');
+    } else if (content !== null && typeof content !== 'string') report.bump('content/nonstandard', 'unmodeled');
+    if (field(raw, 'thoughts') !== undefined) report.bump('thoughts', 'retained');
+  } else if (type === 'info' || type === 'error') report.bump(type, 'retained');
+  else report.bump(`<line>/${type}`, 'unmodeled');
+}
+function grokContent(content, prefix, coverage, report) {
+  if (Array.isArray(content)) for (const item of content) {
+    const modeled = typeof item === 'string' || stringField(item, 'text') != null;
+    report.bump(`${prefix}/content/${stringField(item, 'type') ?? '<no-type>'}`, coverage === 'dropped' ? 'dropped' : modeled ? 'normalized' : 'unmodeled');
+  } else report.bump(`${prefix}/content${content === undefined ? ':<missing>' : ''}`, content === undefined && coverage !== 'dropped' ? 'unmodeled' : coverage);
+}
+function grok(raw, report) {
+  const type = stringField(raw, 'type'), content = field(raw, 'content');
+  if (type === 'system') report.bump('system', typeof content === 'string' ? 'retained' : 'unmodeled');
+  else if (type === 'user') {
+    const text = nativeTrim(typeof content === 'string' ? content : Array.isArray(content)
+      ? content.map(item => stringField(item, 'text') ?? (typeof item === 'string' ? item : null)).filter(item => item != null).join('\n')
+      : content === undefined ? '' : auditJsonText(content));
+    let key = 'user', coverage = 'normalized';
+    if (stringField(raw, 'synthetic_reason') === 'supercode_system_event') key = 'user/system_event';
+    else if (text.startsWith('<user_info>')) { key = 'user/injected_context:user_info'; coverage = 'dropped'; }
+    else if (text.startsWith('<system-reminder>')) { key = 'user/injected_context:system-reminder'; coverage = 'dropped'; }
+    else if (text === '') { key = 'user/empty'; coverage = 'dropped'; }
+    else if (text.startsWith('<user_query>') && text.endsWith('</user_query>') && nativeTrim(text.slice(12, -13)) === '') { key = 'user/empty_query'; coverage = 'dropped'; }
+    report.bump(key, coverage); grokContent(content, key, coverage, report);
+  } else if (type === 'assistant') {
+    report.bump('assistant', content == null || typeof content === 'string' ? 'normalized' : 'unmodeled');
+    const calls = field(raw, 'tool_calls');
+    if (Array.isArray(calls)) for (const call of calls) {
+      const name = stringField(call, 'name'), modeled = name != null && stringField(call, 'id') != null;
+      report.bump(`assistant/tool_call${modeled ? '' : ':invalid'}`, modeled ? 'normalized' : 'unmodeled'); if (name != null) report.tool(name);
+    } else if (calls !== undefined) report.bump('assistant/tool_calls:non-array', 'unmodeled');
+  } else if (type === 'tool_result') {
+    report.bump(type, stringField(raw, 'tool_call_id') != null ? 'normalized' : 'unmodeled'); grokContent(content, type, 'normalized', report);
+  } else if (type === 'reasoning' || type === 'backend_tool_call') report.bump(type, 'dropped');
+  else report.bump(`<line>/${type ?? '<no-type>'}`, 'unmodeled');
+}
+const partTypes = new Set(['text', 'reasoning', 'tool', 'file', 'step-start', 'step-finish', 'snapshot', 'patch', 'agent', 'subtask', 'retry', 'compaction']);
+function opencode(raw, report) {
+  const key = field(raw, 'key'), value = field(raw, 'value');
+  if (!Array.isArray(key)) { report.bump('<line>/<no-key>', 'unmodeled'); return; }
+  const kind = typeof key[0] === 'string' ? key[0] : '<no-kind>';
+  if (kind === 'message') {
+    const role = stringField(value, 'role'); report.bump(['user', 'assistant'].includes(role) ? `message/${role}` : `message/UnknownRole:${role ?? '<none>'}`, ['user', 'assistant'].includes(role) ? 'normalized' : 'unmodeled');
+  } else if (kind === 'part') {
+    const type = stringField(value, 'type');
+    if (!partTypes.has(type)) report.bump(`part/UnknownType:${type ?? '<none>'}`, 'unmodeled');
+    else if (type === 'tool') {
+      const name = stringField(value, 'tool'); if (name != null) report.tool(name);
+      const status = stringField(field(value, 'state'), 'status'), known = ['pending', 'running', 'completed', 'error'].includes(status);
+      report.bump(known ? `part/tool/${status}` : `part/tool/UnknownStatus:${status ?? '<none>'}`, known ? 'normalized' : 'unmodeled');
+    } else if (type === 'text') report.bump(`part/text${field(value, 'ignored') === true ? ':ignored' : ''}`, field(value, 'ignored') === true ? 'dropped' : 'normalized');
+    else if (type === 'file') { const modeled = opencodeImagePart(value) != null; report.bump(`part/file${modeled ? '' : ':residue'}`, modeled ? 'normalized' : 'dropped'); }
+    else report.bump(`part/${type}`, type === 'compaction' ? 'normalized' : 'dropped');
+  } else if (['session', 'session_diff', 'todo'].includes(kind)) report.bump(kind, 'normalized');
+  else report.bump(`<line>/${kind}`, 'unmodeled');
+}
+const classifiers = {'claude-code':claude, codex, pi, opencode, grok, gemini};
+export function auditCorpusLine(line, report) {
+  let raw;
+  try { raw = parseNativeJson(line); classifiers[report.corpus](raw, report); }
+  catch (error) { if (!(error instanceof SyntaxError || error instanceof TypeError)) throw error; report.parse_errors = next(report.parse_errors); }
+  return raw;
+}

package/corpus-audit-render.mjs

@@ -0,0 +1,47 @@
+import {nativeInteger, nativeObjectKeys} from './native-json.mjs';
+
+const scalars = text => [...text];
+const pad = (text, size, left = false) => { const blank = ' '.repeat(Math.max(0, size - scalars(text).length)); return left ? blank + text : text + blank; };
+const truncate = (text, size) => scalars(text).length <= size ? text : `${scalars(text).slice(0, size - 1).join('')}…`;
+const thousands = value => String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
+const count = (value, key = 'count') => nativeInteger(value, key) ?? 0n;
+const descending = (a, b) => a > b ? -1 : a < b ? 1 : 0;
+/** Compact terminal rendering retains the native caps, stable count order,
+ * per-instance block buckets and gap summary. Notes remain in the JSON view. */
+export function formatCorpusAudit(report, {paint = (_role, text) => text, bold = text => text} = {}) {
+  const section = title => `${paint('ACCENT', '▌')} ${bold(title)}`;
+  const rule = paint('DIM', '─'.repeat(60));
+  const kv = (key, value) => `  ${paint('MUTED', pad(key, 12))}  ${value}`;
+  const glyph = coverage => paint(coverage === 'normalized' ? 'OK' : coverage === 'retained' ? 'TEAL' : coverage === 'dropped' ? 'DIM' : 'ERR', coverage === 'normalized' ? '✓' : coverage === 'retained' ? '◆' : coverage === 'dropped' ? '◦' : '✗');
+  const row = (label, value, extra = '') => {
+    const name = truncate(label, 38), number = thousands(count(value));
+    return `  ${glyph(value.coverage)}  ${paint('MUTED', name)} ${paint('DIM', '.'.repeat(Math.max(0, 40 - scalars(name).length)))} ${' '.repeat(Math.max(0, 9 - scalars(number).length))}${bold(number)}${extra}\n`;
+  };
+  const entries = object => nativeObjectKeys(object).map(key => [key, object[key]]);
+  const sort = rows => rows.sort(([, a], [, b]) => descending(count(a), count(b)));
+  let text = `\n${section(`coverage audit · ${report.corpus}`)}\n${rule}\n\n`;
+  text += `${kv('files', bold(String(count(report, 'files'))))}\n${kv('lines', bold(thousands(count(report, 'lines'))))}\n${kv('parse errors', paint(count(report, 'parse_errors') === 0n ? 'OK' : 'ERR', String(count(report, 'parse_errors'))))}\n${rule}\n\n`;
+  const records = entries(report.records), blocks = entries(report.blocks).flatMap(([tag, rows]) => rows.map(value => [tag, value]));
+  text += `${section(`Records  (${records.length} types)`)}\n`;
+  for (const [key, value] of sort(records).slice(0, 12)) text += row(key, value, value.unmodeled_fields.length ? paint('DIM', `  +${value.unmodeled_fields.length}`) : '');
+  if (records.length > 12) text += `  ${paint('DIM', ' ')}  ${paint('DIM', `… ${records.length - 12} more — use --json for the full report`)}\n`;
+  text += '\n';
+  if (blocks.length) { text += `${section(`Content blocks  (${nativeObjectKeys(report.blocks).length} types)`)}\n`; for (const [key, value] of sort(blocks)) text += row(key, value); text += '\n'; }
+  const tools = entries(report.tools).sort(([a], [b]) => descending(count(report.tools, a), count(report.tools, b)));
+  if (tools.length) {
+    text += `${section('Tools observed')}\n`;
+    const max = count(report.tools, tools[0][0]) || 1n;
+    for (const [key] of tools.slice(0, 8)) {
+      const n = count(report.tools, key), bar = '█'.repeat(Math.max(1, Math.round(Number(n) / Number(max) * 28)));
+      text += `     ${paint('MUTED', pad(truncate(key, 16), 16))}  ${paint('DIM', pad(thousands(n), 8, true))}  ${paint('ACCENT', bar)}\n`;
+    }
+    text += '\n';
+  }
+  const gaps = [...entries(report.records).filter(([, value]) => value.coverage === 'unmodeled'), ...entries(report.blocks).flatMap(([tag, rows]) => rows.filter(value => value.coverage === 'unmodeled').map(value => [`content block \`${tag}\``, value]))];
+  if (!gaps.length) text += `  ${paint('OK', '✓')} ${paint('MUTED', 'every record type is modeled — no gaps')}\n`;
+  else {
+    text += `${section('Gaps (not yet modeled)')}\n`;
+    for (const [key, value] of sort(gaps)) text += `  ${paint('ERR', '✗')}  ${paint('MUTED', pad(key, 40))}  ${paint('DIM', `${thousands(count(value))} occurrence(s)`)}\n`;
+  }
+  return `${text}\n`;
+}

package/corpus-audit-schema.mjs

@@ -0,0 +1,89 @@
+// The audit's typed envelope boundary. Unknown variants accept their opaque
+// fields; modeled fields keep their required/default/nullable native meanings.
+import {nativeDuplicateObjectKeys, nativeObjectKeys} from './native-json.mjs';
+import {isObject} from './session-codec-utils.mjs';
+
+const invalid = () => { throw new TypeError('record does not match the corpus schema'); };
+const string = value => typeof value === 'string';
+const nullableString = value => value === null || string(value);
+const any = () => true;
+const required = check => ({check, required:true});
+const fields = (value, model) => {
+  if (!isObject(value)) invalid();
+  if (nativeDuplicateObjectKeys(value).some(key => Object.hasOwn(model, key))) invalid();
+  for (const [key, rule] of Object.entries(model)) {
+    if (!Object.hasOwn(value, key)) { if (rule.required) invalid(); }
+    else if (!(rule.check ?? rule)(value[key])) invalid();
+  }
+  return nativeObjectKeys(value).filter(key => !Object.hasOwn(model, key));
+};
+const tagged = value => { fields(value, {type:required(string)}); return value.type; };
+const blockModels = {
+  text:{text:string}, thinking:{thinking:string}, redacted_thinking:{data:nullableString},
+  tool_use:{id:required(string), name:required(string), input:any},
+  tool_result:{tool_use_id:nullableString, content:any}, image:{source:any},
+  input_text:{text:string}, output_text:{text:string}, input_image:{}, fallback:{},
+};
+
+export function auditBlock(value) {
+  const tag = tagged(value);
+  if (!Object.hasOwn(blockModels, tag)) return {tag:null, value};
+  fields(value, {type:required(string), ...blockModels[tag]});
+  return {tag, value};
+}
+function blocks(value) {
+  if (!Array.isArray(value)) invalid();
+  return value.map(auditBlock);
+}
+const claudeMarkers = new Set(['attachment', 'file-history-snapshot', 'file-history-delta', 'ai-title',
+  'permission-mode', 'mode', 'last-prompt', 'queue-operation', 'pr-link', 'frame-link', 'agent-name',
+  'started', 'result', 'worktree-state', 'fork-context-ref']);
+const meta = {uuid:nullableString, parent_uuid:nullableString, is_sidechain:value => typeof value === 'boolean',
+  session_id:nullableString, cwd:nullableString, git_branch:nullableString};
+
+export function auditClaudeRecord(value) {
+  const tag = tagged(value);
+  if (tag === 'user' || tag === 'assistant') {
+    const extra = fields(value, {type:required(string), message:required(isObject), ...meta});
+    fields(value.message, {role:nullableString, model:nullableString, content:any});
+    const content = Object.hasOwn(value.message, 'content') ? value.message.content : [];
+    return {tag, extra, content:typeof content === 'string' ? content : blocks(content), sidechain:value.is_sidechain === true};
+  }
+  if (tag === 'system') return {tag, extra:fields(value, {type:required(string), subtype:nullableString}), subtype:value.subtype ?? '?'};
+  if (claudeMarkers.has(tag)) return {tag, extra:fields(value, {type:required(string)})};
+  return {tag:null, extra:[]};
+}
+
+const itemModels = {
+  message:{role:required(string), content:any},
+  function_call:{name:required(string), arguments:string, call_id:required(string)},
+  function_call_output:{call_id:required(string), output:any},
+  custom_tool_call:{name:nullableString, input:any, call_id:nullableString},
+  custom_tool_call_output:{call_id:nullableString, output:any},
+  reasoning:{}, web_search_call:{}, tool_search_call:{}, tool_search_output:{}, image_generation_call:{},
+};
+function auditCodexItem(value) {
+  const tag = tagged(value);
+  if (!Object.hasOwn(itemModels, tag)) return {tag:null, extra:[], value};
+  const extra = fields(value, {type:required(string), ...itemModels[tag]});
+  return {tag, extra, value, ...(tag === 'message' ? {content:blocks(Object.hasOwn(value, 'content') ? value.content : [])} : {})};
+}
+const payloadModels = {
+  session_meta:{id:nullableString, cwd:nullableString, base_instructions:any, model_provider:nullableString},
+  turn_context:{model:nullableString, cwd:nullableString}, event_msg:{type:nullableString},
+};
+
+export function auditCodexRecord(value) {
+  fields(value, {timestamp:nullableString});
+  const tag = tagged(value);
+  if (tag === 'response_item') {
+    fields(value, {timestamp:nullableString, type:required(string), payload:required(isObject)});
+    return {tag, item:auditCodexItem(value.payload)};
+  }
+  if (Object.hasOwn(payloadModels, tag)) {
+    fields(value, {timestamp:nullableString, type:required(string), payload:required(isObject)});
+    return {tag, extra:fields(value.payload, payloadModels[tag]), payload:value.payload};
+  }
+  if (tag === 'compacted') { fields(value, {timestamp:nullableString, type:required(string), payload:any}); return {tag, extra:[]}; }
+  return {tag:null, extra:[]};
+}

package/corpus-audit.d.ts

@@ -0,0 +1,16 @@
+export type AuditFormat = 'claude-code' | 'codex' | 'opencode' | 'pi' | 'grok' | 'gemini' | 'goose';
+export type AuditCoverage = 'normalized' | 'retained' | 'dropped' | 'unmodeled';
+export interface AuditTally {coverage:AuditCoverage; count:number; unmodeled_fields:string[]}
+export interface CorpusAuditReport {
+  corpus:AuditFormat; files:number; lines:number; parse_errors:number;
+  records:Record<string, AuditTally>;
+  blocks:Record<string, {coverage:AuditCoverage; count:number}[]>;
+  tools:Record<string, number>; notes:Record<string, number>;
+}
+export const AUDIT_FORMATS: readonly AuditFormat[];
+export const AUDIT_COVERAGE: readonly AuditCoverage[];
+/** Counts expose JS numbers; corpusAuditJson retains the exact native u64 wire values. */
+export function auditCorpus(root:string, options:{format:AuditFormat; limit?:number | bigint | null}):Promise<CorpusAuditReport>;
+export function corpusAuditJson(report:CorpusAuditReport):string;
+export function formatCorpusAuditInventory(report:CorpusAuditReport):string;
+export function formatCorpusAudit(report:CorpusAuditReport, style?:{paint?:(role:string, text:string) => string; bold?:(text:string) => string}):string;

package/corpus-audit.mjs

@@ -0,0 +1,132 @@
+import {readFile, readdir, stat} from 'node:fs/promises';
+import {basename, extname, join} from 'node:path';
+import {decodeSessionUtf8} from './session-file.mjs';
+import {nativeTrim} from './text.mjs';
+import {field} from './session-codec-utils.mjs';
+import {nativeInteger, stringifyNativeJsonPretty} from './native-json.mjs';
+import {readForeignStore} from './foreign-store.mjs';
+import {readGooseSqlite} from './session-goose.mjs';
+import {readOpenCodeCorpus} from './session-opencode-store.mjs';
+import {CorpusAuditReport, auditCorpusLine, auditOrder} from './corpus-audit-records.mjs';
+
+export {formatCorpusAudit} from './corpus-audit-render.mjs';
+export {AUDIT_COVERAGE} from './corpus-audit-records.mjs';
+export const AUDIT_FORMATS = Object.freeze(['claude-code', 'codex', 'opencode', 'pi', 'grok', 'gemini', 'goose']);
+const info = async path => { try { return await stat(path); } catch { return null; } };
+const entries = async path => { try { return await readdir(path, {withFileTypes:true}); } catch { return []; } };
+const cap = (rows, limit) => limit == null || limit >= BigInt(rows.length) ? rows : rows.slice(0, Number(limit));
+function nativeLimit(value) {
+  if (value == null) return null;
+  if (typeof value !== 'bigint' && !Number.isSafeInteger(value)) throw new TypeError('audit limit must be an exact nonnegative integer');
+  const whole = BigInt(value);
+  if (whole < 0n || whole > (1n << 64n) - 1n) throw new RangeError('audit limit is outside native usize');
+  return whole;
+}
+async function jsonlFiles(root) {
+  const out = [], pending = [root];
+  // Native walks ordinary directories without following child symlinks;
+  // root symlinks are resolved by the root's own filesystem lookup.
+  if (extname(root) === '.jsonl' && await info(root)) out.push(root);
+  if (!(await info(root))?.isDirectory()) return out;
+  while (pending.length) {
+    for (const entry of await entries(pending.pop())) {
+      const path = join(entry.parentPath ?? entry.path, entry.name);
+      if (extname(path) === '.jsonl') out.push(path);
+      if (entry.isDirectory()) pending.push(path);
+    }
+  }
+  return out.sort(auditOrder);
+}
+async function openCodeDatabase(root) {
+  const override = process.env.OPENCODE_DB;
+  if (override !== undefined && (await info(override))?.isFile()) return override;
+  const candidates = (await entries(root)).filter(entry => entry.name.startsWith('opencode') && entry.name.endsWith('.db'))
+    .map(entry => join(root, entry.name)).sort(auditOrder);
+  return candidates.find(path => basename(path) === 'opencode.db') ?? candidates[0] ?? null;
+}
+async function goose(root, limit, report) {
+  // Native discovery quietly excludes a store it cannot read. A selected
+  // session that fails full normalization is instead named in report.notes.
+  const direct = join(root, 'sessions.db');
+  const path = (await info(root))?.isFile() ? root : (await info(direct))?.isFile() ? direct : join(root, 'sessions', 'sessions.db');
+  let rows;
+  try { rows = (await readForeignStore('goose', path)).result.rows; }
+  catch { return; }
+  rows.sort((a, b) => {
+    const left = a.updated_at_ms, right = b.updated_at_ms;
+    return left === right ? auditOrder(a.locator.session_id, b.locator.session_id) : left == null ? 1 : right == null ? -1 : right - left;
+  });
+  for (const row of cap(rows, limit)) {
+    const id = row.locator.session_id;
+    let session;
+    try { session = (await readGooseSqlite(path, id)).result; }
+    catch (error) { report.note(`Goose session ${id} failed to load: ${error.message}`); continue; }
+    report.files++;
+    for (const message of session.messages) {
+      report.lines++; report.bump(`message/${message.role.toLowerCase()}`, 'normalized');
+      for (const call of message.tool_calls ?? []) report.tool(call.function.name);
+    }
+  }
+}
+
+/** Read-only corpus audit. JSONL counts selected paths before reading them;
+ * store formats count the native selected sessions, not database files. */
+export async function auditCorpus(root, {format, limit = null} = {}) {
+  if (typeof root !== 'string') throw new TypeError('audit root must be a path string');
+  if (!AUDIT_FORMATS.includes(format)) throw new TypeError(`unknown corpus format ${format}`);
+  limit = nativeLimit(limit);
+  const report = new CorpusAuditReport(format);
+  if (format === 'goose') await goose(root, limit, report);
+  else {
+    const database = format === 'opencode' ? await openCodeDatabase(root) : null;
+    if (database != null) {
+      try {
+        const lines = await readOpenCodeCorpus(database, limit);
+        for (const source of lines) {
+          const line = nativeTrim(source); if (line === '') continue;
+          report.lines++; const raw = auditCorpusLine(line, report);
+          if (field(raw, 'key')?.[0] === 'session') report.files++;
+        }
+      } catch (error) { report.note(`opencode_sqlite_error: ${error.message}`); }
+    } else {
+      let files = await jsonlFiles(root);
+      if (format === 'grok') files = files.filter(path => basename(path) === 'chat_history.jsonl');
+      for (const path of cap(files, limit)) {
+        report.files++;
+        let source;
+        try { source = decodeSessionUtf8(await readFile(path)); } catch { continue; }
+        for (const line of source.split('\n').map(nativeTrim)) {
+          if (line === '') continue;
+          report.lines++; auditCorpusLine(line, report);
+        }
+      }
+    }
+  }
+  return report.toJSON();
+}
+
+/** Native-compatible report bytes, including exact integer counters and LF. */
+export function corpusAuditJson(report) { return `${stringifyNativeJsonPretty(report)}\n`; }
+
+/** The library inventory view, separate from the CLI's compact terminal view. */
+export function formatCorpusAuditInventory(report) {
+  const symbols = {normalized:'✅ normalized', retained:'◆ retained  ', dropped:'➖ dropped   ', unmodeled:'❌ UNMODELED '};
+  const count = (row, key = 'count') => nativeInteger(row, key) ?? 0n;
+  const sorted = object => Object.entries(object).sort(([a], [b]) => auditOrder(a, b));
+  const pad = (text, width) => text + ' '.repeat(Math.max(0, width - [...text].length));
+  let text = `# Coverage audit: ${report.corpus ?? '?'}\nfiles=${count(report, 'files')} lines=${count(report, 'lines')} parse_errors=${count(report, 'parse_errors')}\n\n`;
+  text += '## Records (discriminant → coverage, count, unmodeled fields)\n';
+  for (const [key, row] of sorted(report.records)) text += `  ${symbols[row.coverage]}  ${pad(key, 40)} ${String(count(row)).padStart(9)}${row.unmodeled_fields.length ? `   unmodeled fields: ${row.unmodeled_fields.join(', ')}` : ''}\n`;
+  const blocks = sorted(report.blocks).flatMap(([key, rows]) => rows.map(row => [key, row]));
+  if (blocks.length) { text += '\n## Content blocks\n'; for (const [key, row] of blocks) text += `  ${symbols[row.coverage]}  ${pad(key, 28)} ${String(count(row)).padStart(9)}\n`; }
+  if (Object.keys(report.notes).length) { text += '\n## Structural notes\n'; for (const [key] of sorted(report.notes)) text += `  ${key}: ${count(report.notes, key)}\n`; }
+  if (Object.keys(report.tools).length) {
+    text += '\n## Tools observed (top 30 by frequency)\n';
+    const rows = sorted(report.tools).sort(([a], [b]) => count(report.tools, a) > count(report.tools, b) ? -1 : count(report.tools, a) < count(report.tools, b) ? 1 : 0);
+    for (const [key] of rows.slice(0, 30)) text += `  ${String(count(report.tools, key)).padStart(9)}  ${key}\n`;
+  }
+  text += '\n## Summary of gaps (UNMODELED or dropped, non-UI)\n';
+  for (const [key, row] of sorted(report.records)) if (row.coverage === 'unmodeled') text += `  ❌ ${key} (${count(row)} occurrences) — not modeled\n`;
+  for (const [key, row] of blocks) if (row.coverage === 'unmodeled') text += `  ❌ content block \`${key}\` (${count(row)}) — not modeled\n`;
+  return text;
+}

package/discovery.mjs

@@ -10,7 +10,7 @@ export async function enrichDescriptor(query, descriptor, { codexTopics, metrics
   const row = { ...descriptor };
   const take = async (read) => {
     try { return await read(); }
-    catch (error) { previewFailure?.(row.locator, error); return []; }
+    catch (error) { if (metrics) metrics.preview_failures = (metrics.preview_failures ?? 0) + 1; previewFailure?.(row.locator, error); return []; }
   };
   if (query.include_topic_candidates) {
     const history = row.locator.harness === 'codex' ? codexTopics?.get(row.locator.session_id) : null;
@@ -30,8 +30,19 @@ export async function enrichDescriptor(query, descriptor, { codexTopics, metrics
 export const readsPreviews = (query) => query.include_latest_candidates !== false || Boolean(query.include_topic_candidates);
 
 export async function enrichIndexPage(query, rows, options) {
-  const result = [];
-  for (const row of rows) result.push(await enrichDescriptor(query, row, options));
+  // A page used to await every open/read/stat/close before starting the next row. Keep a small number of
+  // independent reads in flight, preserve row order and retain every fingerprint/failure check. Never fan out
+  // with Promise.all(rows): a whole-index request can contain thousands of transcripts and expanded tails.
+  const result = new Array(rows.length);
+  let next = 0;
+  const read = async () => {
+    while (next < rows.length) {
+      const at = next++;
+      result[at] = await enrichDescriptor(query, rows[at], options);
+    }
+  };
+  const finished = await Promise.allSettled(Array.from({ length: Math.min(4, rows.length) }, read));
+  for (const worker of finished) if (worker.status === 'rejected') throw worker.reason;
   return result;
 }
 
@@ -46,9 +57,10 @@ export function namesWholeSessionId(query) {
  * title supplier chooses one id by source mtime, before applying the other
  * query filters, as native claude_session_named does. Neither is caller proof. */
 export async function discoverFromIndex(query, descriptors, options = {}) {
+  const step = options.step ?? ((_name, work) => work());
   const raw = [...descriptors];
-  if (query.search_previews) return previewPage(query, raw, options);
-  const page = projectMetadataPage(query, raw, options);
+  if (query.search_previews) return step('preview search', () => previewPage(query, raw, options));
+  const page = await step('metadata projection', () => projectMetadataPage(query, raw, options));
   const name = queryName(query);
   if (query.cursor == null && name) {
     const selected = query.harnesses;
@@ -78,7 +90,7 @@ export async function discoverFromIndex(query, descriptors, options = {}) {
   }
   // Keep the native receipt from projection: its counts precede the exact-name
   // post-filter and late-name fallback. Do not silently rewrite that contract.
-  page.sessions = await enrichIndexPage(query, page.sessions, options);
+  page.sessions = await step('preview reads', () => enrichIndexPage(query, page.sessions, options));
   return page;
 }