memhouse 0.18.12 → 0.20.1

package/memhouse/shipper/forks.js

@@ -0,0 +1,233 @@
+'use strict';
+// Fork billing once (plan v4 M4; sessions-model §2.5). A fork copies its origin's messages, and
+// usually their usage with them: summed per session, the house bills the copied prefix twice, once
+// in each session. This finds the copies and takes their usage out of the fork's totals, through one
+// more `files` row on the fork — role 'inherited', like dedupe's 'legacy' row — whose contrib carries
+// the copied usage with a minus sign. buildSession sums it like a file. Nothing else is touched: the
+// copied messages stay in the fork, searchable and counted as messages; only their tokens are not
+// billed again.
+//
+// Which sessions are forks (§2.5 "Inferred links belong to the consumer"):
+// - an explicit link on the session record (kind fork or rewind_copy, in its own store); or
+// - for the harnesses that keep no fork pointer and mint new ids (OpenCode, VS Code, cursor-agent,
+//   Cursor IDE, Goose), a session born later in the same store whose first assistant text is that
+//   of another session (nomination only).
+// Either way the fork is VERIFIED on full content before anything is billed: its messages,
+// contiguous from its first, against the origin's from its first, each compared on the sha256 of
+// its full parts (kind and text, in order: tool args, full outputs, attachments by their bytes'
+// sha), main line only. A truncated text on either side leaves the fork unresolved. A copied
+// message is billed once only where its usage equals the origin's: a message the fork re-ran with
+// its own usage is billed where it ran.
+
+const INHERITED_ROLE = 'inherited';
+const INHERITED_FILE_KEY = 'inherited:fork';
+const COPY_KINDS = new Set(['fork', 'rewind_copy']);
+// Harnesses that keep no pointer from a fork to its origin and mint new ids for the copies
+// (sessions-model §2.5): only these are nominated by content.
+const INFERRED = new Set(['opencode', 'vscode', 'cursor-agent', 'cursor', 'goose']);
+// Native transcripts: no fork link and no nomination in M4 (Claude Code's and Codex's own copies are
+// not verified to carry usage), so the pass never queries for them.
+const NATIVE = new Set(['claude-code', 'codex']);
+const EPOCH_TS = '1970-01-01 00:00:00.000';
+// Nomination by content is bounded: a first reply shorter than this ("Hello! How can I help?")
+// nominates nothing, and a pass verifies at most FORK_BUDGET sessions; the rest wait for the next.
+const MIN_NOMINATION_TEXT = 40;
+const FORK_BUDGET = 100;
+const ZERO_SHA = '0'.repeat(64);
+const TOK = ['input', 'output', 'cache_read', 'cache_write', 'reasoning'];
+const sameTok = (a, b) => TOK.every((k) => Number(a[k]) === Number(b[k]));
+const anyTok = (m) => TOK.some((k) => Number(m[k]) > 0);
+
+/**
+ * The verified copied prefix of `child` from `origin` (both: house.sessionMessages lists, in order).
+ * Returns { pairs: [[childMsg, originMsg]], unresolved: reason | null }. `through`: an explicit
+ * link's last copied message id (the match stops after it).
+ */
+function verifyPrefix(child, origin, { through = '' } = {}) {
+  const pairs = [];
+  const n = Math.min(child.length, origin.length);
+  for (let i = 0; i < n; i++) {
+    const c = child[i]; const o = origin[i];
+    if (c.role !== o.role || c.sha !== o.sha) break;
+    if (c.truncated || o.truncated) return { pairs: [], unresolved: `message ${c.message_id}: truncated text, cannot verify in full` };
+    pairs.push([c, o]);
+    if (through && (c.message_id === through || o.message_id === through)) break;
+  }
+  return { pairs, unresolved: null };
+}
+
+/** The contrib of the 'inherited' row: the copied usage, negated, in the shape buildSession sums. */
+function inheritedContrib(pairs, inherits) {
+  const c = { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0, reasoning_tokens: 0, usage: [], inherits };
+  const usage = new Map();
+  let billedOnce = 0;
+  for (const [cm, om] of pairs) {
+    if (!anyTok(cm) || !sameTok(cm, om)) continue;
+    billedOnce++;
+    for (const k of TOK) c[`${k}_tokens`] -= Number(cm[k]) || 0;
+    const key = `${cm.hour}\u0000${cm.model}`;
+    const u = usage.get(key) || { hour: cm.hour, model: cm.model, messages: 0, input: 0, output: 0, cache_read: 0, cache_write: 0, reasoning: 0 };
+    for (const k of TOK) u[k] -= Number(cm[k]) || 0;
+    usage.set(key, u);
+  }
+  c.usage = [...usage.values()].sort((a, b) => a.hour.localeCompare(b.hour) || a.model.localeCompare(b.model));
+  c.inherits = { ...inherits, copied: pairs.length, billed_once: billedOnce };
+  return billedOnce ? c : null;
+}
+
+function inheritedRow(ref, contrib, prev, host) {
+  const fgen = prev ? Number(prev.fgen) + 1 : 0;
+  return {
+    harness: ref.harness, harness_path: ref.harness_path, session_id: ref.session_id, incarnation: ref.incarnation || '',
+    file_key: INHERITED_FILE_KEY, agent_id: '', role: INHERITED_ROLE, agent_type: '', agent_desc: '',
+    fgen, upto: contrib ? contrib.inherits.copied : 0, reparse_from: 0, derive_state: '{}',
+    // Never a tombstone: a deleted row leaves sessions.rev (Σ live files.ver), and the fork's rebuilt
+    // row would carry a LOWER rev than the one it replaces. Cleared, it stays live with nothing to subtract.
+    adapter_ver: 0, size: 0, inode: 0, anchor_sha: ZERO_SHA, prefix_sha: ZERO_SHA, is_deleted: 0,
+    contrib: JSON.stringify(contrib || {}), first_ts: EPOCH_TS, last_ts: EPOCH_TS, host: String(host || ''),
+  };
+}
+
+/** A comparable form of an inherited row's contrib (no derived counts that could only drift with it). */
+const contribKey = (c) => JSON.stringify(c && c.inherits ? c : null);
+
+/**
+ * The origin this session copies, verified, or null. `rows`: the session's files rows.
+ * Returns { origin: ref, pairs, inherits } | { unresolved } | null.
+ */
+async function findOrigin(ctx, ref, rows, child, meta) {
+  const store = { harness: ref.harness, host: ref.host, harness_path: ref.harness_path };
+  // Explicit links first: the adapter stated them (child side).
+  const links = (meta.links || []).filter((l) => COPY_KINDS.has(l.kind) && l.origin && !l.origin_store);
+  for (const l of links) {
+    const found = await ctx.house.sessionsById(store, String(l.origin));
+    if (found.length !== 1) return { unresolved: `explicit ${l.kind} link to ${l.origin}: ${found.length} sessions of that id in the store` };
+    const origin = { ...store, session_id: String(l.origin), incarnation: found[0].incarnation || '' };
+    const v = verifyPrefix(child, await ctx.house.sessionMessages(origin), { through: String(l.through || '') });
+    if (v.unresolved) return { unresolved: v.unresolved };
+    if (!v.pairs.length) return { unresolved: `explicit ${l.kind} link to ${l.origin}: the first message does not match` };
+    return { origin, pairs: v.pairs, inherits: { kind: l.kind, session_id: origin.session_id, incarnation: origin.incarnation, evidence: 'explicit', at: String(l.at || ''), through: v.pairs[v.pairs.length - 1][1].message_id } };
+  }
+  if (!INFERRED.has(ref.harness)) return null;
+  // Nominated by content: sessions of the store born earlier whose first assistant text is this one's.
+  const first = child.find((m) => m.role === 'assistant' && m.first_text);
+  if (!first || first.first_text.length < MIN_NOMINATION_TEXT || !Number.isFinite(meta.created)) return null;
+  const seen = new Set();
+  const cands = (await ctx.house.sessionsWithText(store, first.first_text))
+    .filter((s) => !(s.session_id === ref.session_id && (s.incarnation || '') === (ref.incarnation || '')))
+    .filter((s) => { const k = `${s.session_id}\t${s.incarnation || ''}`; if (seen.has(k)) return false; seen.add(k); return true; });
+  // The longest verified prefix wins. Two forks of one origin each verify against the other's copies
+  // as far as against the origin: an equal length goes to the earliest born (the copies' source),
+  // and only an equal length at an equal birth stays unresolved.
+  let best = null; let tie = false;
+  for (const s of cands.slice(0, 20)) {
+    const origin = { ...store, session_id: s.session_id, incarnation: s.incarnation || '' };
+    const oRows = await ctx.house.filesRows([origin]);
+    const oCreated = createdOf(oRows);
+    // Born strictly earlier: the copy cannot be the origin. No birth time on either side: no verdict.
+    if (!Number.isFinite(oCreated) || !(oCreated < meta.created)) continue;
+    const v = verifyPrefix(child, await ctx.house.sessionMessages(origin));
+    if (v.unresolved || !v.pairs.length) continue;
+    if (!best || v.pairs.length > best.pairs.length || (v.pairs.length === best.pairs.length && oCreated < best.created)) { best = { origin, pairs: v.pairs, created: oCreated }; tie = false; }
+    else if (v.pairs.length === best.pairs.length && oCreated === best.created) tie = true;
+  }
+  if (!best) return null;
+  if (tie) return { unresolved: 'two origins born at the same moment verify to the same length' };
+  return { origin: best.origin, pairs: best.pairs, inherits: { kind: 'fork', session_id: best.origin.session_id, incarnation: best.origin.incarnation, evidence: 'inferred', at: '', through: best.pairs[best.pairs.length - 1][1].message_id } };
+}
+
+/** The session's birth time from its main files rows (the log's session record `created`), or NaN. */
+function createdOf(rows) {
+  for (const r of rows) {
+    if (r.role !== 'main' || Number(r.is_deleted)) continue;
+    const c = typeof r.contrib === 'string' ? JSON.parse(r.contrib || '{}') : (r.contrib || {});
+    const v = Number(c.meta && c.meta.created);
+    if (Number.isFinite(v) && v > 0) return v;
+  }
+  return NaN;
+}
+function metaOf(rows) {
+  const meta = { links: [], created: createdOf(rows) };
+  for (const r of rows) {
+    if (r.role !== 'main' || Number(r.is_deleted)) continue;
+    const c = typeof r.contrib === 'string' ? JSON.parse(r.contrib || '{}') : (r.contrib || {});
+    if (c.meta && Array.isArray(c.meta.links)) meta.links = c.meta.links;
+  }
+  return meta;
+}
+
+/**
+ * The 'inherited' rows these sessions need, and the forks of any of them that is an origin. Writes
+ * nothing: returns { writes: [{ ref, row }], deferred: [ref] } — the caller saves the rows as owed
+ * work before it inserts them, so no failure between the insert and the sessions rebuild loses one.
+ * `deferred`: sessions past the pass's budget, for the next pass.
+ */
+async function forkPass(ctx, items, stats, { budget = FORK_BUDGET } = {}) {
+  items = items.filter((x) => !NATIVE.has(x.ref.harness));
+  if (!items.length || typeof ctx.house.sessionMessages !== 'function') return { writes: [], deferred: [] };
+  const refs = items.map((x) => x.ref);
+  const known = new Map(items.map((x) => [skey(x.ref), x.rows]));
+  const want = new Map(refs.map((r) => [skey(r), r]));
+  // A changed session may be an origin: its forks are verified again — the ones already resolved
+  // (inheritedFrom), and the ones that could not be while it was missing (a fork captured first, a
+  // backfill split across passes): sessions whose explicit link names it, and, for the harnesses
+  // nominated by content, the sessions sharing its first assistant text (their own check decides
+  // which side was born first).
+  const msgs = new Map();
+  const messagesOf = async (ref) => { const k = skey(ref); if (!msgs.has(k)) msgs.set(k, await ctx.house.sessionMessages(ref)); return msgs.get(k); };
+  // A session nominated ONLY by origins' first text must be born after one of them: the earliest
+  // such origin's birth is its bound. Wanted for any other reason (touched, a resolved fork, a link
+  // naming the origin), it is verified with no bound.
+  const byText = new Map();
+  const unbounded = new Set(want.keys());
+  const add = (store, s, bornAfter = null) => {
+    const ref = { harness: store.harness, host: store.host, harness_path: store.harness_path, session_id: s.session_id, incarnation: s.incarnation || '' };
+    const k = skey(ref);
+    if (!want.has(k)) want.set(k, ref);
+    if (bornAfter === null) { unbounded.add(k); byText.delete(k); }
+    else if (!unbounded.has(k)) byText.set(k, byText.has(k) ? Math.min(byText.get(k), bornAfter) : bornAfter);
+  };
+  for (const r of refs) {
+    const store = { harness: r.harness, host: r.host, harness_path: r.harness_path };
+    for (const f of await ctx.house.inheritedFrom(store, r.session_id)) add(store, f);
+    for (const f of await ctx.house.linkedChildren(store, r.session_id)) add(store, f);
+    if (INFERRED.has(r.harness)) {
+      const first = (await messagesOf(r)).find((m) => m.role === 'assistant' && m.first_text);
+      const born = createdOf(known.get(skey(r)) || await ctx.house.filesRows([r]));
+      if (first && first.first_text.length >= MIN_NOMINATION_TEXT && Number.isFinite(born)) {
+        for (const f of await ctx.house.sessionsWithText(store, first.first_text)) add(store, f, born);
+      }
+    }
+  }
+  const writes = [];
+  const deferred = [];
+  let verified = 0;
+  for (const ref of want.values()) {
+    const rows = known.get(skey(ref)) || await ctx.house.filesRows([ref]);
+    if (!rows.some((r) => r.role === 'main' && !Number(r.is_deleted))) continue;
+    // Nominated as a possible fork by an origin's text: only one born after it can be.
+    if (byText.has(skey(ref)) && !(createdOf(rows) > byText.get(skey(ref)))) continue;
+    const prev = rows.find((r) => r.role === INHERITED_ROLE) || null;
+    const live = prev && !Number(prev.is_deleted) && JSON.parse(prev.contrib || '{}').inherits ? prev : null;
+    const meta = metaOf(rows);
+    const linked = (meta.links || []).some((l) => COPY_KINDS.has(l.kind));
+    if (!linked && !INFERRED.has(ref.harness) && !live) continue;
+    if (verified >= budget) { deferred.push(ref); continue; }
+    verified++;
+    const child = await messagesOf(ref);
+    const found = child.length ? await findOrigin(ctx, ref, rows, child, meta) : null;
+    let contrib = null;
+    if (found && found.unresolved) stats.forksUnresolved = (stats.forksUnresolved || 0) + 1;
+    else if (found) contrib = inheritedContrib(found.pairs, found.inherits);
+    const prevContrib = live ? JSON.parse(live.contrib || '{}') : null;
+    if (contribKey(contrib) === contribKey(prevContrib)) continue;
+    writes.push({ ref, row: inheritedRow(ref, contrib, prev, ctx.host) });
+    if (contrib) stats.forksBilledOnce = (stats.forksBilledOnce || 0) + 1;
+  }
+  if (deferred.length) stats.forksDeferred = (stats.forksDeferred || 0) + deferred.length;
+  return { writes, deferred };
+}
+// One pass is one host (capture builds every ref with refOf(…, ctx.host)); the host is in the key all the same.
+const skey = (r) => `${r.host || ''}\t${r.harness}\t${r.harness_path}\t${r.session_id}\t${r.incarnation || ''}`;
+
+module.exports = { forkPass, verifyPrefix, inheritedContrib, INHERITED_ROLE, INHERITED_FILE_KEY, INFERRED, NATIVE, COPY_KINDS, MIN_NOMINATION_TEXT, FORK_BUDGET };

package/memhouse/shipper/log-adapter.js

@@ -0,0 +1,219 @@
+'use strict';
+// The normalised session logs (editors/sessions-log.js, docs/design/sessions-log.md) as a session
+// adapter, so the profile engine (profile-capture.js) captures them as it captures the package's
+// harnesses. Its rows carry adapter 'memhouse-bridge': harness-sessions-sql §4.1's named exemption for
+// harnesses no session adapter covers yet, whose archive is memhouse's own log.
+//
+// THE ADAPTER (each line is a decision)
+//   list      one store per log directory ($MEMHOUSE_HOME/session-logs/<harness>/<harness_path>),
+//             one session per log file, one stream source per session (the file's name)
+//   read      a stream of complete lines from a byte offset: unit_key the line's offset. A file
+//             replaced (another inode or device), shorter than the offset, or whose first line or
+//             last read line changed (a rewrite that reused the inode) is a reset: writeLog rewrites a
+//             log whose history changed, so the engine re-derives and keeps what vanished as removed.
+//             A line longer than the read cap is read whole: a read always makes progress
+//   derive    the log's records as sessions-model records, the last line of a key winning:
+//     session   title, cwd, version, created, updated, archived, mode, kind, engine, client; its
+//               `links` (stated on the child) as link records
+//     message   order_key the native `order`, then the line's offset; ts the source's or null; a
+//               `tool` part as a tool_call part and, once it has an output or an error, a
+//               tool_result part; `file` as an attachment, `error` and `notice` as theirs; its
+//               `usage` as a usage record on the message (granularity per_message)
+//     usage     a usage record on the session (a store's ledger row with no message)
+//     child     a link record stated on the origin
+//     usage flags  the record's own; else from `basis` (net: false/false, gross: true/true,
+//               anthropic: false/true); else the harness's unflagged default (sessions-log.js)
+
+const crypto = require('crypto');
+const fs = require('fs');
+const path = require('path');
+
+const slog = require('../../editors/sessions-log');
+
+const ADAPTER = 'memhouse-bridge';
+const ADAPTER_VER = '1.0.1';
+const STATE_VER = 1;
+const MAX_READ_BYTES = 64 * 1024 * 1024;
+const UNFLAGGED = { 'gemini-cli': { input_includes_cache: true, output_includes_reasoning: false } };
+const BASIS = { net: [false, false], gross: [true, true], anthropic: [false, true] };
+
+const sha256 = (b) => crypto.createHash('sha256').update(b).digest('hex');
+const iso = (ms) => (Number(ms) > 0 ? new Date(Number(ms)).toISOString() : null);
+const pad = (n, w) => String(Math.max(0, Math.trunc(Number(n) || 0))).padStart(w, '0');
+const canon = (v) => (Array.isArray(v) ? `[${v.map(canon).join(',')}]` : v && typeof v === 'object'
+  ? `{${Object.keys(v).sort().map((k) => `${JSON.stringify(k)}:${canon(v[k])}`).join(',')}}` : JSON.stringify(v === undefined ? null : v));
+
+/** A record's key in its session (as adapter-capture recordKey). */
+function keyOf(r) {
+  switch (r.type) {
+    case 'session': return '';
+    case 'message': return String(r.message_id);
+    case 'usage': return String(r.usage_id);
+    case 'link': return canon([r.kind, r.stated_on === 'origin' ? r.child : r.origin, r.at ?? '']);
+    default: return '';
+  }
+}
+
+/** The flags of one log usage object (header). */
+function flagsOf(u, harness) {
+  if (typeof u.input_includes_cache === 'boolean' || typeof u.output_includes_reasoning === 'boolean') {
+    return [typeof u.input_includes_cache === 'boolean' ? u.input_includes_cache : null, typeof u.output_includes_reasoning === 'boolean' ? u.output_includes_reasoning : null];
+  }
+  if (BASIS[u.basis]) return BASIS[u.basis];
+  if (UNFLAGGED[harness]) return [UNFLAGGED[harness].input_includes_cache, UNFLAGGED[harness].output_includes_reasoning];
+  return [null, null];
+}
+const num = (v) => (v == null || v === '' || !Number.isFinite(Number(v)) ? null : Number(v));
+function usageRecord(id, anchor, u, harness, model) {
+  const [iic, oir] = flagsOf(u, harness);
+  return { type: 'usage', usage_id: id, anchor, input: num(u.input), output: num(u.output), cache_read: num(u.cache_read), cache_write: num(u.cache_write),
+    reasoning: num(u.reasoning), cost: num(u.cost), input_includes_cache: iic, output_includes_reasoning: oir, granularity: 'per_message', model: String(model || '') };
+}
+
+/** One log line (its offset `at`) as sessions-model records. */
+function recordsOf(o, at, harness) {
+  if (!o || typeof o !== 'object') return [];
+  if (o.k === 'session') {
+    const s = { type: 'session', title: String(o.title || ''), title_set_by: 'unknown', cwd: String(o.cwd || ''), harness_version: String(o.version || ''),
+      created_at: iso(o.created), updated_at: iso(o.updated), state: o.archived ? 'archived' : 'active', mode: String(o.mode || ''), kind: String(o.kind || ''),
+      engine: String(o.engine || ''), client: String(o.client || ''), reuse: '', ts_source: '', aliases: [], fingerprint: { cmp_shas: [] }, setup: null };
+    const links = (Array.isArray(o.links) ? o.links : []).map((l) => ({ type: 'link', kind: String(l.kind || ''), stated_on: 'child', child: {},
+      origin: { harness, session_id: String(l.origin || ''), ...(l.origin_store ? { harness_path: String(l.origin_store) } : {}) }, at: String(l.at || ''),
+      inherits: l.through ? { through_message_id: String(l.through) } : {}, evidence: l.evidence === 'inferred' ? 'inferred' : 'explicit' }));
+    return [s, ...links];
+  }
+  if (o.k === 'usage') {
+    if (!o.usage) return [];
+    return [usageRecord(o.usage_id != null && o.usage_id !== '' ? `usage:${o.usage_id}` : `usage:@${at}`, {}, o.usage, harness, o.model)];
+  }
+  if (o.k === 'child') {
+    return [{ type: 'link', kind: String(o.kind || ''), stated_on: 'origin', origin: {}, child: { harness, session_id: String(o.session_id || '') },
+      at: String(o.at || ''), evidence: o.evidence === 'inferred' ? 'inferred' : 'explicit' }];
+  }
+  if (o.k !== 'message' || o.message_id == null) return [];
+  const id = String(o.message_id);
+  const parts = [];
+  // A part's id: its own, else its position in the message.
+  const list = Array.isArray(o.parts) ? o.parts : [];
+  for (let k = 0; k < list.length; k++) {
+    const p = list[k];
+    const pid = String(p.part_id ?? k);
+    switch (p.kind) {
+      case 'tool':
+        parts.push({ part_id: `${pid}:call`, kind: 'tool_call', call_id: String(p.call_id || ''), name: String(p.tool || ''), args: p.args ?? {} });
+        if (p.output != null || p.is_error) {
+          parts.push({ part_id: `${pid}:result`, kind: 'tool_result', call_id: String(p.call_id || ''), output: String(p.output ?? ''), is_error: !!p.is_error,
+            ...(p.exit_code != null ? { exit_code: p.exit_code } : {}), ...(p.status ? { status: String(p.status) } : {}) });
+        }
+        break;
+      case 'reasoning': parts.push({ part_id: pid, kind: 'reasoning', text: String(p.text ?? ''), ...(p.opaque ? { opaque: true } : {}) }); break;
+      case 'summary': parts.push({ part_id: pid, kind: 'summary', text: String(p.text ?? ''), ...(p.replaces ? { replaces: p.replaces } : {}) }); break;
+      case 'file': parts.push({ part_id: pid, kind: 'attachment', media_type: String(p.media_type || ''), ...(p.ref ? { ref: String(p.ref) } : {}), ...(p.att_len != null ? { bytes_len: p.att_len } : {}) }); break;
+      case 'error': parts.push({ part_id: pid, kind: 'error', message: String(p.text ?? '') }); break;
+      case 'notice': parts.push({ part_id: pid, kind: 'notice', text: String(p.text ?? '') }); break;
+      default: parts.push({ part_id: pid, kind: 'text', text: String(p.text ?? '') });
+    }
+  }
+  // Every part id is unique within its message (a repeat takes ~n), so a log that mixes parts with and
+  // without ids, or an id that looks like a derived one (p1:call), never puts two parts on one key.
+  const seen = new Set();
+  for (const q of parts) {
+    if (seen.has(q.part_id)) { let n = 1; while (seen.has(`${q.part_id}~${n}`)) n++; q.part_id = `${q.part_id}~${n}`; }
+    seen.add(q.part_id);
+  }
+  const role = ['user', 'assistant', 'system', 'tool'].includes(o.role) ? o.role : 'assistant';
+  const out = [{ type: 'message', message_id: id, role, ts: iso(o.ts), order: `${pad(o.order != null ? o.order : at, 16)}|${pad(at, 12)}`,
+    status: String(o.status || 'complete'), state: String(o.state || 'current'), parent_id: String(o.parent_id || ''), agent_id: String(o.agent_id || ''),
+    served_model: String(o.model || ''), requested_model: String(o.model_requested || ''), effort: String(o.effort || ''), parts }];
+  if (o.usage && typeof o.usage === 'object') out.push(usageRecord(`message:${id}`, { message_id: id }, o.usage, harness, o.model));
+  return out;
+}
+
+/** The adapter for one harness's logs under `home` (the profile engine's interface: describe, list, read, derive). */
+function logAdapter(home, harness, { maxReadBytes = MAX_READ_BYTES } = {}) {
+  const dirOf = (store) => slog.logDir(home, harness, store.harness_path);
+  return {
+    describe: () => ({ adapter: ADAPTER, adapter_ver: ADAPTER_VER, state_ver: STATE_VER, model_version: 'sessions-model/1', harness }),
+    // A store folder that is not there lists nothing; any other folder error (one that cannot be read)
+    // throws, so the store pass reports it instead of taking an empty list for an empty store. A log
+    // gone between the folder read and its stat is left out; one whose stat fails otherwise is left out
+    // too and named in the list's `errors`, which the store pass reports, so it never blocks the others.
+    list(store) {
+      let names = [];
+      try { names = fs.readdirSync(dirOf(store)).filter((x) => x.endsWith('.jsonl')); } catch (e) { if (e && e.code === 'ENOENT') return []; throw e; }
+      const out = [];
+      out.errors = [];
+      for (const x of names) {
+        const p = slog.parseFileName(x);
+        if (!p || !p.session_id) continue;
+        let st;
+        try { st = fs.statSync(path.join(dirOf(store), x)); } catch (e) {
+          if (!(e && e.code === 'ENOENT')) out.errors.push({ code: e && e.code, error: `${x}: ${String(e && e.message).split('\n')[0]}` });
+          continue;
+        }
+        out.push({ session_id: p.session_id, incarnation: p.incarnation, updated_hint: [st.size, Math.trunc(st.mtimeMs), st.ino],
+          sources: [{ source_key: x, mode: 'stream', role: 'main' }] });
+      }
+      return out;
+    },
+    read(store, key, sourceKey, from) {
+      const file = path.join(dirOf(store), sourceKey);
+      let fd;
+      // Gone between list and read: an empty source this pass (the next list no longer names it).
+      try { fd = fs.openSync(file, 'r'); } catch { return { continuity: from ? 'unchanged' : 'reset', units: [], gone: [], position: from || null }; }
+      try {
+        const st = fs.fstatSync(fd);
+        const bytesAt = (at, n) => { const b = Buffer.alloc(n); return at + n <= st.size ? b.subarray(0, fs.readSync(fd, b, 0, n, at)) : null; };
+        // The bytes the last read rests on, still there: its first line, and the last line it consumed.
+        const same = (m) => !m || (() => { const b = bytesAt(Number(m.at), Number(m.len)); return !!b && b.length === Number(m.len) && sha256(b) === m.sha; })();
+        const reset = !from || from.ino !== st.ino || (from.dev != null && from.dev !== st.dev) || Number(from.offset) > st.size || !same(from.head) || !same(from.tail);
+        const start = reset ? 0 : Number(from.offset);
+        let len = Math.min(st.size - start, maxReadBytes);
+        let buf = Buffer.alloc(Math.max(len, 0));
+        if (len > 0) fs.readSync(fd, buf, 0, len, start);
+        // A single line longer than the cap is read whole.
+        while (len > 0 && start + len < st.size && buf.indexOf(0x0a) < 0) {
+          const more = Math.min(st.size - start - len, maxReadBytes);
+          const next = Buffer.alloc(len + more);
+          buf.copy(next, 0, 0, len); fs.readSync(fd, next, len, more, start + len);
+          buf = next; len += more;
+        }
+        const units = [];
+        let pos = 0; let tail = null;
+        for (;;) {
+          const nl = buf.indexOf(0x0a, pos);
+          if (nl < 0) break;   // a partial last line is not a unit: held back until whole
+          const line = buf.subarray(pos, nl);
+          if (line.length) units.push({ source_key: sourceKey, unit_key: String(start + pos), enc: 'utf8', bytes: line.length, sha256: sha256(line), redacted: false, body: line.toString('utf8') });
+          tail = { at: start + pos, len: nl + 1 - pos, sha: sha256(buf.subarray(pos, nl + 1)) };
+          pos = nl + 1;
+        }
+        let head = !reset && from.head ? from.head : null;
+        if (!head && start === 0 && pos > 0) { const nl = buf.indexOf(0x0a); head = { at: 0, len: nl + 1, sha: sha256(buf.subarray(0, nl + 1)) }; }
+        const position = { offset: start + pos, ino: st.ino, dev: st.dev, head, tail: tail || (!reset && from.tail ? from.tail : null) };
+        return { continuity: reset ? 'reset' : units.length ? 'appended' : 'unchanged', units, gone: [], position, held_back: [], more: start + len < st.size };
+      } finally { fs.closeSync(fd); }
+    },
+    derive(key, input, state) {
+      const st = state ? { lines: { ...state.lines }, emitted: { ...state.emitted } } : { lines: {}, emitted: {} };
+      for (const src of input) {
+        if (src.complete) st.lines = {};
+        for (const u of src.units) { try { st.lines[u.unit_key] = JSON.parse(u.body); } catch { /* not a record */ } }
+      }
+      const byKey = new Map();
+      for (const at of Object.keys(st.lines).sort((a, b) => Number(a) - Number(b))) {
+        for (const r of recordsOf(st.lines[at], Number(at), harness)) byKey.set(`${r.type}\u0000${keyOf(r)}`, r);
+      }
+      const changed = []; const live = {}; const emitted = {};
+      for (const [k, r] of byKey) {
+        const j = JSON.stringify(r);
+        emitted[k] = j;
+        if (st.emitted[k] !== j) changed.push(r);
+        (live[r.type] = live[r.type] || []).push(keyOf(r));
+      }
+      return { changed, live, held: {}, state: { lines: st.lines, emitted }, checkpoint: true, adapter_ver: ADAPTER_VER, model_version: 'sessions-model/1', state_ver: STATE_VER };
+    },
+  };
+}
+
+module.exports = { logAdapter, recordsOf, flagsOf, ADAPTER, ADAPTER_VER };

package/memhouse/shipper/maintenance.js

@@ -0,0 +1,49 @@
+'use strict';
+// House maintenance — what the capture engine itself never does. capture.js is insert-only by
+// rule (misc/capture-test.js): every change it makes is an insert at a higher rev. Compacting the
+// tombstones those inserts leave behind is a separate, explicit step, here.
+
+/**
+ * Tombstone cleanup. A tombstone (is_deleted = 1: a full copy of the row at rev + 1) must outlive
+ * every older version of its key, which is why merges keep it; once a merge has both in one part,
+ * ReplacingMergeTree WITH CLEANUP drops the pair. On the dev house every tombstone was a converted
+ * 0.18 row that dedupe replaced — 31% of turns. Per partition holding at least 10,000 tombstones, or
+ * 1,000 that are 5% of it (any number when forced): enable the table setting, then
+ * OPTIMIZE … PARTITION … FINAL CLEANUP.
+ * Safe because nothing writes an older version of a tombstoned key again: capture revs only rise,
+ * and the conversion refuses to run once the member has deduped sessions (convert-v3).
+ */
+async function tombstoneCleanup(house, { force = false } = {}) {
+  const out = [];
+  if (typeof house.query !== 'function' || !house.t) return out;   // DryHouse, MemoryHouse
+  for (const kind of ['turns', 'calls']) {
+    const t = house.t[kind];
+    // Per partition (turns and calls are partitioned by month; an unpartitioned table is one
+    // partition, 'all'): only the partitions that hold tombstones are rewritten. Measured on a copy
+    // of the dev house: one month 8.4 s / 167 MiB against 37 s / 682 MiB for the whole table.
+    const parts = async () => (await house.query(`SELECT _partition_id AS p, countIf(is_deleted = 1) AS dead, count() AS n FROM ${t} GROUP BY p ORDER BY p`))
+      .map((x) => ({ p: String(x.p), dead: Number(x.dead) || 0, n: Number(x.n) || 0 }));
+    const before = await parts();
+    const dead = before.reduce((a, x) => a + x.dead, 0); const n = before.reduce((a, x) => a + x.n, 0);
+    const r = { kind, tombstones: dead, rows: n, cleaned: false, partitions: [] };
+    // Each partition decides on its own: at least 10,000 tombstones, or 1,000 that are 5% of it.
+    const due = (x) => x.dead > 0 && (force || x.dead >= 10000 || (x.dead >= 1000 && x.dead / Math.max(1, x.n) >= 0.05));
+    if (before.some(due)) {
+      const t0 = Date.now();
+      try {
+        await house.query(`ALTER TABLE ${t} MODIFY SETTING allow_experimental_replacing_merge_with_cleanup = 1`);
+        for (const x of before) {
+          if (!due(x)) continue;
+          await house.query(`OPTIMIZE TABLE ${t} PARTITION ID '${x.p.replace(/[^0-9A-Za-z_-]/g, '')}' FINAL CLEANUP`);
+          r.partitions.push(x.p);
+        }
+        const after = await parts();
+        Object.assign(r, { cleaned: r.partitions.length > 0, after_tombstones: after.reduce((a, x) => a + x.dead, 0), after_rows: after.reduce((a, x) => a + x.n, 0), ms: Date.now() - t0 });
+      } catch (e) { r.error = e.message.split('\n')[0]; }
+    }
+    out.push(r);
+  }
+  return out;
+}
+
+module.exports = { tombstoneCleanup };