memhouse 0.18.12 → 0.20.1

package/memhouse/house/reconcile-v3.js

@@ -0,0 +1,192 @@
+// Reconciliation — does a session's new form hold everything its old form held?
+// (docs/design/source-identity.md draft 5 §7 steps 4–5.)
+//
+// Two questions, asked per session with the same instruments:
+//
+//   reconcileConversion   legacy current parse (<db>_archive.<m>_*_pre_v3)  vs  the converted
+//                         `legacy:` rows in turns. Exact: same rows, same role/agent/ts, same
+//                         text, same tokens (legacy input as the conversion maps it,
+//                         convert-v3 legacyInput). Run after convert-v3, before anything is
+//                         tombstoned.
+//
+//   reconcileCapture      a session's converted `legacy:ship:` rows  vs  the rows the capture
+//                         layer derived from disk (every message_id not starting 'legacy:').
+//                         Not exact — the two parsers cut a transcript differently (a legacy row
+//                         is a whole record or Codex turn, a new row one block) and legacy Claude
+//                         tokens are per record (about double) — so it measures instead:
+//                           counts   kind='text' rows per (role, agent_id), user and assistant;
+//                                    the capture side must hold at least as many
+//                           text     every line of every legacy user/assistant text row (subagent tag and
+//                                    marker lines '[thinking] ', '[tool-call: ', '[tool-result: '
+//                                    dropped) must occur as a line of some captured row
+//                           tokens   both sums, reported, never a pass/fail (per_record vs
+//                                    per_response)
+//                         The capture stream calls this before tombstoning a session's legacy
+//                         ship rows; a session that does not reconcile stays legacy (§7 step 5).
+//
+// Reads use FINAL and drop tombstones (is_deleted = 0). Nothing here writes. Transport-free.
+
+const { kindSQL, harnessSQL, sessionSQL, currentParseSQL, legacyInputSQL, lit } = require('./convert-v3');
+
+const SETTINGS = { final: 0, join_use_nulls: 0, output_format_json_quote_64bit_integers: 0 };
+
+function sessionFilter(h, s, session) {
+  return session ? ` AND ${h} = ${lit(session.harness)} AND ${s} = ${lit(session.session_id)}` : '';
+}
+
+/** Legacy current-parse rows, v3-named: h, s, rl, aid, mid, t, tokens, th (text hash). */
+function legacyRowsSQL(N, session) {
+  const L = N.legacy;
+  return `SELECT * FROM (
+    SELECT ${harnessSQL('m')} AS h, ${sessionSQL('m')} AS s, toString(m.role) AS rl,
+           JSONExtractString(toJSONString(m.extra), 'agent', 'id') AS aid,
+           concat('legacy:', toString(m.origin), ':', toString(m.epoch), ':', toString(m.seq)) AS mid,
+           m.ts AS t, ${kindSQL('m.role', 'm.text', 'h')} AS kd,
+           ${legacyInputSQL('m', 'h')} AS ti, m.output_tokens AS tout, m.cache_read_tokens AS tcr, m.cache_write_tokens AS tcw,
+           cityHash64(m.text) AS th
+    FROM ${L.messages} AS m FINAL
+    WHERE ${currentParseSQL('m', L.messages, N.member)})
+    WHERE 1${sessionFilter('h', 's', session)}`;
+}
+
+function convertedRowsSQL(N, session, prefix = 'legacy:') {
+  return `SELECT toString(harness) AS h, session_id AS s, toString(role) AS rl, agent_id AS aid, message_id AS mid,
+           ts AS t, toString(kind) AS kd,
+           input_tokens AS ti, output_tokens AS tout, cache_read_tokens AS tcr, cache_write_tokens AS tcw,
+           cityHash64(text) AS th
+    FROM ${N.turns} FINAL
+    WHERE is_deleted = 0 AND startsWith(message_id, ${lit(prefix)})${sessionFilter('harness', 'session_id', session)}`;
+}
+
+/**
+ * Legacy vs converted, per session. `session` ({ harness, session_id }) narrows to one.
+ * @returns {{ sessions, ok_sessions, ok, mismatched: [...], all: [...] }}
+ */
+async function reconcileConversion(q, N, { session = null } = {}) {
+  const Lg = legacyRowsSQL(N, session);
+  const Cv = convertedRowsSQL(N, session);
+  // Row by row on the full old key: missing, extra, text and field differences.
+  const rows = await q.rows(`
+    SELECT if(l.p = 1, l.h, c.h) AS kh, if(l.p = 1, l.s, c.s) AS ks,
+           countIf(l.p = 1) AS legacy_rows, countIf(c.p = 1) AS converted_rows,
+           countIf(l.p = 1 AND c.p = 0) AS missing, countIf(c.p = 1 AND l.p = 0) AS extra,
+           countIf(l.p = 1 AND c.p = 1 AND l.th != c.th) AS text_diff,
+           countIf(l.p = 1 AND c.p = 1 AND (l.rl != c.rl OR l.aid != c.aid OR l.t != c.t OR l.kd != c.kd)) AS field_diff,
+           sumIf(l.ti, l.p = 1) AS l_in, sumIf(c.ti, c.p = 1) AS c_in,
+           sumIf(l.tout, l.p = 1) AS l_out, sumIf(c.tout, c.p = 1) AS c_out,
+           sumIf(l.tcr, l.p = 1) AS l_cr, sumIf(c.tcr, c.p = 1) AS c_cr,
+           sumIf(l.tcw, l.p = 1) AS l_cw, sumIf(c.tcw, c.p = 1) AS c_cw
+    FROM (SELECT *, 1 AS p FROM (${Lg})) AS l
+    FULL OUTER JOIN (SELECT *, 1 AS p FROM (${Cv})) AS c ON l.h = c.h AND l.s = c.s AND l.mid = c.mid
+    GROUP BY kh, ks ORDER BY kh, ks`, SETTINGS);
+  // Counts per (role, agent): the shape the design names (§7 step 5). Distinct ids, so a
+  // legacy collision (two user_ids, one key — reported by convert) is not a mismatch here.
+  const groups = await q.rows(`
+    SELECT if(l.p = 1, l.h, c.h) AS kh, if(l.p = 1, l.s, c.s) AS ks, countIf(l.n != c.n) AS group_diff
+    FROM (SELECT h, s, rl, aid, uniqExact(mid) AS n, 1 AS p FROM (${Lg}) GROUP BY h, s, rl, aid) AS l
+    FULL OUTER JOIN (SELECT h, s, rl, aid, uniqExact(mid) AS n, 1 AS p FROM (${Cv}) GROUP BY h, s, rl, aid) AS c
+      ON l.h = c.h AND l.s = c.s AND l.rl = c.rl AND l.aid = c.aid
+    GROUP BY kh, ks`, SETTINGS);
+  const g = new Map(groups.map((x) => [`${x.kh}\u0000${x.ks}`, Number(x.group_diff) || 0]));
+  const all = rows.map((x) => {
+    const r = {
+      harness: x.kh, session_id: x.ks,
+      legacy_rows: Number(x.legacy_rows), converted_rows: Number(x.converted_rows),
+      missing: Number(x.missing), extra: Number(x.extra),
+      text_diff: Number(x.text_diff), field_diff: Number(x.field_diff),
+      group_diff: g.get(`${x.kh}\u0000${x.ks}`) || 0,
+      tokens: {
+        legacy: { input: Number(x.l_in), output: Number(x.l_out), cache_read: Number(x.l_cr), cache_write: Number(x.l_cw) },
+        converted: { input: Number(x.c_in), output: Number(x.c_out), cache_read: Number(x.c_cr), cache_write: Number(x.c_cw) },
+      },
+    };
+    r.token_diff = ['input', 'output', 'cache_read', 'cache_write'].some((k) => r.tokens.legacy[k] !== r.tokens.converted[k]);
+    r.ok = !r.missing && !r.extra && !r.text_diff && !r.field_diff && !r.group_diff && !r.token_diff;
+    return r;
+  });
+  const mismatched = all.filter((r) => !r.ok);
+  return { sessions: all.length, ok_sessions: all.length - mismatched.length, ok: mismatched.length === 0, mismatched, all };
+}
+
+// Lines of a text, normalised for containment: the subagent tag dropped, each line trimmed,
+// empty lines and legacy marker lines skipped. The same expression on both sides.
+const LINES = (text) => `arrayFilter(x -> x != '', arrayMap(x -> trimBoth(x), splitByChar('\\n',
+    if(startsWith(${text}, '[subagent] '), substring(${text}, 12), ${text}))))`;
+const NOT_MARKER = (x) => `NOT (startsWith(${x}, '[thinking] ') OR startsWith(${x}, '[tool-call: ') OR startsWith(${x}, '[tool-result: '))`;
+
+/**
+ * One session: its converted `legacy:ship:` rows vs the rows the capture layer wrote.
+ * `minContainment` is the share of legacy lines that must be found (default 1 — every line).
+ *
+ * @returns {{ harness, session_id, legacy_rows, captured_rows, groups: [...], lines: {legacy, found,
+ *             containment, missing_message_ids}, tokens: {legacy, captured}, ok, reasons: [] }}
+ */
+async function reconcileCapture(q, N, { harness, session_id }, { minContainment = 1 } = {}) {
+  const sess = { harness, session_id };
+  const where = `harness = ${lit(harness)} AND session_id = ${lit(session_id)} AND is_deleted = 0`;
+  const legacy = `startsWith(message_id, 'legacy:ship:')`;
+  const captured = `NOT startsWith(message_id, 'legacy:')`;
+
+  const groups = (await q.rows(`
+    SELECT if(l.p = 1, l.rl, c.rl) AS role, if(l.p = 1, l.aid, c.aid) AS agent_id,
+           l.n AS legacy, c.n AS captured
+    FROM (SELECT toString(role) AS rl, agent_id AS aid, count() AS n, 1 AS p FROM ${N.turns} FINAL
+          WHERE ${where} AND ${legacy} AND kind = 'text' AND role IN ('user', 'assistant') GROUP BY rl, aid) AS l
+    FULL OUTER JOIN
+         (SELECT toString(role) AS rl, agent_id AS aid, count() AS n, 1 AS p FROM ${N.turns} FINAL
+          WHERE ${where} AND ${captured} AND kind = 'text' AND role IN ('user', 'assistant') GROUP BY rl, aid) AS c
+      ON l.rl = c.rl AND l.aid = c.aid
+    ORDER BY role, agent_id`, SETTINGS)).map((x) => ({ role: x.role, agent_id: x.agent_id, legacy: Number(x.legacy), captured: Number(x.captured) }));
+
+  const capturedLines = `SELECT DISTINCT cityHash64(line) AS lh, 1 AS hit
+    FROM (SELECT arrayJoin(${LINES('text')}) AS line FROM ${N.turns} FINAL WHERE ${where} AND ${captured})`;
+  const legacyLines = `SELECT message_id AS mid, cityHash64(line) AS lh
+    FROM (SELECT message_id, arrayJoin(${LINES('text')}) AS line FROM ${N.turns} FINAL
+          WHERE ${where} AND ${legacy} AND kind = 'text' AND role IN ('user', 'assistant'))
+    WHERE ${NOT_MARKER('line')}`;
+  const lines = (await q.rows(`
+    SELECT uniqExact(l.lh) AS legacy_lines, uniqExactIf(l.lh, c.hit = 1) AS found,
+           arraySlice(arraySort(groupUniqArrayIf(l.mid, c.hit = 0)), 1, 10) AS missing_message_ids
+    FROM (${legacyLines}) AS l LEFT JOIN (${capturedLines}) AS c ON l.lh = c.lh`, SETTINGS))[0] || {};
+
+  const tok = (await q.rows(`
+    SELECT count() AS n, sumIf(1, ${legacy}) AS l_rows, sumIf(1, ${captured}) AS c_rows,
+           sumIf(input_tokens, ${legacy}) AS l_in, sumIf(input_tokens, ${captured}) AS c_in,
+           sumIf(output_tokens, ${legacy}) AS l_out, sumIf(output_tokens, ${captured}) AS c_out,
+           sumIf(cache_read_tokens, ${legacy}) AS l_cr, sumIf(cache_read_tokens, ${captured}) AS c_cr,
+           sumIf(cache_write_tokens, ${legacy}) AS l_cw, sumIf(cache_write_tokens, ${captured}) AS c_cw
+    FROM ${N.turns} FINAL WHERE ${where}`, SETTINGS))[0] || {};
+
+  const legacyLinesN = Number(lines.legacy_lines) || 0;
+  const found = Number(lines.found) || 0;
+  const containment = legacyLinesN ? found / legacyLinesN : 1;
+  const reasons = [];
+  if (!Number(tok.l_rows)) reasons.push('no legacy ship rows for this session');
+  if (!Number(tok.c_rows)) reasons.push('no captured rows for this session');
+  for (const x of groups) {
+    if (x.captured < x.legacy) reasons.push(`${x.role}${x.agent_id ? `/${x.agent_id}` : ''}: ${x.captured} captured text rows < ${x.legacy} legacy`);
+  }
+  if (containment < minContainment) reasons.push(`text: ${found}/${legacyLinesN} legacy lines found (${(containment * 100).toFixed(2)}%)`);
+  return {
+    harness, session_id: sess.session_id,
+    legacy_rows: Number(tok.l_rows) || 0, captured_rows: Number(tok.c_rows) || 0,
+    groups,
+    lines: { legacy: legacyLinesN, found, containment, missing_message_ids: lines.missing_message_ids || [] },
+    tokens: {
+      legacy: { input: Number(tok.l_in), output: Number(tok.l_out), cache_read: Number(tok.l_cr), cache_write: Number(tok.l_cw) },
+      captured: { input: Number(tok.c_in), output: Number(tok.c_out), cache_read: Number(tok.c_cr), cache_write: Number(tok.c_cw) },
+    },
+    ok: reasons.length === 0, reasons,
+  };
+}
+
+/** draft 5 §2.6: any (harness, session_id, message_id) — or call_id — with more than one ts. */
+async function duplicateTs(q, N) {
+  const one = async (t, id) => Number((await q.rows(`
+    SELECT count() AS n FROM (SELECT harness, session_id, ${id} AS i FROM ${t} FINAL WHERE is_deleted = 0
+      GROUP BY harness, host, harness_path, session_id, incarnation, i HAVING uniqExact(ts) > 1)`, SETTINGS))[0]?.n || 0);
+  // Per message part (a part may carry its own ts) and per call, in the session's store.
+  return { turns: await one(N.turns, "concat(message_id, '#', part_id)"), calls: await one(N.calls, 'call_id') };
+}
+
+module.exports = { reconcileConversion, reconcileCapture, duplicateTs, legacyRowsSQL, convertedRowsSQL };

package/memhouse/house/records-to-profile.js

@@ -0,0 +1,174 @@
+'use strict';
+// sessions-model records as rows of harness-sessions-sql's profile tables (step B2, the writer). Nearly 1:1: a message's parts are its session_parts rows, a usage record a
+// session_usage row (numbers and flags as given, null for unknown), a link a session_links row
+// (owner 'adapter'), a lifecycle record a session_lifecycle row, the session record the spec
+// columns of its sessions row. Pure: records in, rows out. The engine assigns rev, adapter,
+// adapter_ver, model_version and the derived columns (folder, project, totals).
+//
+// content_sha (harness-sessions-sql §4): a record's own sha (sessions-model §2: the JCS of the
+// record without content_sha and cmp_sha) on every row; on a part row, the sha of the part object
+// as it sits in its message's parts, with message_content_sha the message's. A message's header
+// row (part_id '', kind 'none') carries the message's sha in both.
+
+const crypto = require('crypto');
+
+/** RFC 8785 for JSON values: object keys sorted, ECMAScript serialisation of scalars. */
+function jcs(v) {
+  if (Array.isArray(v)) return `[${v.map(jcs).join(',')}]`;
+  if (v && typeof v === 'object') return `{${Object.keys(v).filter((k) => v[k] !== undefined).sort().map((k) => `${JSON.stringify(k)}:${jcs(v[k])}`).join(',')}}`;
+  return JSON.stringify(v === undefined ? null : v);
+}
+const sha = (v) => crypto.createHash('sha256').update(jcs(v)).digest('hex');
+/** A record's content_sha: the one it carries (sealed by the package), else computed the same way. */
+function recordSha(r) {
+  if (/^[0-9a-f]{64}$/.test(String(r.content_sha || ''))) return r.content_sha;
+  const { content_sha: _c, cmp_sha: _m, ...rest } = r;
+  return sha(rest);
+}
+
+const str = (v) => (v == null ? '' : String(v));
+const nb = (v) => (typeof v === 'boolean' ? v : null);
+const nn = (v) => (Number.isFinite(Number(v)) && v !== null && v !== '' ? Number(v) : null);
+const tsOrNull = (v) => (v && Number.isFinite(Date.parse(v)) ? v : null);
+
+/** The part columns of one part. */
+function partColumns(p) {
+  const ref = (x) => (x && typeof x === 'object' ? x : (x ? { ref: String(x) } : null));
+  const out = {
+    part_id: str(p.part_id), kind: str(p.kind), text: '',
+    call_id: '', tool_name: '', args: '', server: '', is_error: null, result_status: '', output: '', output_ref: '', output_ref_sha256: '', output_ref_bytes: null,
+    output_len: null, truncated: null, exit_code: null, media_type: '', inline_b64: '', ref: '', bytes_len: null, opaque: null, signature_len: null,
+    replaces_form: '', replaces_value: '', error_kind: '', error_message: '', http_status: null, level: '', native_type: '',
+    part_source: str(p.source), part_state: str(p.state),
+  };
+  switch (p.kind) {
+    case 'text': case 'notice': case 'summary': case 'reasoning':
+      out.text = str(p.text);
+      if (p.kind === 'notice') { out.level = str(p.level); out.native_type = str(p.native_type); }
+      if (p.kind === 'reasoning') { out.opaque = nb(p.opaque); out.signature_len = nn(p.signature_len); }
+      if (p.kind === 'summary' && p.replaces && typeof p.replaces === 'object') {
+        const [form] = Object.keys(p.replaces);
+        if (form) { out.replaces_form = form; out.replaces_value = typeof p.replaces[form] === 'string' ? p.replaces[form] : JSON.stringify(p.replaces[form]); }
+      }
+      break;
+    case 'tool_call':
+      out.call_id = str(p.call_id); out.tool_name = str(p.name); out.args = p.args == null ? '' : JSON.stringify(p.args); out.server = str(p.server);
+      break;
+    case 'tool_result': {
+      out.call_id = str(p.call_id); out.is_error = nb(p.is_error); out.result_status = str(p.status); out.output = str(p.output);
+      out.output_len = nn(p.output_len); out.truncated = nb(p.truncated); out.exit_code = nn(p.exit_code);
+      const r = ref(p.output_ref);
+      if (r) { out.output_ref = str(r.ref); out.output_ref_sha256 = str(r.sha256); out.output_ref_bytes = nn(r.bytes); }
+      break;
+    }
+    case 'attachment': {
+      out.media_type = str(p.media_type); out.inline_b64 = str(p.inline); out.bytes_len = nn(p.bytes_len);
+      const r = ref(p.ref);
+      if (r) out.ref = str(r.ref);
+      break;
+    }
+    case 'error':
+      out.error_kind = str(p.error_kind); out.error_message = str(p.message); out.http_status = nn(p.http_status);
+      break;
+    default: break;
+  }
+  return out;
+}
+
+/**
+ * Profile rows for one session's record view.
+ * @param {object[]} records  every live record of the session
+ * @param {object} key        { host, harness, harness_path, session_id, incarnation }
+ * @param {object} [o]        { firstSeen(messageId) → ISO }: the engine's first-seen time
+ * @returns {{ parts, usage, links, lifecycle, session, calls }}
+ */
+function profileRows(records, key, { firstSeen = () => '1970-01-01T00:00:00.000Z' } = {}) {
+  const K = { host: key.host, harness: key.harness, harness_path: key.harness_path, session_id: key.session_id, incarnation: key.incarnation || '' };
+  const messages = records.filter((r) => r.type === 'message');
+  const parts = []; const calls = new Map();
+  const tsOfMessage = new Map(); const byResponse = new Map();
+  for (const m of messages) {
+    const msgSha = recordSha(m);
+    const ts = tsOrNull(m.ts);
+    tsOfMessage.set(String(m.message_id), ts);
+    // A response's time is its earliest message's that has one: a first message without a time does
+    // not leave the response's usage untimed.
+    if (m.response_id && ts && (!byResponse.get(m.response_id) || Date.parse(ts) < Date.parse(byResponse.get(m.response_id)))) byResponse.set(m.response_id, ts);
+    const message = {
+      ...K, message_id: str(m.message_id), parent_id: str(m.parent_id), order_key: str(m.order), agent_id: str(m.agent_id), role: str(m.role),
+      source: str(m.source), to_user: nb(m.to_user), to_model: nb(m.to_model), status: str(m.status), state: str(m.state), on_active_branch: nb(m.on_active_branch),
+      ts, requested_model: str(m.requested_model), served_model: str(m.served_model), effort: str(m.effort), response_id: str(m.response_id),
+      message_aliases: Array.isArray(m.aliases) ? m.aliases.map((a) => ({ id: str(a.id), kind: str(a.kind) })) : [],
+      cmp_sha: str(m.cmp_sha), message_content_sha: msgSha, first_seen: firstSeen(str(m.message_id)),
+    };
+    const ps = Array.isArray(m.parts) ? m.parts : [];
+    ps.forEach((p, i) => {
+      const row = { ...message, part_index: i, ...partColumns(p), content_sha: sha(p) };
+      parts.push(row);
+      // A call a revert hides (its part's state, else its message's) counts nowhere, as its parts do not:
+      // it is left out of {m}_calls, and a call written before the revert is retired with them.
+      const gone = ['reverted', 'retracted'].includes(String(p.state || m.state || ''));
+      if (p.kind === 'tool_call' && p.call_id && !gone) {
+        calls.set(p.call_id, { result_ts: null, is_error: null, output_len: null, ...(calls.get(p.call_id) || {}), ...K, call_id: str(p.call_id), message_id: message.message_id, agent_id: message.agent_id, ts, first_seen: message.first_seen,
+          tool_name: str(p.name), args: p.args == null ? '' : JSON.stringify(p.args), model: message.served_model || message.requested_model, content_sha: row.content_sha });
+      }
+      if (p.kind === 'tool_result' && p.call_id && !gone) {
+        calls.set(p.call_id, { ...(calls.get(p.call_id) || {}), result_ts: ts, is_error: nb(p.is_error), output_len: nn(p.output_len) ?? (p.output == null ? null : String(p.output).length) });
+      }
+    });
+    if (!ps.length) parts.push({ ...message, part_index: 0, ...partColumns({ part_id: '', kind: 'none' }), content_sha: msgSha });
+  }
+  const anchorTs = (a) => (a.response_id != null ? byResponse.get(a.response_id) : (a.message_id != null ? tsOfMessage.get(String(a.message_id)) : null)) ?? null;
+  const usage = records.filter((r) => r.type === 'usage').map((u) => {
+    const a = u.anchor || {};
+    const kind = a.response_id != null ? 'response' : a.message_id != null ? 'message' : a.compaction != null ? 'compaction' : 'session';
+    const id = kind === 'response' ? a.response_id : kind === 'message' ? a.message_id : kind === 'compaction' ? a.compaction : '';
+    return {
+      ...K, usage_id: str(u.usage_id), anchor_kind: kind, anchor_id: str(id),
+      input: nn(u.input), output: nn(u.output), cache_read: nn(u.cache_read), cache_write: nn(u.cache_write), reasoning: nn(u.reasoning),
+      tool_prompt: nn(u.tool_prompt), cost: nn(u.cost), input_includes_cache: nb(u.input_includes_cache), output_includes_reasoning: nb(u.output_includes_reasoning),
+      granularity: str(u.granularity), model: str(u.model), content_sha: recordSha(u), ts: anchorTs(a),
+    };
+  });
+  const refCols = (prefix, r) => ({ [`${prefix}_host`]: str(r && r.host), [`${prefix}_harness_path`]: str(r && r.harness_path), [`${prefix}_harness`]: str(r && r.harness),
+    [`${prefix}_session_id`]: str(r && r.session_id), [`${prefix}_incarnation`]: r && r.incarnation != null ? String(r.incarnation) : null });
+  const links = records.filter((r) => r.type === 'link').map((l) => {
+    const other = l.stated_on === 'origin' ? l.child : l.origin;
+    const inh = l.inherits || {};
+    return {
+      ...K, owner: 'adapter', kind: str(l.kind), other_host: str(other && other.host), other_harness_path: str(other && other.harness_path), other_harness: str(other && other.harness),
+      other_session_id: str(other && other.session_id), other_incarnation: str(other && other.incarnation), at: str(l.at), stated_on: str(l.stated_on),
+      ...refCols('child', l.child), ...refCols('origin', l.origin),
+      through_message_id: str(inh.through_message_id), ids_preserved: nb(inh.ids_preserved), marked: nb(inh.marked), usage_copied: nb(inh.usage_copied),
+      child_stored: typeof l.child_stored === 'boolean' ? l.child_stored : null, evidence: str(l.evidence) || 'explicit', verified: null, statements: [], content_sha: recordSha(l),
+    };
+  });
+  const lifecycle = records.filter((r) => r.type === 'lifecycle').map((x) => {
+    const t = x.target || {};
+    const kind = t.session ? 'session' : t.range ? 'range' : 'message';
+    return {
+      ...K, lifecycle_id: str(x.lifecycle_id), op: str(x.op), target_kind: kind, target_id: str(t.message_id),
+      target_from: str(t.range && t.range.from_id), target_through: str(t.range && t.range.through_id), at: str(x.at), ts: tsOrNull(x.ts),
+      evidence: str(x.evidence) || 'explicit', content_sha: recordSha(x),
+    };
+  });
+  const s = records.find((r) => r.type === 'session') || null;
+  const session = s ? {
+    ...K, engine: str(s.engine), client: str(s.client), harness_version: str(s.harness_version), title: str(s.title), title_set_by: str(s.title_set_by) || 'unknown',
+    cwd: str(s.cwd), last_cwd: str(s.last_cwd), created_at: tsOrNull(s.created_at), updated_at: tsOrNull(s.updated_at), state: str(s.state), pinned: nb(s.pinned),
+    mode: str(s.mode), kind: str(s.kind), reuse: str(s.reuse), ts_source: str(s.ts_source),
+    aliases: Array.isArray(s.aliases) ? s.aliases.map((a) => ({ id: str(a.id), kind: str(a.kind) })) : [],
+    fingerprint: s.fingerprint && Array.isArray(s.fingerprint.cmp_shas) ? s.fingerprint.cmp_shas.map(String) : [],
+    setup: s.setup == null ? null : jcs(s.setup), content_sha: recordSha(s),
+  } : null;
+  return { parts, usage, links, lifecycle, session, calls: [...calls.values()].filter((c) => c.call_id && c.tool_name !== undefined) };
+}
+
+/**
+ * The version of this mapping. A change to what rows the same records make (2: a reverted call counts
+ * nowhere; a response takes its earliest time) moves it, and every captured session is mapped again
+ * once from the view its local state keeps: no source read or derive, only the rows that differ written.
+ */
+const ROWS_VER = 2;
+
+module.exports = { profileRows, partColumns, jcs, recordSha, ROWS_VER };

package/memhouse/house/records-to-rows.js

@@ -0,0 +1,169 @@
+'use strict';
+// sessions-model records — what the session adapters' derive emits — as schema-3 rows. Step 2 of
+// the switch to @agent-realm/harness-session-adapters (decided 2026-10-23: the package derives the
+// records, memhouse owns the mapping from records to its rows). The engine keeps each session's
+// record view (the package's applyDerive) and calls rowsOf on it; it assigns rev, adapter_ver,
+// folder/project, host, origin and ingested_at. Pure: records in, rows out.
+//
+// THE MAPPING (each line is a decision)
+//
+//   message           one turns row per part: message_id, the part's own part_id, part_ord = its
+//                     position in parts. A message with no parts keeps a header row (part_id '',
+//                     kind 'none'), which carries its status and its usage.
+//     text            text
+//     reasoning       thinking; an opaque one has empty text and extra.opaque
+//     summary         summary
+//     notice, error   event: extra.notice is the part kind, plus the error's fields
+//     attachment      attachment: `<media_type> <sha256 | ref>`, extra.media_type/att_sha/att_len
+//     tool_call       tool_call: `<name> <args JSON>`, and a calls row (call_id, tool_name, args)
+//     tool_result     tool_result, role user (as 0.20 stores results); its call's row gets
+//                     result_ts, is_error and result_len
+//   role              user | assistant | system; a tool message's rows are user rows
+//   model             served_model, else requested_model
+//   state             '' for current, else the record's (retracted, superseded, …)
+//   extra             status when not complete; on_active_branch when false; source when it is
+//                     neither user nor model; the record's order
+//   ts                the message's ts. One without takes tsOf(message_id), the engine's first-seen
+//                     time, so its key (toDate(ts) and ts are in the ORDER BY) never moves.
+//   ord               the message's rank among the session's messages by `order`
+//   usage             tokens on its anchor's first row: response_id, the first message (by order)
+//                     with that response; message_id, that message. A session or compaction
+//                     anchor, or an anchor no message has, is a row of its own (message_id
+//                     'usage:<usage_id>', kind 'usage'; with '~' added while a message of the
+//                     session has that id). Several usage records on one row add up.
+//                     houseTokens applies input_includes_cache / output_includes_reasoning.
+//   session           the sessions row's metadata: title, cwd as folder, created_at, updated_at,
+//                     harness_version, mode
+//   link              stated by the child: the sessions row's links (kind, origin, origin_store,
+//                     evidence, at, through); stated by the origin (a sub-agent it started):
+//                     `children`, for the child's side
+//   lifecycle         no row: its effect is the message's state, as the 0.20 log derive did.
+//                     Step B gives it harness-sessions-sql's session_lifecycle.
+
+const ROLES = new Set(['user', 'assistant', 'system']);
+
+function n(v) { const x = Math.trunc(Number(v)); return Number.isFinite(x) && x > 0 ? x : 0; }
+
+/** A usage record's tokens in the house split (sessions-model §2.4): input without cached input,
+ * output with reasoning. A flag that is not a boolean stores the number as given. `tool_prompt`
+ * (v1: prompt tokens of tool use that `input` leaves out, Gemini's toolUsePromptTokenCount) is input. */
+function houseTokens(u) {
+  const input = n(u.input); const cr = n(u.cache_read); const cw = n(u.cache_write); const rs = n(u.reasoning);
+  return {
+    input_tokens: (u.input_includes_cache === true ? Math.max(input - cr - cw, 0) : input) + n(u.tool_prompt),
+    output_tokens: n(u.output) + (u.output_includes_reasoning === false ? rs : 0),
+    cache_read_tokens: cr, cache_write_tokens: cw, reasoning_tokens: rs,
+  };
+}
+const TOKENS = ['input_tokens', 'output_tokens', 'cache_read_tokens', 'cache_write_tokens', 'reasoning_tokens'];
+const anyTokens = (t) => TOKENS.some((k) => t[k] > 0);
+
+function partRow(p) {
+  switch (p.kind) {
+    case 'text': return { kind: 'text', text: String(p.text ?? '') };
+    case 'reasoning': return p.opaque ? { kind: 'thinking', text: '', extra: { opaque: true } } : { kind: 'thinking', text: String(p.text ?? '') };
+    case 'summary': return { kind: 'summary', text: String(p.text ?? '') };
+    case 'notice': return { kind: 'event', text: String(p.text ?? ''), extra: { notice: 'notice', ...(p.level ? { level: p.level } : {}), ...(p.native_type ? { native_type: p.native_type } : {}) } };
+    case 'error': return { kind: 'event', text: String(p.message ?? ''), extra: { notice: 'error', error_kind: p.error_kind ?? null, ...(p.http_status != null ? { http_status: p.http_status } : {}) } };
+    case 'attachment': {
+      const ref = p.ref && typeof p.ref === 'object' ? p.ref : null;
+      const sha = (ref && ref.sha256) || p.sha256 || '';
+      return { kind: 'attachment', text: `${p.media_type || 'file'} ${sha || (ref && ref.ref) || ''}`.trim(),
+        extra: { media_type: p.media_type ?? null, att_sha: sha || null, att_len: p.bytes_len ?? (ref && ref.bytes) ?? null } };
+    }
+    case 'tool_call': return { kind: 'tool_call', text: `${p.name ?? ''} ${JSON.stringify(p.args ?? {})}` };
+    case 'tool_result': return { kind: 'tool_result', text: String(p.output ?? ''), role: 'user' };
+    default: return { kind: 'text', text: String(p.text ?? ''), extra: { part_kind: String(p.kind) } };
+  }
+}
+
+/**
+ * Rows for one session's record view.
+ * @param {object[]} records  every live record of the session (any order)
+ * @param {object} [o]
+ * @param {(messageId: string) => string} [o.tsOf]  a first-seen ts (ISO) for a message without one
+ * @returns {{ turns: object[], calls: object[], meta: object|null, links: object[], children: object[] }}
+ */
+function rowsOf(records, { tsOf = () => '1970-01-01T00:00:00.000Z' } = {}) {
+  const messages = records.filter((r) => r.type === 'message')
+    .sort((a, b) => (String(a.order) < String(b.order) ? -1 : String(a.order) > String(b.order) ? 1 : (a.message_id < b.message_id ? -1 : 1)));
+  const turns = [];
+  const calls = new Map();
+  const firstRow = new Map();            // message_id -> its first turns row
+  const byResponse = new Map();          // response_id -> the first message with it
+  messages.forEach((m, rank) => {
+    if (m.response_id && !byResponse.has(m.response_id)) byResponse.set(m.response_id, m.message_id);
+    const ts = m.ts || tsOf(m.message_id);
+    const role = ROLES.has(m.role) ? m.role : 'user';
+    const extra = {};
+    if (m.status && m.status !== 'complete') extra.status = m.status;
+    if (m.on_active_branch === false) extra.on_active_branch = false;
+    if (m.source && m.source !== 'user' && m.source !== 'model') extra.source = m.source;
+    if (m.order != null) extra.order = String(m.order);
+    const base = {
+      session_id: m.session_id, incarnation: m.incarnation ?? '', message_id: String(m.message_id),
+      parent_id: m.parent_id ?? '', response_id: m.response_id ?? '', agent_id: m.agent_id ?? '',
+      ts, ord: rank, state: m.state && m.state !== 'current' ? String(m.state) : '',
+      model: m.served_model || m.requested_model || '', effort: m.effort || '',
+    };
+    const parts = Array.isArray(m.parts) ? m.parts : [];
+    parts.forEach((p, i) => {
+      const r = partRow(p);
+      const row = { ...base, part_id: String(p.part_id ?? i), part_ord: i, role: r.role || role, kind: r.kind, text: r.text, extra: { ...extra, ...(r.extra || {}) } };
+      turns.push(row);
+      if (!firstRow.has(base.message_id)) firstRow.set(base.message_id, row);
+      if (p.kind === 'tool_call' && p.call_id) {
+        calls.set(p.call_id, { ...(calls.get(p.call_id) || {}), session_id: base.session_id, incarnation: base.incarnation, call_id: String(p.call_id),
+          message_id: base.message_id, agent_id: base.agent_id, ts, tool_name: String(p.name ?? ''), args: JSON.stringify(p.args ?? {}), model: base.model });
+      }
+      if (p.kind === 'tool_result' && p.call_id) {
+        calls.set(p.call_id, { ...(calls.get(p.call_id) || {}), result_ts: ts, is_error: !!p.is_error, result_len: n(p.output_len) || String(p.output ?? '').length });
+      }
+    });
+    if (!parts.length) {
+      const row = { ...base, part_id: '', part_ord: 0, role, kind: 'none', text: '', extra };
+      turns.push(row);
+      firstRow.set(base.message_id, row);
+    }
+  });
+
+  // A usage row's id must not be a message's: the (message_id, part_id) key would be shared.
+  const taken = new Set(messages.map((m) => String(m.message_id)));
+  for (const u of records.filter((r) => r.type === 'usage').sort((a, b) => (a.usage_id < b.usage_id ? -1 : 1))) {
+    const tok = houseTokens(u);
+    if (!anyTokens(tok)) continue;
+    const a = u.anchor || {};
+    const mid = a.response_id != null ? byResponse.get(a.response_id) : (a.message_id != null ? a.message_id : null);
+    let row = mid != null ? firstRow.get(String(mid)) : null;
+    if (!row) {
+      let id = `usage:${u.usage_id}`;
+      while (taken.has(id)) id += '~';
+      taken.add(id);
+      row = { session_id: u.session_id, incarnation: u.incarnation ?? '', message_id: id, parent_id: '', response_id: a.response_id ?? '',
+        agent_id: '', ts: tsOf(id), ord: messages.length, state: '', model: u.model || '', effort: '',
+        part_id: '', part_ord: 0, role: 'assistant', kind: 'usage', text: '', extra: { anchor: Object.keys(a)[0] || 'none', granularity: u.granularity ?? null } };
+      turns.push(row);
+    }
+    for (const k of TOKENS) row[k] = (row[k] || 0) + tok[k];
+    if (!row.model && u.model) row.model = u.model;
+  }
+  for (const r of turns) for (const k of TOKENS) r[k] = r[k] || 0;
+
+  const s = records.find((r) => r.type === 'session') || null;
+  const meta = s ? { title: s.title ?? '', folder: s.cwd ?? '', created_at: s.created_at ?? null, updated_at: s.updated_at ?? null,
+    harness_version: s.harness_version ?? '', mode: s.mode ?? '' } : null;
+  const linkRecs = records.filter((r) => r.type === 'link');
+  // What the session states about its own origin goes on its row, as 0.20's links did; what it
+  // states about a child (a sub-agent it started) is returned apart, for the child's side.
+  const links = linkRecs.filter((l) => l.stated_on !== 'origin').map((l) => ({
+    kind: String(l.kind || ''), origin: String((l.origin || {}).session_id || ''), origin_store: String((l.origin || {}).harness_path || ''),
+    evidence: String(l.evidence || ''), at: l.at == null ? '' : String(l.at),
+    through: l.inherits && l.inherits.through_message_id ? String(l.inherits.through_message_id) : '',
+  }));
+  const children = linkRecs.filter((l) => l.stated_on === 'origin').map((l) => ({
+    kind: String(l.kind || ''), session_id: String((l.child || {}).session_id || ''), evidence: String(l.evidence || ''), at: l.at == null ? '' : String(l.at),
+  }));
+  return { turns, calls: [...calls.values()].filter((c) => c.call_id), meta, links, children };
+}
+
+module.exports = { rowsOf, houseTokens };

package/memhouse/house/relocate.js

@@ -19,6 +19,7 @@ const DERIVED = ['text_ngram', 'text_word'];
 function isDurableMetaKey(key) {
   return key === 'schema_version'
     || key === 'min_writer_schema'
+    || key === 'profile_converted'   // the profile conversion ran to the end (schema-choice.js)
     || key === 'house_id'            // forward-compat: harmless if absent today
     || /^share:/.test(key);
 }
@@ -41,7 +42,7 @@ function copyColumns(destCols, srcCols) {
  *
  * @param url        the source MEMHOUSE_URL (http(s)://host:8123)
  * @param opts.host  override the host — the DESTINATION's route to the source may differ
- *                   from the pilot's URL (NAT, split-horizon DNS, a private peering name)
+ *                   from the user's URL (NAT, split-horizon DNS, a private peering name)
  * @param opts.port  override the native port
  * @param opts.insecure  use remote() + plain 9000 instead of remoteSecure() + 9440
  * @returns { fn: 'remoteSecure'|'remote', host, port, addr: 'host:port' }