memhouse 0.18.12 → 0.20.1

package/memhouse/house/doctor-v3.js

@@ -0,0 +1,324 @@
+// `memhouse doctor` on a schema-3 house (harness-sessions-sql's profile): the house-side checks,
+// transport-free.
+//
+//   tables          the six conversation tables in <db> and the two capture tables in <db>_archive
+//                   exist, with exactly the sorting keys schema-profile.js PROFILE_KEYS names
+//   columns         every column of the template is on the server with its type and, where the
+//                   template says MATERIALIZED, still materialized (a lost MATERIALIZED search_word
+//                   is a search that silently finds nothing)
+//   identity        no row with an empty writer (an async insert skips the stamp)
+//   ts stable       no message part or call whose ts moved between its versions: sessions-model
+//                   §1 rule 7 says a record's ts never changes
+//   dedupe          captured sessions of this host whose converted rows still wait for the dedupe,
+//                   and deduped ones that keep converted rows the capture lacks (profile-dedupe.js)
+//   pricing         every model in the house has a price
+//   token capture   no harness ships assistant messages with zero tokens
+//   forks           the engine edges (profile-edges.js): forks billed once, the copied tokens counted
+//                   only in the origin, and the edges not verified (informational)
+//   host identity   which machines wrote sessions, and whether this one is among them
+//
+// NO WHOLE-TABLE SCANS BY DEFAULT. Everything on `sessions` reads it FINAL (one row per session,
+// small by design). On `session_parts` and `calls` the identity and ts checks read only the last
+// `days` days (`ts >= …`, the minmax index), and `full: true` (`doctor --full`) widens them to the
+// whole tables, which on a big house is the one expensive thing doctor can be asked to do.
+//
+// Every entry point takes a transport `{ exec, q }` (q-cli.js httpTransport) and returns
+// checks: [{ ok, label, hint }], the shape bin/memhouse.js renders.
+
+'use strict';
+
+const V3 = require('./schema-profile');
+const Q = require('./queries-v3');
+const H = require('./house');
+
+const SETTINGS = {};
+
+function lit(s) { return `'${String(s).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`; }
+
+/** Split a column declaration's type from its DEFAULT / MATERIALIZED / CODEC / … tail (top level only). */
+function splitDecl(decl) {
+  const s = String(decl).trim();
+  let depth = 0; let inStr = false;
+  for (let i = 0; i < s.length; i++) {
+    const c = s[i];
+    if (inStr) { if (c === '\\') i++; else if (c === "'") inStr = false; continue; }
+    if (c === "'") { inStr = true; continue; }
+    if (c === '(') depth++;
+    else if (c === ')') depth--;
+    else if (depth === 0 && /\s/.test(c)) {
+      const rest = s.slice(i).trimStart();
+      const m = /^(DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL|CODEC|COMMENT|TTL)\b/i.exec(rest);
+      if (m) return { type: s.slice(0, i).trim(), kind: ['DEFAULT', 'MATERIALIZED', 'ALIAS', 'EPHEMERAL'].includes(m[1].toUpperCase()) ? m[1].toUpperCase() : (/\b(DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL)\b/i.exec(rest.slice(m[0].length)) || [null, ''])[1].toUpperCase() };
+    }
+  }
+  return { type: s, kind: '' };
+}
+
+/** Split `a T, b U(x, y)` at top-level commas. */
+function topSplit(s) {
+  const out = []; let depth = 0; let buf = '';
+  for (const c of s) {
+    if (c === '(') depth++;
+    if (c === ')') depth--;
+    if (c === ',' && depth === 0) { out.push(buf.trim()); buf = ''; continue; }
+    buf += c;
+  }
+  if (buf.trim()) out.push(buf.trim());
+  return out;
+}
+
+/**
+ * The columns the server should report for one table, from the template:
+ * [{ name, type, materialized }]. A Nested column becomes its `name.field Array(T)` columns.
+ */
+function expectedColumns(tpl, type) {
+  const cols = V3.templateColumns(tpl)[type] || [];
+  const out = [];
+  for (const c of cols) {
+    const nm = /^Nested\s*\(([\s\S]*)\)\s*$/i.exec(c.type.trim());
+    if (nm) {
+      for (const f of topSplit(nm[1])) {
+        const sp = f.indexOf(' ');
+        out.push({ name: `${c.name}.${f.slice(0, sp)}`, type: `Array(${f.slice(sp + 1).trim()})`, materialized: false });
+      }
+      continue;
+    }
+    const d = splitDecl(c.type);
+    out.push({ name: c.name, type: d.type, materialized: d.kind === 'MATERIALIZED' });
+  }
+  return out;
+}
+
+const norm = (t) => String(t).replace(/\s+/g, ' ').replace(/\(\s+/g, '(').replace(/\s+\)/g, ')').trim();
+
+/** Compare the server's system.columns rows for one table against the template. */
+function columnProblems(fq, expected, serverCols) {
+  const by = new Map(serverCols.map((r) => [r.name, r]));
+  const missing = []; const wrong = [];
+  for (const c of expected) {
+    const got = by.get(c.name);
+    if (!got) { missing.push(`${fq}.${c.name}`); continue; }
+    if (norm(got.type) !== norm(c.type)) wrong.push(`${fq}.${c.name} is ${got.type}, template says ${c.type}`);
+    else if (c.materialized && got.default_kind !== 'MATERIALIZED') wrong.push(`${fq}.${c.name} lost its MATERIALIZED clause`);
+  }
+  return { missing, wrong };
+}
+
+/**
+ * The house-side checks.
+ * @param {{exec, q}} t
+ * @param {{ db, member, full?, days?, hostId?, tpl?, pricing?, user?, url? }} o  user/url: who connects, to name a refused credential
+ */
+async function houseChecks(t, { db, member, full = false, days = 14, hostId = null, hostAliases = [], tpl = null, calculateCost = null, user = '', url = '', pricingHint = null } = {}) {
+  const checks = [];
+  const add = (ok, label, hint, o = {}) => checks.push({ ok, label, hint, ...o });
+  // A credential the server refused is named as that, by ClickHouse's code (schema-choice.js), not
+  // as tables that could not be listed.
+  const SC = require('./schema-choice');
+  const listFailed = (what, e) => (SC.isAuthFailure(e) ? `${SC.authFailureText(user, url)} — fix the connection above first` : `could not list ${what}: ${e.message}`);
+  const N = V3.profileNames(db, member);
+  const rooms = Q.v3Rooms(member, db);
+  const rows = (sql, settings) => t.q.rows(sql, { ...SETTINGS, ...(settings || {}) });
+  rooms.hostAliases = await require('./host-alias').load((sql) => rows(sql), rooms);
+  const template = tpl || V3.readTemplate();
+
+  // 1. tables and keys
+  let present = new Map();
+  try {
+    present = new Map((await rows(`SELECT database, name, sorting_key FROM system.tables
+      WHERE (database = ${lit(N.db)} AND name IN (${V3.CONVERSATION_TYPES.map((x) => lit(`${member}_${x}`)).join(', ')}))
+         OR (database = ${lit(N.archive)} AND name IN (${V3.CAPTURE_TYPES.map((x) => lit(`${member}_${x}`)).join(', ')}))`))
+      .map((r) => [`${r.database}.${r.name}`, r.sorting_key]));
+  } catch (e) { add(false, 'schema-3 tables', listFailed('them', e)); return checks; }
+  // The statements that create the missing tables, from the template --ensure-schema applies, and
+  // for capture tables their grant: what an admin runs. Never an install as the member, which
+  // would configure the admin's machine as that member.
+  // In the server's own text-index dialect, as --ensure-schema writes it (house.js textIndexDialectFor).
+  const createSql = async (types) => {
+    let legacy = false;
+    try { legacy = H.textIndexDialectFor(String(((await rows('SELECT version() AS v'))[0] || {}).v || '')) === 'legacy'; } catch { /* the modern grammar */ }
+    const stmts = V3.renderStatements(template, { db, member }).filter((s) => types.includes(s.type)).map((s) => `${legacy ? H.legacyTextIndexDialect(s.sql) : s.sql};`);
+    const grants = V3.captureGrants(db, member).filter((g) => types.some((x) => V3.CAPTURE_TYPES.includes(x) && g.includes(` ${N[x]} `))).map((g) => `${g};`);
+    const dbs = types.some((x) => V3.CAPTURE_TYPES.includes(x)) ? [`CREATE DATABASE IF NOT EXISTS ${N.archive};`] : [];
+    return [...dbs, ...stmts, ...grants].join('\n');
+  };
+  const adminFix = async (types) => `as an admin, run (on servers before 26.x with SETTINGS allow_experimental_full_text_index = 1):\n${await createSql(types)}`;
+
+  // 1b. a member name that collides with another (bob, bob_smith): the grant <db>.<name>_* of one
+  // reaches the other's tables. Seen from here as tables we can list beyond our own names.
+  try {
+    const names = (await rows(`SELECT name FROM system.tables WHERE database = ${lit(N.db)}`)).map((r) => r.name);
+    const collide = require('../share').memberCollisions(member, names);
+    add(!collide.length, collide.length ? `member names collide: ${collide.join(', ')}` : 'member name: no other member\'s tables within reach',
+      collide.length ? `'${member}' and ${collide.join(', ')}: one name is the other's plus an underscore, so a grant <db>.<name>_* reaches the other's tables, to read and to write — docs/guides/troubleshooting.md, "Two member names that collide"` : undefined);
+  } catch (e) { add(false, 'member name', listFailed("the house's tables", e)); }
+  // 1c. sessions `memhouse import` wrote on the development layout (<m>_turns): a recapture cannot
+  // bring them back, so they are named before anyone moves the member (schema-choice.js).
+  try {
+    const sc = require('./schema-choice');
+    const det = await sc.detectHouseSchema((sql) => rows(sql), N.db, member);
+    if (det.devLayout) {
+      const notice = sc.importsNotice(det, member);
+      add(!notice, notice ? `${det.devImports === null ? 'some' : det.devImports} imported session(s) need a re-import` : `${member}_turns: no imported sessions to carry over`,
+        notice || `${member}_turns is the development builds' layout: its admin recaptures or converts '${member}' (docs/guides/upgrade-to-0.20.md)`);
+    }
+  } catch (e) { add(false, 'imported sessions', `could not check: ${e.message}`); }
+  for (const [layer, types, where] of [['conversation', V3.CONVERSATION_TYPES, N.db], ['capture', V3.CAPTURE_TYPES, N.archive]]) {
+    const missing = types.filter((x) => !present.has(N[x]));
+    const keyBad = types.filter((x) => present.has(N[x])).map((x) => [x, V3.keyProblem(x, present.get(N[x]))]).filter(([, p]) => p);
+    add(!missing.length && !keyBad.length,
+      missing.length ? `${layer} tables: ${types.length - missing.length}/${types.length} in '${where}' — missing ${missing.map((x) => N[x]).join(', ')}`
+        : keyBad.length ? `${layer} tables: wrong sorting key on ${keyBad.map(([x, p]) => `${N[x]} ${p}`).join('; ')}`
+          : `${layer} tables: ${types.length}/${types.length} in '${where}' (${types.map((x) => N[x]).join(', ')}), keys correct`,
+      missing.length ? (layer === 'capture' ? `the shipper cannot write without them, and a member cannot create them; ${await adminFix(missing)}`
+        : `memhouse ship --ensure-schema creates them (the member holds ${N.db}.${member}_*); if it is refused, ${await adminFix(missing)}`)
+        : 'a wrong key breaks replacement (§2): the table has to be rebuilt from the template by its owner');
+  }
+
+  // 2. columns against the template
+  try {
+    const missing = []; const wrong = []; let seen = 0;
+    for (const x of V3.PROFILE_TYPES) {
+      if (!present.has(N[x])) continue;
+      seen++;
+      const [d, tb] = N[x].split('.');
+      const cols = await rows(`SELECT name, type, default_kind FROM system.columns WHERE database = ${lit(d)} AND table = ${lit(tb)}`);
+      const p = columnProblems(N[x], expectedColumns(template, x), cols);
+      missing.push(...p.missing); wrong.push(...p.wrong);
+    }
+    const bad = missing.length + wrong.length;
+    add(bad === 0 && seen === V3.PROFILE_TYPES.length,
+      seen !== V3.PROFILE_TYPES.length ? `columns: only ${seen}/${V3.PROFILE_TYPES.length} tables exist, so the template comparison is incomplete`
+        : bad === 0 ? 'columns: every schema-3 table matches the template (name, type and kind)'
+          : `columns: ${missing.length} missing, ${wrong.length} wrong — ${[...missing, ...wrong].slice(0, 4).join('; ')}${bad > 4 ? ' …' : ''}`,
+      'a missing column, or a type retyped in harness-sessions-sql v1: memhouse ship --ensure-schema (the archive tables: an admin, docs/guides/troubleshooting.md); any other changed type, or a lost MATERIALIZED clause, needs the table rebuilt by its owner');
+  } catch (e) { add(false, 'columns', `could not compare against the template: ${e.message}`); }
+
+  const haveConv = V3.CONVERSATION_TYPES.every((x) => present.has(N[x]));
+  if (!haveConv) return checks;
+  const window = full ? '' : ` AND ts >= now64(3) - toIntervalDay(${Number(days)})`;
+  const scope = full ? 'all rows' : `last ${days} days`;
+
+  // 3. identity stamping
+  try {
+    const s = (await rows(`SELECT count() AS n, countIf(writer = '') AS blank FROM ${N.sessions} FINAL WHERE is_deleted = 0`))[0] || {};
+    const tr = (await rows(`SELECT count() AS n, countIf(writer = '') AS blank FROM ${N.session_parts} WHERE 1${window}`))[0] || {};
+    const cr = (await rows(`SELECT count() AS n, countIf(writer = '') AS blank FROM ${N.calls} WHERE 1${window}`))[0] || {};
+    const blank = Number(s.blank || 0) + Number(tr.blank || 0) + Number(cr.blank || 0);
+    const total = Number(s.n || 0) + Number(tr.n || 0) + Number(cr.n || 0);
+    add(blank === 0, total === 0 ? 'identity stamping (no rows yet)'
+      : blank === 0 ? `identity stamping: ${s.n} sessions, ${tr.n} part and ${cr.n} call rows (${scope}), all attributed`
+        : `identity stamping: ${blank} of ${total} rows have an empty writer (${scope})`,
+    'a writer used async_insert=1 — the MATERIALIZED currentUser() stamp does not run during an async flush');
+  } catch (e) { add(false, 'identity stamping', e.message); }
+
+  // 4. ts stable (sessions-model §1 rule 7): a part's or call's versions all carry one ts. The
+  //    profile keys rows without ts, so a moved ts replaces; it is still an adapter breaking the rule.
+  try {
+    const dupSql = (table, id) => `SELECT count() AS n, groupArray(3)(concat(harness, ':', session_id, ' ', i)) AS sample FROM (
+      SELECT harness, session_id, ${id} AS i FROM ${table} WHERE is_deleted = 0${window}
+      GROUP BY harness, host, harness_path, session_id, incarnation, i HAVING uniqExact(ts) > 1)`;
+    const dt = (await rows(dupSql(N.session_parts, "concat(message_id, '#', part_id)")))[0] || {};
+    const dc = (await rows(dupSql(N.calls, 'call_id')))[0] || {};
+    const n = Number(dt.n || 0) + Number(dc.n || 0);
+    add(n === 0,
+      n === 0 ? `ts stable: no message part or call whose ts moved (${scope})`
+        : `ts stable: ${dt.n} message part(s), ${dc.n} call(s) whose ts moved between versions (${scope}) — e.g. ${[...(dt.sample || []), ...(dc.sample || [])].slice(0, 3).join('; ')}`,
+      'sessions-model §1 rule 7: a record\'s ts never changes; the adapter that wrote them has an unstable ts');
+  } catch (e) { add(false, 'ts stable', e.message); }
+
+  // 5. the dedupe (profile-dedupe.js): this host's captured sessions whose converted rows still wait
+  //    for it, and deduped sessions that keep converted rows the capture lacks (their transcript
+  //    lost them; kept, and said so).
+  if (hostId) {
+    try {
+      const pd = require('./profile-dedupe');
+      const pending = await rows(pd.storesWithLegacySQL(N, hostId, { marked: false, aliases: hostAliases }));
+      const kept = await rows(`SELECT count() AS n, sum(JSONExtractUInt(state, 'dedupe', 'kept', 'turns') + JSONExtractUInt(state, 'dedupe', 'kept', 'calls')) AS rows
+        FROM ${N.session_sources} FINAL WHERE is_deleted = 0 AND host = ${lit(hostId)} AND role = 'legacy' AND NOT JSONExtractBool(state, 'pending')
+          AND JSONExtractUInt(state, 'dedupe', 'kept', 'turns') + JSONExtractUInt(state, 'dedupe', 'kept', 'calls') > 0`);
+      const k = kept[0] || {};
+      add(pending.length === 0,
+        (pending.length === 0 ? 'dedupe: no captured session here waits for its converted rows to be matched'
+          : `dedupe: ${pending.length} captured session(s) here still hold converted rows not yet matched — e.g. ${pending.slice(0, 3).map((x) => `${x.harness}:${x.session_id}`).join(', ')}`)
+          + (Number(k.n || 0) ? `; ${k.n} deduped session(s) keep ${k.rows} converted row(s) their transcript no longer has` : ''),
+        'the capture dedupes a session once it has captured it to the end; one pass (memhouse ship) usually clears it');
+    } catch (e) { add(false, 'dedupe', e.message); }
+  }
+
+  // 6. pricing and token capture — from sessions FINAL (usage entries per session)
+  try {
+    const cost = calculateCost || require('../../pricing').calculateCost;
+    const models = await t.exec(Q.aggregate(rooms, { source: 'usage', dims: ['model'], metrics: ['messages'], where: { excludeModels: true } }));
+    const unpriced = models.filter((m) => cost(m.model, 1e6, 0, 0, 0) === null);
+    const total = models.reduce((a, m) => a + Number(m.messages), 0);
+    const missed = unpriced.reduce((a, m) => a + Number(m.messages), 0);
+    const pct = total ? (missed / total) * 100 : 0;
+    const share = pct >= 0.5 ? `${Math.round(pct)}% of messages` : `${missed} message${missed === 1 ? '' : 's'}`;
+    // An unpriced model is a gap in the cost figures, not a fault in the house: a warning, never a
+    // failure. The caller says whether an update would bring prices (pricingHint); without it, the
+    // hint names both cases rather than sending a current install to `memhouse update`.
+    if (!unpriced.length) add(true, `pricing: every model in this house has a price (${models.length} models)`);
+    else {
+      add(true, `pricing: ${unpriced.length} model${unpriced.length > 1 ? 's' : ''} unpriced — ${share} cost nothing in the dashboard (${unpriced.map((m) => m.model).join(', ')})`,
+        `${pricingHint || 'prices ship with each release: a newer one may add them (memhouse update), and on the latest one their cost stays unknown until a release does'}; memhouse q spend lists them`,
+        { warn: true });
+    }
+    const zero = await rows(`SELECT harness, sum(assistant_messages) AS n FROM ${N.sessions} FINAL
+      WHERE is_deleted = 0 AND origin = 'ship' GROUP BY harness
+      HAVING n > 0 AND sum(input_tokens) + sum(output_tokens) = 0 ORDER BY harness`);
+    add(zero.length === 0,
+      zero.length === 0 ? 'token capture: every harness shipping here reports usage'
+        : `token capture: ${zero.map((r) => `${r.harness} (${r.n} messages)`).join(', ')} ship ZERO tokens — their cost shows as $0, not as unknown`,
+      'that adapter does not extract usage; the messages are stored, the numbers are not');
+  } catch (e) { add(false, 'pricing', e.message); }
+
+  // 6b. removed content (plan v4 M4): what every read hides, kept as removed versions. Informational.
+  try {
+    const rc = await rows(`SELECT toString(harness) AS h, uniqExact(session_id, harness_path, incarnation) AS s, uniqExact(session_id, harness_path, incarnation, message_id) AS m
+      FROM ${N.session_parts} FINAL WHERE removed = 1 AND is_deleted = 0${window} GROUP BY h ORDER BY m DESC`);
+    const m = rc.reduce((a, r) => a + Number(r.m), 0); const s = rc.reduce((a, r) => a + Number(r.s), 0);
+    add(true, m === 0 ? `removed content: none (${scope})`
+      : `removed content: ${m} message(s) in ${s} session(s) (${scope}) — ${rc.map((r) => `${r.h} ${r.m}`).join(', ')}; kept as removed versions, searchable with \`memhouse q removed\`, never counted`);
+  } catch (e) { add(false, 'removed content', e.message); }
+
+  // 6c. forks (plan v4 M4): the engine edges, and the copied usage they keep out of the forks'
+  //     totals (billed once, in the origin). Informational.
+  try {
+    const edges = `FROM ${N.session_links} FINAL WHERE owner = 'engine' AND removed = 0 AND kind IN ('fork', 'rewind_copy')`;
+    // A lineage fork (verified null: Claude Code's and Codex's copies) is neither verified nor failed.
+    const fk = (await rows(`SELECT count() AS n, countIf(ifNull(verified, false)) AS ok, countIf(verified = false) AS failed, countIf(verified IS NULL) AS lineage,
+        countIf(ifNull(verified, false) AND length(r_copied_usage) > 0) AS billed, sum(length(r_copied_usage)) AS out ${edges}`))[0] || {};
+    // In the house split the rollup bills (profile-rollup.js: cached input counted once, reasoning in output).
+    // Every listed row is summed: a per_record response's earlier records too (only converted Claude Code
+    // rows are per_record, and the edges compare captured rows first, so they are not listed in practice).
+    const tk = Number(fk.out || 0) ? (await rows(`SELECT sum(toUInt64(greatest(if(ifNull(input_includes_cache, false), toInt64(ifNull(input, 0)) - toInt64(ifNull(cache_read, 0)) - toInt64(ifNull(cache_write, 0)), toInt64(ifNull(input, 0))), 0))
+        + ifNull(tool_prompt, 0) + ifNull(output, 0) + if(ifNull(output_includes_reasoning, true), 0, ifNull(reasoning, 0)) + ifNull(cache_read, 0) + ifNull(cache_write, 0)) AS t
+      FROM ${N.session_usage} FINAL WHERE removed = 0 AND (host, toString(harness), toString(harness_path), session_id, incarnation, usage_id) IN (
+        SELECT host, toString(harness), toString(harness_path), session_id, incarnation, arrayJoin(r_copied_usage) ${edges} AND ifNull(verified, false))`))[0] || {} : {};
+    const n = Number(fk.n || 0); const unverified = Number(fk.failed || 0); const lineage = Number(fk.lineage || 0);
+    add(true, n === 0 ? 'forks: no fork link in this house'
+      : `forks: ${n} fork link(s), ${fk.billed} billed once (${Number(tk.t || 0)} copied tokens counted only in the origin)`
+        + (unverified ? `; ${unverified} not verified, counted in both sessions` : '')
+        + (lineage ? `; ${lineage} of Claude Code or Codex, lineage only` : ''));
+  } catch (e) { add(false, 'forks', e.message); }
+
+  // 7. host identity
+  try {
+    // Folded as status and `q machines` fold them: a 0.18 id this member's machine recorded as
+    // its own reads as that machine (host-alias.js).
+    const hosts = require('./q-cli').machinesFolded(await t.exec(Q.machines(rooms)), rooms);
+    // host is sessions-model/host/v0 (a 32-hex machine fingerprint); the label is for people.
+    const name = (h) => (h.label ? `${h.label} (${h.host.slice(0, 8)}…)` : h.host);
+    const mine = hostId ? hosts.find((h) => h.host === hostId) : null;
+    const before = hosts.filter((h) => hostAliases.includes(h.host));
+    const others = hosts.filter((h) => h.host !== hostId && !hostAliases.includes(h.host));
+    add(true, hosts.length === 0 ? `host identity ${hostId || '?'} (nothing shipped from here yet)`
+      : `host identity ${hostId || '?'} — ${mine ? `${mine.sessions} sessions from this machine${mine.aliases ? ` (its pre-0.20 id ${mine.aliases.join(', ')} folded in)` : ''}` : 'no sessions from this machine yet'}`
+        + (before.length ? `, ${before.reduce((a, h) => a + Number(h.sessions), 0)} under its pre-0.20 id ${before.map((h) => h.host).join(', ')}` : '')
+        + (others.length ? `, ${others.length} other host${others.length > 1 ? 's' : ''}: ${others.map((h) => `${name(h)} (${h.sessions})`).join(', ')}` : ''));
+  } catch { /* the reads above already reported anything that would break this */ }
+  return checks;
+}
+
+module.exports = { houseChecks, expectedColumns, columnProblems, splitDecl, topSplit };

package/memhouse/house/host-alias.js

@@ -0,0 +1,85 @@
+// One machine, two ids: the 0.18 one and the sessions-model one.
+//
+// 0.18 named a machine `<short>-<8 hex>` (host.js identity()); 0.20 names it by
+// sessions-model/host/v0, a 32-hex digest of the OS machine id. Rows converted from the 0.18 rooms
+// keep the 0.18 id, captured rows carry the new one, so after a move one machine read as two
+// writers. Neither id can be computed from the other — the 8 hex are a different, truncated digest
+// — so the conversion, which sees only the house, cannot map them. The machine itself knows both.
+// It records the pair in its member's own meta room (`host_alias:<0.18 id>` = `<new id>`) when it
+// switches and on each capture pass, and readers fold the 0.18 id's rows into the machine's at
+// query time. Rows are never rewritten. An alias is read from a member's own meta and applied to
+// that member's own rows only, so no member can fold another's machines.
+
+const KEY = 'host_alias:';
+const OLD_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;   // the 0.18 `<short>-<fp8>` (and anything of its shape)
+const NEW_RE = /^(r-)?[0-9a-f]{32}$/;                    // sessions-model/host/v0, or its random fallback
+
+const lit = (s) => `'${String(s).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+const valid = (old, neu) => typeof old === 'string' && typeof neu === 'string' && old !== neu && OLD_RE.test(old) && NEW_RE.test(neu);
+
+/** The recorded aliases of one member, latest per 0.18 id. `meta` is that member's meta room. */
+function readSQL(meta) {
+  return `SELECT substring(key, ${KEY.length + 1}) AS old, argMax(value, updated_at) AS new
+FROM ${meta} WHERE startsWith(key, ${lit(KEY)}) GROUP BY key`;
+}
+
+/** { old: new } from rows of readSQL, dropping anything that is not of the two ids' shapes. */
+function parse(rows) {
+  const map = {};
+  for (const r of rows || []) if (valid(r.old, r.new)) map[r.old] = r.new;
+  return map;
+}
+
+/** A member's aliases, or {} when its meta cannot be read (a housemate's share does not grant it). */
+async function load(rows, rooms) {
+  if (!rooms || !rooms.meta) return {};
+  try { return parse(await rows(readSQL(rooms.meta))); } catch { return {}; }
+}
+
+/**
+ * Record `old` -> `neu` in the member's meta when rows exist under `old` and the pair is not
+ * recorded yet. `ph` answers rows(sql) and run(sql) (profile-house.js, or a transport).
+ * @returns {Promise<boolean>} whether it wrote
+ */
+async function record(ph, { meta, sessions }, old, neu) {
+  if (!valid(old, neu)) return false;
+  const have = parse(await ph.rows(readSQL(meta)));
+  if (have[old] === neu) return false;
+  const under = await ph.rows(`SELECT count() AS n FROM ${sessions} WHERE host = ${lit(old)} LIMIT 1`);
+  if (!under.length || !Number(under[0].n)) return false;
+  await ph.run(`INSERT INTO ${meta} (key, value) VALUES (${lit(KEY + old)}, ${lit(neu)})`);
+  return true;
+}
+
+/** The id a row's host reads as. */
+const fold = (host, aliases) => (aliases && Object.prototype.hasOwnProperty.call(aliases, host) ? aliases[host] : host);
+
+/** Every id that reads as `host`: itself and the 0.18 ids aliased to it. */
+function idsOf(host, aliases) {
+  return [host, ...Object.keys(aliases || {}).filter((o) => aliases[o] === host && o !== host)];
+}
+
+/**
+ * Merge per-host rows whose hosts fold to one machine. `how` names the columns: sum, min, max,
+ * union (arrays, sorted), any (first non-empty). Row order is kept by first appearance.
+ */
+function foldRows(rows, aliases, how = {}) {
+  if (!aliases || !Object.keys(aliases).length) return rows;
+  const key = how.key || 'host';
+  const out = new Map();
+  for (const r of rows) {
+    const k = fold(r[key], aliases);
+    const cur = out.get(k);
+    if (!cur) { out.set(k, { ...r, [key]: k, ...(k !== r[key] ? { aliases: [r[key]] } : {}) }); continue; }
+    if (k !== r[key]) cur.aliases = [...(cur.aliases || []), r[key]];
+    for (const c of how.sum || []) cur[c] = Number(cur[c] || 0) + Number(r[c] || 0);
+    for (const c of how.max || []) cur[c] = Math.max(Number(cur[c] || 0), Number(r[c] || 0));
+    for (const c of how.min || []) cur[c] = [cur[c], r[c]].filter((v) => v !== null && v !== undefined && v !== '').map(Number).reduce((a, b) => Math.min(a, b), Infinity);
+    for (const c of how.union || []) cur[c] = [...new Set([...(cur[c] || []), ...(r[c] || [])])].sort();
+    for (const c of how.any || []) if (!cur[c] && r[c]) cur[c] = r[c];
+  }
+  for (const r of out.values()) for (const c of how.min || []) if (r[c] === Infinity) r[c] = null;
+  return [...out.values()];
+}
+
+module.exports = { KEY, readSQL, parse, load, record, fold, idsOf, foldRows, valid };

package/memhouse/house/house.js

@@ -80,7 +80,7 @@ const MIGRATIONS = {
  * mixed-fleet story: every release from 0.10.0 on REFUSES a house whose recorded schema
  * it does not support, in both directions — a house too new says "update memhouse", a
  * house too old says "memhouse migrate". The backward half cannot be code: releases
- * already on the registry read none of this, so for them the enforcement is the pilot's
+ * already on the registry read none of this, so for them the enforcement is the user's
  * GRANTs (revoke the mutation privileges after migrating, and a pre-0.10 shipper's
  * delete fails loudly instead of destroying retained parses). `memhouse migrate` prints
  * that advice.
@@ -184,8 +184,8 @@ function installCommand() {
 // user names. Refuse what would need quoting rather than quote it: a name a person cannot
 // type bare into their own SQL is a name they will get wrong. `default` is deliberately
 // ALLOWED as a house name — a stock container's default database is a real place to keep
-// a house, and the pilot may pick user 'alice' with house 'default'. Only the names that
-// are not a database at all are refused.
+// a house, and the person running memhouse may pick user 'alice' with house 'default'.
+// Only the names that are not a database at all are refused.
 const RESERVED_DBS = new Set(['system', 'information_schema']);
 function assertUsableName(name, what = 'name') {
   if (typeof name !== 'string' || !/^[A-Za-z][A-Za-z0-9_]*$/.test(name)) {

package/memhouse/house/migrate.js

@@ -23,7 +23,7 @@
 // measured losses and are not per-migration choices:
 //
 //   NOTHING IS DELETED. A replaced object is renamed to `<name>_pre_<suffix>` and left
-//   for the pilot to drop. A leftover `__migrating` table from an interrupted run is
+//   for the user to drop. A leftover `__migrating` table from an interrupted run is
 //   never cleared automatically — it may hold the only copy of mid-flight rows.
 //
 //   PROVENANCE IS COPIED, NOT RESTAMPED. `user_id` is MATERIALIZED currentUser(), so a
@@ -193,7 +193,7 @@ async function rebuildRoom(q, ctx, { type, name, keepSuffix }, ui) {
  * ADD COLUMN anything the template declares that the live table lacks — for objects a
  * migration does NOT rebuild. `sessions` keeps its key across schema 2 but still gains
  * `epoch`; without this, the very next ship opens with a "fields are being DISCARDED"
- * warning and sends the pilot to a second command.
+ * warning and sends the user to a second command.
  */
 async function healColumns(q, ctx, { type, name }, ui) {
   const { templateColumns } = require('../shipper/ship');

package/memhouse/house/profile-dedupe.js

@@ -0,0 +1,183 @@
+// The dedupe writer on harness-sessions-sql's profile (step B4): one session's converted rows (adapter
+// 'legacy') matched against its adapter-derived rows by dedupe-profile.js, the matched ones
+// tombstoned, the session marked deduped. The profile's form of capture.js dedupeSession.
+//
+// THE WRITER (each line is a decision)
+//   stores      the captured session's store, and the same session id in the unknown store ('': a
+//               0.18 row without a transcript path), under this host and its aliases (the old host
+//               id where the conversion kept it); the captured session's incarnation
+//   reads       the legacy session_parts, calls and session_usage in EVERY version (no FINAL: a
+//               tombstoned row keeps the run it matched, and the tokens a Codex re-report proof reads),
+//               the store's live adapter-derived parts
+//   nothing     a session with no legacy rows is { none }; one with no captured parts is { skipped }
+//               and gets no marker: matched against nothing, every legacy row would be kept for good
+//   tombstones  server-side, by key: the live row (FINAL) written again at rev + 1, is_deleted 1. In
+//               order: a pending marker, parts, calls, then their usage, the sessions refresh, then
+//               the marker. A pass stopped anywhere between them leaves the pending marker (or live
+//               legacy rows), so the sweep finds the session and the next pass finishes it; its plan
+//               decides the same way (a dead message's leftover usage goes with it; a message still
+//               live keeps the usage its proof reads). Two passes racing on one session converge
+//   marker      a session_sources row in the captured store: source_key 'legacy:rows', role
+//               'legacy', mode 'snapshot', generation 0, progress 1 + the legacy ship rows dead
+//               (at least the previous marker's + 1, so a repair outranks it under FINAL), state
+//               { v: 1, dedupe: <report> }. The pending marker is the same row with `pending: true`,
+//               at the previous marker's progress + 1 (1 with none), written once before the first
+//               tombstone; the marker then outranks it
+//   sessions    every store's sessions row of the session written again through the shared rollup
+//               (profile-rollup.js refreshSessionsSQL), before the marker; a store with no live row
+//               left is retired. A store that keeps legacy rows the capture lacks stays a session of
+//               its own: the profile counts a session per key, and harness-sessions-sql does not merge two
+//               hosts' copies
+//   repair      { redo: true } writes nothing when the plan tombstones nothing new and the marker
+//               already reports the same kept counts
+
+'use strict';
+
+const dp = require('../shipper/dedupe-profile');
+const dedupe = require('../shipper/dedupe-v3');
+const { lit } = require('./convert-v3');
+const rollup = require('./profile-rollup');
+
+const STORE = '(host, harness, harness_path, session_id, incarnation)';
+const MARKER_KEY = dedupe.LEGACY_FILE_KEY;
+const MARKER_ROLE = dedupe.LEGACY_ROLE;
+
+const tuple = (vals) => `(${vals.map(lit).join(', ')})`;
+const storeTuple = (r) => tuple([r.host, r.harness, r.harness_path, r.session_id, r.incarnation || '']);
+/** The stores a session's legacy rows may sit in (header). */
+function legacyStores(ref, aliases = []) {
+  const out = [];
+  for (const host of [ref.host, ...aliases]) for (const hp of [...new Set([ref.harness_path, ''])]) out.push({ ...ref, host, harness_path: hp });
+  return out;
+}
+const inStores = (stores) => `${STORE} IN (${stores.map(storeTuple).join(', ')})`;
+
+const PART_READ = `host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation, message_id, part_id, part_index,
+  order_key, agent_id, toString(role) AS role, toString(ts) AS ts, toString(served_model) AS served_model, toString(requested_model) AS requested_model,
+  response_id, toString(kind) AS kind, text, toString(tool_name) AS tool_name, args, output, call_id, error_message, extra, toString(rev) AS rev, is_deleted, removed`;
+const CALL_READ = `host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation, call_id, message_id, agent_id,
+  toString(ts) AS ts, toString(tool_name) AS tool_name, args, toString(rev) AS rev, is_deleted, removed`;
+
+// A converted row the conversion wrote removed (content only an older 0.18 copy held) is history: the
+// dedupe never matches, keeps or retires it.
+function legacyPartsSQL(P, stores) { return `SELECT ${PART_READ} FROM ${P.session_parts} WHERE ${inStores(stores)} AND adapter = 'legacy' AND removed = 0`; }
+function legacyCallsSQL(P, stores) { return `SELECT ${CALL_READ} FROM ${P.calls} WHERE ${inStores(stores)} AND adapter = 'legacy' AND removed = 0`; }
+function legacyUsageSQL(P, stores) {
+  return `SELECT host, toString(harness_path) AS harness_path, usage_id, toString(anchor_kind) AS anchor_kind, anchor_id, input, output,
+      cache_read, cache_write, input_includes_cache, toString(rev) AS rev, is_deleted
+    FROM ${P.session_usage} WHERE ${inStores(stores)} AND adapter = 'legacy' AND removed = 0`;
+}
+function capturedPartsSQL(P, ref) {
+  return `SELECT ${PART_READ} FROM ${P.session_parts} FINAL WHERE ${inStores([ref])} AND adapter != 'legacy' AND removed = 0`;
+}
+function capturedUsageSQL(P, ref) {
+  return `SELECT usage_id, toString(anchor_kind) AS anchor_kind, anchor_id, toString(ts) AS ts, toString(model) AS model, toString(input) AS input, toString(output) AS output,
+      toString(cache_read) AS cache_read, toString(cache_write) AS cache_write, input_includes_cache
+    FROM ${P.session_usage} FINAL WHERE ${inStores([ref])} AND adapter != 'legacy' AND removed = 0`;
+}
+function markerSQL(P, ref) {
+  return `SELECT toString(progress) AS progress, state FROM ${P.session_sources} FINAL
+    WHERE ${inStores([ref])} AND source_key = ${lit(MARKER_KEY)} AND role = ${lit(MARKER_ROLE)} AND is_deleted = 0`;
+}
+
+/** The live legacy rows of `table` at these keys, written again at rev + 1 as tombstones. `id`: the key's own columns. */
+function tombstoneSQL(P, table, stores, id, keys) {
+  // The filter sits in a subquery: ClickHouse resolves a SELECT alias in WHERE (profile-house.js retire).
+  return `INSERT INTO ${P[table]} SELECT * REPLACE (rev + 1 AS rev, toUInt8(1) AS is_deleted, now64(3) AS ingested_at)
+    FROM (SELECT * FROM ${P[table]} FINAL
+      WHERE ${inStores(stores)} AND adapter = 'legacy' AND (host, harness_path, ${id.join(', ')}) IN (${keys.map((k) => tuple([k.host, k.harness_path, ...id.map((c) => k[c])])).join(', ')}))`;
+}
+const TOMBSTONED = [['session_parts', ['message_id', 'part_id']], ['calls', ['call_id']], ['session_usage', ['usage_id']]];
+
+/** The marker row (header); `pending`: the one written before the first tombstone. */
+function markerRow(ref, plan, prev, { pending = false } = {}) {
+  let progress = pending ? 1 : plan.dead + 1;
+  if (prev && Number(prev.progress) >= progress) progress = Number(prev.progress) + 1;
+  return {
+    host: ref.host, harness: ref.harness, harness_path: ref.harness_path, session_id: ref.session_id, incarnation: ref.incarnation || '',
+    source_key: MARKER_KEY, mode: 'snapshot', position: '', generation: 0, progress,
+    state: JSON.stringify({ v: 1, ...(pending ? { pending: true } : {}), dedupe: plan.report }),
+    state_ver: 1, adapter: 'legacy', adapter_ver: '0', role: MARKER_ROLE, is_deleted: 0,
+  };
+}
+const keptOf = (m) => { try { const k = JSON.parse(m.state).dedupe.kept; return `${k.turns}/${k.calls}`; } catch { return null; } };
+const isPending = (m) => { try { return JSON.parse(m.state).pending === true; } catch { return false; } };
+const insertMarker = (db, P, m) => db.run(`INSERT INTO ${P.session_sources} FORMAT JSONEachRow\n${JSON.stringify(m)}`);
+
+/**
+ * Dedupe one captured session on the profile.
+ * @param {{rows: (sql: string) => Promise<object[]>, run: (sql: string) => Promise<unknown>}} db
+ * @param {object} P        profileNames(db, member)
+ * @param {object} ref      the captured session's store: host, harness, harness_path, session_id, incarnation
+ * @param {object} [o]
+ * @param {string[]} [o.aliases]   this host's other ids
+ * @param {boolean} [o.redo]       a repair re-plan
+ * @param {(ref: object) => Promise<Array>} [o.repeatsOf]  Codex re-report proofs, read only when the plan needs them
+ * @returns {Promise<{none?: true, skipped?: true, unchanged?: true, plan?: object, marker?: object}>}
+ */
+async function dedupeProfileSession(db, P, ref, o = {}) {
+  const stores = legacyStores(ref, o.aliases || []);
+  const [legacyParts, legacyCalls] = [await db.rows(legacyPartsSQL(P, stores)), await db.rows(legacyCallsSQL(P, stores))];
+  if (!legacyParts.length && !legacyCalls.length) return { none: true };
+  const capturedParts = await db.rows(capturedPartsSQL(P, ref));
+  if (!capturedParts.length) return { skipped: true };
+  const legacyUsage = await db.rows(legacyUsageSQL(P, stores));
+  const capturedUsage = await db.rows(capturedUsageSQL(P, ref));
+  const input = { harness: ref.harness, session_id: ref.session_id, legacyParts, legacyCalls, legacyUsage, capturedParts, capturedUsage };
+  let plan = dp.planProfileDedupe(input);
+  if (plan.report.rereport_unproven > 0 && o.repeatsOf) {
+    const repeats = await o.repeatsOf(ref);
+    if (repeats.length) plan = dp.planProfileDedupe({ ...input, repeats });
+  }
+  const prev = (await db.rows(markerSQL(P, ref)))[0];
+  const T = plan.tombstones;
+  if (o.redo && !T.session_parts.length && !T.calls.length && !T.session_usage.length
+    && (!prev || (!isPending(prev) && keptOf(prev) === `${plan.report.kept.turns}/${plan.report.kept.calls}`))) return { plan, unchanged: true };
+  let last = prev;
+  if (!prev || !isPending(prev)) { last = markerRow(ref, plan, prev, { pending: true }); await insertMarker(db, P, last); }
+  for (const [table, id] of TOMBSTONED) {
+    const keys = T[table];
+    for (let i = 0; i < keys.length; i += 500) await db.run(tombstoneSQL(P, table, stores, id, keys.slice(i, i + 500)));
+  }
+  const keys = new Map([[storeTuple(ref), ref]]);
+  for (const r of [...legacyParts, ...legacyCalls]) keys.set(storeTuple(r), r);
+  await db.run(rollup.refreshSessionsSQL(P, `${STORE} IN (${[...keys.keys()].join(', ')})`));
+  const marker = markerRow(ref, plan, last);
+  await insertMarker(db, P, marker);
+  return { plan, marker };
+}
+
+/**
+ * The captured stores of this host whose session has live legacy rows in a store dedupe reads.
+ * `marked`: with a marker (repair candidates: live legacy ship rows, or a marker reporting kept rows),
+ * else without one (the sweep). A capture's store is one with a session_sources row of a source.
+ * `stores`: only these ({harness, harness_path}), so a LIMIT is not filled by stores a pass skips.
+ */
+function storesWithLegacySQL(P, host, { marked, aliases = [], stores = null }) {
+  const hosts = [host, ...aliases].map(lit).join(', ');
+  const prefix = lit(marked ? 'legacy:ship:' : 'legacy:');
+  const legacy = (t, id) => `SELECT DISTINCT toString(harness) AS h, toString(harness_path) AS p, session_id AS s, incarnation AS i FROM ${t} FINAL
+      WHERE host IN (${hosts}) AND adapter = 'legacy' AND removed = 0 AND startsWith(${id}, ${prefix})`;
+  // The sweep also takes live legacy usage alone: a pass of an older build stopped after the parts and calls.
+  const usage = `SELECT DISTINCT toString(harness) AS h, toString(harness_path) AS p, session_id AS s, incarnation AS i FROM ${P.session_usage} FINAL
+      WHERE host IN (${hosts}) AND adapter = 'legacy' AND removed = 0`;
+  const L = `${legacy(P.session_parts, 'message_id')} UNION DISTINCT ${legacy(P.calls, 'call_id')}${marked ? '' : ` UNION DISTINCT ${usage}`}`;
+  const src = (role) => `SELECT DISTINCT toString(harness) AS h, toString(harness_path) AS p, session_id AS s, incarnation AS i,
+      JSONExtractUInt(state, 'dedupe', 'kept', 'turns') + JSONExtractUInt(state, 'dedupe', 'kept', 'calls') AS k
+    FROM ${P.session_sources} FINAL WHERE is_deleted = 0 AND host = ${lit(host)} AND ${role}`;
+  const isMarker = `source_key = ${lit(MARKER_KEY)} AND role = ${lit(MARKER_ROLE)}`;
+  // A pending marker is a dedupe that stopped part way: the sweep's, until the marker replaces it.
+  const markers = (pending) => `SELECT toString(harness), toString(harness_path), session_id, incarnation FROM ${P.session_sources} FINAL
+        WHERE is_deleted = 0 AND host = ${lit(host)} AND ${isMarker} AND ${pending ? '' : 'NOT '}JSONExtractBool(state, 'pending')`;
+  return `SELECT h AS harness, p AS harness_path, s AS session_id, i AS incarnation FROM (
+      ${src(marked ? isMarker : `role NOT IN ('side', ${lit(MARKER_ROLE)}, 'inherited')`)})
+    WHERE ((h, p, s, i) IN (${L}) OR (h, '', s, i) IN (${L})${marked ? ' OR k > 0' : ` OR (h, p, s, i) IN (${markers(true)})`})
+      ${marked ? '' : `AND (h, p, s, i) NOT IN (${markers(false)})`}
+      ${stores ? `AND (h, p) IN (${stores.length ? stores.map((x) => tuple([x.harness, x.harness_path])).join(', ') : "('', '')"})` : ''}
+    ORDER BY h, p, s`;
+}
+
+module.exports = {
+  dedupeProfileSession, storesWithLegacySQL, legacyStores, legacyPartsSQL, legacyCallsSQL, legacyUsageSQL, capturedPartsSQL, capturedUsageSQL, markerSQL,
+  tombstoneSQL, markerRow, MARKER_KEY, MARKER_ROLE,
+};