memhouse 0.18.12 → 0.20.1

package/memhouse/house/profile-house.js

@@ -0,0 +1,430 @@
+// The capture engine's reads and writes on harness-sessions-sql's profile tables (step B2, the writer):
+// the profile's form of capture.js HttpHouse's archive, cursor and row methods. It takes a
+// transport ({ query(sql, settings), post(params, body, gzip) }: HttpHouse is one) and the
+// member's profile names.
+//
+//   archive     session_units: a unit's body in chunks, and the read manifests (unit_key '')
+//   replay      the units of one source generation, folded from its manifests in progress order
+//               (harness-sessions-sql §4.1): each adds its units and drops its gone keys; a reset manifest
+//               starts the fold over. Only the folded units' chunks are read
+//   archive state  per source, the newest generation the archive holds, its greatest manifest
+//               progress and its fold: what a pass stopped before its cursor write left ahead
+//   cursor      session_sources rows of the session's sources (the legacy marker left out)
+//   current     the session's live adapter-derived keys per table, with their rev and first_seen:
+//               what a refold reads once
+//   retire      a key the view no longer makes: the live row again at the pass's rev, removed 1,
+//               server-side
+//   session     the sessions row: the spec columns given, memhouse's derived columns from the
+//               shared rollup (profile-rollup.js) over the rows just written
+
+'use strict';
+
+const rollup = require('./profile-rollup');
+const { lit } = require('./convert-v3');
+
+const STORE = '(host, harness, harness_path, session_id, incarnation)';
+// The session key as the many-session reads return it: aliases that shadow no column (a SELECT alias
+// named like a column would also stand for it in WHERE).
+const STORE_OUT = 'host AS zo, toString(harness) AS zh, toString(harness_path) AS zp, session_id AS zs, incarnation AS zi';
+const storeKeyOf = (r) => [r.host, r.harness, r.harness_path, r.session_id, r.incarnation || ''].map((x) => String(x ?? '')).join('\u0000');
+const storeWhere = (ref) => `${STORE} = (${[ref.host, ref.harness, ref.harness_path, ref.session_id, ref.incarnation || ''].map(lit).join(', ')})`;
+
+/** Each table's own key columns after the session key; a removed version is written by these. */
+const KEY_COLUMNS = {
+  session_parts: ['message_id', 'part_id'],
+  session_usage: ['usage_id'],
+  session_links: ['owner', 'kind', 'other_host', 'other_harness_path', 'other_harness', 'other_session_id', 'other_incarnation', 'at'],
+  session_lifecycle: ['lifecycle_id'],
+  calls: ['call_id'],
+};
+const ENTITY_TABLES = Object.keys(KEY_COLUMNS);
+/** A row's key within its session, as one string. */
+function keyOf(table, r) { return KEY_COLUMNS[table].map((c) => String(r[c] ?? '')).join('\u0000'); }
+
+// How long a read of the adapter_ver types stands: an admin may retype a table under a running shipper.
+const VER_TYPES_TTL = 60_000;
+const INSERT_SETTINGS = { async_insert: '0', input_format_skip_unknown_fields: '0', date_time_input_format: 'best_effort' };
+
+class ProfileHouse {
+  /** @param {{query: Function, post: Function}} transport @param {object} P profileNames(db, member) */
+  constructor(transport, P) { this.t = transport; this.P = P; this.inserted = {}; this.clock = Date.now; this.batchBytes = 8 << 20; }
+  rows(sql, settings) { return this.t.query(sql, settings); }
+  run(sql, settings = {}) { return this.t.post(Object.fromEntries(Object.entries(settings).map(([k, v]) => [k, String(v)])), sql, false); }
+
+  /**
+   * Rows into `table`, ~8 MiB a statement. `whole`: every row in one statement, however big (the
+   * capture's read manifests of one pass: one archive write, so one captured_at, is one pass).
+   */
+  async insert(table, rows, { whole = false } = {}) {
+    if (!rows.length) return;
+    // A table not yet brought to harness-sessions-sql v1's types (a member holds no ALTER on its archive, so
+    // an admin's schema step heals those) still takes adapter_ver as the old number; it reads back as
+    // the version through the capture's verOf.
+    if (await this.numericVer(table)) rows = rows.map((r) => (r.adapter_ver === undefined ? r : { ...r, adapter_ver: oldVer(r.adapter_ver) }));
+    for (let i = 0; i < rows.length;) {
+      // ~8 MiB a batch (batchBytes): an archive chunk is up to 1 MiB of text.
+      const lines = []; let size = 0;
+      for (; i < rows.length && (whole || size < this.batchBytes || !lines.length); i++) { const l = JSON.stringify(rows[i]); lines.push(l); size += l.length; }
+      try { await this.t.post({ query: `INSERT INTO ${this.P[table]} FORMAT JSONEachRow`, ...INSERT_SETTINGS }, lines.join('\n'), true); }
+      catch (e) { this.verTypes = null; throw e; }   // a retyped table: the next pass reads the types again
+    }
+    this.inserted[table] = (this.inserted[table] || 0) + rows.length;
+  }
+
+  /** Whether a table's adapter_ver is still the UInt32 from before harness-sessions-sql v1's types (read again after VER_TYPES_TTL or a failed insert). */
+  async numericVer(table) {
+    if (!this.verTypes || this.clock() - this.verAt > VER_TYPES_TTL) {
+      this.verAt = this.clock();
+      const dbs = [this.P.db, this.P.archive].filter(Boolean).map(lit).join(', ');
+      this.verTypes = new Map((await this.rows(`SELECT database, table, type FROM system.columns WHERE name = 'adapter_ver' AND database IN (${dbs})`))
+        .map((r) => [`${r.database}.${r.table}`, /^UInt/.test(String(r.type))]));
+    }
+    return !!this.verTypes.get(this.P[table]);
+  }
+
+  /** The session's source cursors (FINAL), the legacy marker left out. */
+  async sources(ref) {
+    return this.rows(`SELECT source_key, toString(mode) AS mode, position, generation, toString(progress) AS progress, state, state_ver, toString(adapter) AS adapter,
+        adapter_ver, toString(role) AS role, known, is_deleted
+      FROM ${this.P.session_sources} FINAL WHERE ${storeWhere(ref)} AND role != 'legacy'`);
+  }
+
+  /**
+   * The manifests of one source generation, folded: unit_key -> sha256 in delivery order, and the greatest
+   * progress. `upto`: only the manifests at or below this progress (the generation as a cursor saw it).
+   */
+  async foldOf(ref, sourceKey, generation, { upto = null } = {}) {
+    const where = `${storeWhere(ref)} AND source_key = ${lit(sourceKey)} AND generation = ${Number(generation)}`;
+    const manifests = (await this.rows(`SELECT DISTINCT raw FROM ${this.P.session_units} WHERE ${where} AND unit_key = '' AND enc = 'manifest'`))
+      .map((r) => JSON.parse(r.raw)).filter((m) => upto == null || BigInt(m.progress) <= BigInt(upto));
+    return foldManifests(manifests);
+  }
+
+  /** The units of one source generation, in delivery order (harness-sessions-sql §4.1 replay). */
+  async archiveUnits(ref, sourceKey, generation, { upto = null } = {}) {
+    const { fold } = await this.foldOf(ref, sourceKey, generation, { upto });
+    return this.unitsOf(ref, sourceKey, generation, [...fold]);
+  }
+
+  /** The units `wanted` ([unit_key, sha256] pairs, in the order given) of one source generation, from their chunks. */
+  async unitsOf(ref, sourceKey, generation, wanted) {
+    if (!wanted.length) return [];
+    const where = `${storeWhere(ref)} AND source_key = ${lit(sourceKey)} AND generation = ${Number(generation)}`;
+    // One row per chunk: two copies of a chunk that differ outside the key (an orig_sha256 written as ''
+    // before harness-sessions-sql v1's types and as null after) must not double the body.
+    const by = new Map();
+    for (let i = 0; i < wanted.length; i += 1000) {
+      const tuples = wanted.slice(i, i + 1000).map(([k, sha]) => `(${lit(k)}, ${lit(sha)})`).join(', ');
+      const chunks = await this.rows(`SELECT DISTINCT unit_key, sha256, chunk, toString(enc) AS enc, redacted, orig_sha256, raw FROM ${this.P.session_units}
+        WHERE ${where} AND unit_key != '' AND (unit_key, sha256) IN (${tuples})`);
+      for (const c of chunks) {
+        const k = `${c.unit_key}\u0000${c.sha256}`;
+        if (!by.has(k)) by.set(k, new Map());
+        if (!by.get(k).has(String(c.chunk))) by.get(k).set(String(c.chunk), c);
+      }
+    }
+    const out = [];
+    for (const [k, sha] of wanted) {
+      const cs = [...(by.get(`${k}\u0000${sha}`) || new Map()).values()].sort((a, b) => Number(a.chunk) - Number(b.chunk));
+      if (!cs.length) throw new Error(`archive: ${sourceKey} generation ${generation} lacks unit ${k} (${sha.slice(0, 12)})`);
+      out.push(unitOf(sourceKey, cs));
+    }
+    return out;
+  }
+
+  /**
+   * The archived size (bytes of text, each chunk once) of each of the units `wanted` ([unit_key,
+   * sha256] pairs) of one source generation: what a replay batches its unit reads by.
+   */
+  async unitSizes(ref, sourceKey, generation, wanted) {
+    const where = `${storeWhere(ref)} AND source_key = ${lit(sourceKey)} AND generation = ${Number(generation)}`;
+    const out = new Map();
+    for (let i = 0; i < wanted.length; i += 1000) {
+      const tuples = wanted.slice(i, i + 1000).map(([k, sha]) => `(${lit(k)}, ${lit(sha)})`).join(', ');
+      // One row per chunk (#64: a chunk can be held twice, '' and null orig_sha256).
+      for (const r of await this.rows(`SELECT unit_key, sha256, toString(sum(b)) AS n FROM (
+          SELECT unit_key, sha256, chunk, max(length(raw)) AS b FROM ${this.P.session_units}
+          WHERE ${where} AND unit_key != '' AND (unit_key, sha256) IN (${tuples}) GROUP BY unit_key, sha256, chunk)
+        GROUP BY unit_key, sha256`)) out.set(`${r.unit_key}\u0000${r.sha256}`, Number(r.n));
+    }
+    return out;
+  }
+
+  /**
+   * Every read manifest of the session, each once, with the adapter that wrote it and the archive
+   * write that holds it (`at`, its captured_at; a manifest written again byte for byte counts as the
+   * latest write's): what a replay from the archive alone folds pass by pass (passesOf).
+   */
+  async manifests(ref) {
+    // Aliases that shadow no column (a SELECT alias named like a column would also stand for it in WHERE).
+    const rows = await this.rows(`SELECT source_key, generation, sha256, any(raw) AS m_raw, toString(max(captured_at)) AS m_at, toString(any(adapter)) AS m_adapter
+      FROM ${this.P.session_units} WHERE ${storeWhere(ref)} AND unit_key = '' AND enc = 'manifest' GROUP BY source_key, generation, sha256`);
+    return rows.map((r) => ({ ...JSON.parse(r.m_raw), source_key: r.source_key, generation: Number(r.generation), sha256: r.sha256, at: String(r.m_at), adapter: String(r.m_adapter) }));
+  }
+
+  /**
+   * Per source: the newest generation the archive holds, the greatest progress of its manifests and
+   * their fold. Ahead of the cursor only when a pass stopped between its archive and cursor writes.
+   */
+  async archiveState(ref) {
+    const gens = await this.rows(`SELECT source_key, max(generation) AS g, toString(max(captured_at)) AS at FROM ${this.P.session_units}
+      WHERE ${storeWhere(ref)} AND unit_key = '' AND enc = 'manifest' GROUP BY source_key`);
+    const out = new Map();
+    for (const r of gens) out.set(r.source_key, { gen: Number(r.g), at: String(r.at), ...(await this.foldOf(ref, r.source_key, r.g)) });
+    return out;
+  }
+
+  /** The session's latest archive write (ms since the epoch; 0 for none): its manifests' greatest captured_at. */
+  async archivedAt(ref) {
+    const [r] = await this.rows(`SELECT toString(max(captured_at)) AS at, count() AS n FROM ${this.P.session_units}
+      WHERE ${storeWhere(ref)} AND unit_key = '' AND enc = 'manifest'`);
+    return r && Number(r.n) ? Date.parse(`${String(r.at).replace(' ', 'T')}Z`) : 0;
+  }
+
+  /** The session's live adapter-derived keys per table, with rev and first_seen (a refold reads them once). */
+  async current(ref) {
+    const out = {};
+    for (const table of ENTITY_TABLES) {
+      const extra = table === 'session_parts' ? ', toString(first_seen) AS first_seen' : table === 'calls' ? ', message_id' : '';
+      out[table] = await this.rows(`SELECT ${KEY_COLUMNS[table].map((c) => `toString(${c}) AS ${c}`).join(', ')}, toString(rev) AS rev, adapter_ver, toString(adapter) AS adapter${extra}
+        FROM ${this.P[table]} FINAL WHERE ${storeWhere(ref)} AND removed = 0 AND ${table === 'session_links' ? "owner = 'adapter'" : "adapter != 'legacy'"}`);
+    }
+    const s = await this.rows(`SELECT toString(rev) AS rev, adapter_ver, toString(adapter) AS adapter FROM ${this.P.sessions} FINAL WHERE ${storeWhere(ref)}`);
+    out.sessionRev = s.length ? s[0].rev : null;
+    out.sessionVer = s.length ? String(s[0].adapter_ver) : null; out.sessionAdapter = s.length ? s[0].adapter : null;
+    return out;
+  }
+
+  /**
+   * The session's removed versions this adapter wrote (removed 1), per table, with rev, adapter_ver and
+   * first_seen: an upgrade's re-derive writes again those the new version makes, and tombstones the rest.
+   */
+  async removedOf(ref, adapter) {
+    const out = {};
+    for (const table of ENTITY_TABLES) {
+      const extra = table === 'session_parts' ? ', toString(first_seen) AS first_seen' : table === 'calls' ? ', message_id' : '';
+      out[table] = await this.rows(`SELECT ${KEY_COLUMNS[table].map((c) => `toString(${c}) AS ${c}`).join(', ')}, toString(rev) AS rev, adapter_ver, toString(adapter) AS adapter${extra}
+        FROM ${this.P[table]} FINAL WHERE ${storeWhere(ref)} AND removed = 1 AND adapter = ${lit(adapter)}${table === 'session_links' ? " AND owner = 'adapter'" : ''}`);
+    }
+    return out;
+  }
+
+  /**
+   * The live rows another adapter wrote in this session (a harness that moved to a new adapter): their
+   * keys and what they say, to tell the new adapter's twins (duplicates) from the rest (history).
+   */
+  async foreignContent(ref, adapter) {
+    const own = `${storeWhere(ref)} AND removed = 0 AND adapter != ${lit(adapter)} AND adapter != 'legacy'`;
+    return {
+      session_parts: await this.rows(`SELECT message_id, part_id, toString(role) AS role, toString(kind) AS kind, text, toString(tool_name) AS tool_name, args, output
+        FROM ${this.P.session_parts} FINAL WHERE ${own}`),
+      calls: await this.rows(`SELECT call_id, toString(tool_name) AS tool_name, args FROM ${this.P.calls} FINAL WHERE ${own}`),
+      session_usage: await this.rows(`SELECT usage_id, toString(input) AS input, toString(output) AS output, toString(cache_read) AS cache_read, toString(cache_write) AS cache_write
+        FROM ${this.P.session_usage} FINAL WHERE ${own}`),
+    };
+  }
+
+  // ── the same reads for many sessions at once (a pass's first sight of a group of sessions) ──
+  // One query per table for a group instead of one per session: a capture over a remote house is
+  // bound by round trips. Keyed by storeKeyOf(ref); a session with nothing stored is absent.
+  async manyRows(refs, sql) {
+    const out = new Map();
+    for (let i = 0; i < refs.length; i += 200) {
+      const part = refs.slice(i, i + 200);
+      const inRefs = `${STORE} IN (${part.map((r) => `(${[r.host, r.harness, r.harness_path, r.session_id, r.incarnation || ''].map(lit).join(', ')})`).join(', ')})`;
+      for (const r of await this.rows(sql(inRefs))) {
+        const k = storeKeyOf({ host: r.zo, harness: r.zh, harness_path: r.zp, session_id: r.zs, incarnation: r.zi });
+        if (!out.has(k)) out.set(k, []);
+        out.get(k).push(r);
+      }
+    }
+    return out;
+  }
+  /** sources() for many sessions. */
+  async sourcesMany(refs) {
+    return this.manyRows(refs, (inRefs) => `SELECT ${STORE_OUT}, source_key, toString(mode) AS mode, position, generation, toString(progress) AS progress, state, state_ver,
+        toString(adapter) AS adapter, adapter_ver, toString(role) AS role, known, is_deleted
+      FROM ${this.P.session_sources} FINAL WHERE ${inRefs} AND role != 'legacy'`);
+  }
+  /** archiveState() for many sessions: each source's newest generation, folded. */
+  async archiveStateMany(refs) {
+    const rows = await this.manyRows(refs, (inRefs) => `SELECT ${STORE_OUT}, source_key, toString(generation) AS generation, raw, toString(max(captured_at)) AS m_at FROM ${this.P.session_units}
+      WHERE ${inRefs} AND unit_key = '' AND enc = 'manifest' GROUP BY host, harness, harness_path, session_id, incarnation, source_key, generation, raw`);
+    const out = new Map();
+    for (const [k, list] of rows) {
+      const gens = new Map(); const ats = new Map();
+      for (const r of list) {
+        gens.set(r.source_key, Math.max(gens.has(r.source_key) ? gens.get(r.source_key) : -1, Number(r.generation)));
+        if (!ats.has(r.source_key) || String(r.m_at) > ats.get(r.source_key)) ats.set(r.source_key, String(r.m_at));
+      }
+      const state = new Map();
+      for (const [sk, g] of gens) state.set(sk, { gen: g, at: ats.get(sk), ...foldManifests(list.filter((r) => r.source_key === sk && Number(r.generation) === g).map((r) => JSON.parse(r.raw))) });
+      out.set(k, state);
+    }
+    return out;
+  }
+  /** current() for many sessions. */
+  async currentMany(refs) {
+    const out = new Map();
+    const of = (k) => { if (!out.has(k)) out.set(k, { ...Object.fromEntries(ENTITY_TABLES.map((t) => [t, []])), sessionRev: null, sessionVer: null, sessionAdapter: null }); return out.get(k); };
+    for (const k of refs.map(storeKeyOf)) of(k);
+    for (const table of ENTITY_TABLES) {
+      const extra = table === 'session_parts' ? ', toString(first_seen) AS first_seen' : table === 'calls' ? ', message_id' : '';
+      const rows = await this.manyRows(refs, (inRefs) => `SELECT ${STORE_OUT}, ${KEY_COLUMNS[table].map((c) => `toString(${c}) AS ${c}`).join(', ')}, toString(rev) AS rev, adapter_ver, toString(adapter) AS adapter${extra}
+        FROM ${this.P[table]} FINAL WHERE ${inRefs} AND removed = 0 AND ${table === 'session_links' ? "owner = 'adapter'" : "adapter != 'legacy'"}`);
+      for (const [k, list] of rows) of(k)[table] = list.map(({ zo, zh, zp, zs, zi, ...r }) => r);
+    }
+    const s = await this.manyRows(refs, (inRefs) => `SELECT ${STORE_OUT}, toString(rev) AS rev, adapter_ver, toString(adapter) AS adapter FROM ${this.P.sessions} FINAL WHERE ${inRefs}`);
+    for (const [k, list] of s) Object.assign(of(k), { sessionRev: list[0].rev, sessionVer: String(list[0].adapter_ver), sessionAdapter: list[0].adapter });
+    return out;
+  }
+
+  /**
+   * Retire these keys' live rows at `rev`, server-side: a removed version (removed 1: the key left its
+   * source), or with `tombstone` a tombstone (is_deleted 1: a key an upgraded adapter no longer
+   * produces, harness-sessions-sql §2 principle 3). With `removed`, the keys' removed versions are the rows
+   * tombstoned (an upgrade's re-derive).
+   */
+  async retire(table, ref, keys, rev, { tombstone = false, removed = false } = {}) {
+    if (removed && !tombstone) throw new Error('retire: a removed version is only ever tombstoned');
+    const cols = KEY_COLUMNS[table];
+    const mark = tombstone ? 'toUInt8(1) AS is_deleted' : 'toUInt8(1) AS removed';
+    for (let i = 0; i < keys.length; i += 500) {
+      const tuples = keys.slice(i, i + 500).map((k) => `(${k.split('\u0000').map(lit).join(', ')})`).join(', ');
+      // The filter sits in a subquery: ClickHouse resolves a SELECT alias in WHERE, so `1 AS removed`
+      // beside `WHERE removed = 0` would select nothing.
+      await this.run(`INSERT INTO ${this.P[table]} SELECT * REPLACE (toUInt64(${BigInt(rev)}) AS rev, ${mark}, now64(3) AS ingested_at)
+        FROM (SELECT * FROM ${this.P[table]} FINAL WHERE ${storeWhere(ref)} AND removed = ${removed ? 1 : 0} AND (${cols.join(', ')}) IN (${tuples}))`);
+    }
+  }
+
+  /**
+   * Write the session's sessions row: `row` holds the spec columns and memhouse's own (folder,
+   * project, origin), the shared rollup the derived ones. Run after the rows it should count.
+   */
+  async writeSession(ref, row) {
+    const [r] = await this.rows(`SELECT ${rollup.ROLLUP_COLUMNS.map(([c, e]) => `${e} AS ${c}`).join(', ')}
+      FROM (${rollup.sessionRollupSQL(this.P, storeWhere(ref))}) AS r`);
+    // Another writer may have written the row above the pass's rev (the dedupe's refresh writes
+    // rev + 1): the row goes above it, never at the same rev with other content.
+    const [h] = await this.rows(`SELECT toString(max(rev)) AS rev FROM ${this.P.sessions} WHERE ${storeWhere(ref)}`);
+    let rev = BigInt(row.rev);
+    if (h && h.rev != null && BigInt(h.rev) >= rev) rev = BigInt(h.rev) + 1n;
+    await this.insert('sessions', [{ ...row, ...(r || {}), rev: String(rev) }]);
+    return rev;
+  }
+
+  /** Write the session's live sessions row again with its rollup recomputed (a fork whose edge moved). */
+  async refreshSession(ref) {
+    await this.run(rollup.refreshSessionsSQL(this.P, storeWhere(ref)));
+  }
+}
+
+/** A version string as the UInt32 from before harness-sessions-sql v1's types (adapter-capture verNumber); a plain number stays itself. */
+function oldVer(v) {
+  const s = String(v ?? '');
+  if (/^\d+$/.test(s)) return Number(s);
+  const m = /^(\d+)\.(\d+)\.(\d+)/.exec(s);
+  return m ? 1e9 + Number(m[1]) * 1e6 + Number(m[2]) * 1e3 + Number(m[3]) : 0;
+}
+
+/**
+ * Manifests folded in progress order (harness-sessions-sql §4.1): each adds its units and drops its gone
+ * keys, and a reset starts over (a reset opens its generation; a pass that stopped before its cursor
+ * write left one ahead, which the next pass's reset supersedes).
+ */
+function foldManifests(manifests) {
+  const sorted = [...manifests].sort((a, b) => (BigInt(a.progress) < BigInt(b.progress) ? -1 : BigInt(a.progress) > BigInt(b.progress) ? 1 : 0));
+  const fold = new Map();
+  let progress = null;
+  let gone = false;   // the newest manifest says the source vanished (its generation holds no read)
+  for (const m of sorted) {
+    if (m.continuity === 'reset') fold.clear();
+    for (const [k, sha] of m.units || []) { fold.delete(k); fold.set(k, sha); }
+    for (const k of m.gone || []) fold.delete(k);
+    progress = String(m.progress);
+    gone = !!m.source_gone;
+  }
+  return { fold, progress, gone };
+}
+
+const cmpProgress = (a, b) => { const x = BigInt(a.progress ?? 0); const y = BigInt(b.progress ?? 0); return x < y ? -1 : x > y ? 1 : 0; };
+
+/**
+ * A session's manifests (ProfileHouse manifests()) as the passes that wrote them, in pass order
+ * (harness-sessions-sql §4.1: a replay derives once per pass). memhouse's `progress` counts per source (a
+ * stream's byte offset, else the source's reads), so it orders one source's manifests but does not
+ * group a session's: a pass is the manifests of one archive write, which share its `at`. Along each
+ * source, in (generation, progress) order, a write that reads as earlier than the one before it is
+ * taken as that one's, so no pass folds a source's later read before an earlier one. Each pass
+ * lists its manifests by source, each source's in (generation, progress) order.
+ */
+function passesOf(manifests) {
+  const bySource = new Map();
+  for (const m of manifests) {
+    if (!bySource.has(m.source_key)) bySource.set(m.source_key, []);
+    bySource.get(m.source_key).push(m);
+  }
+  const passes = new Map();
+  for (const list of bySource.values()) {
+    list.sort((a, b) => a.generation - b.generation || cmpProgress(a, b));
+    let at = '';
+    for (const m of list) {
+      if (String(m.at) > at) at = String(m.at);
+      if (!passes.has(at)) passes.set(at, []);
+      passes.get(at).push(m);
+    }
+  }
+  const order = (a, b) => (a.source_key < b.source_key ? -1 : a.source_key > b.source_key ? 1 : a.generation - b.generation || cmpProgress(a, b));
+  return [...passes.keys()].sort().map((at) => passes.get(at).sort(order));
+}
+
+/**
+ * One manifest folded into its source's replay state (harness-sessions-sql §4.1 "Replay from the archive
+ * alone"). `sources`: source_key -> { generation, units (unit_key -> sha256, delivery order), from
+ * (the generation whose rows hold those units; null when none do), gone (null, or the policy of the
+ * source_gone generation it is in), progress }. A source_gone manifest replays as its recorded policy
+ * says: `kept` keeps the generation before it current, `removed` (and no policy, an archive from
+ * before policy existed) leaves the source empty. A reset, or a generation's first manifest, starts
+ * the source over; any other adds its units and drops its gone keys.
+ */
+function foldManifest(sources, m) {
+  const s = sources.get(m.source_key);
+  if (m.source_gone) {
+    const policy = m.policy == null ? 'removed' : String(m.policy);
+    const kept = policy === 'kept' && s;
+    sources.set(m.source_key, { generation: m.generation, units: kept ? s.units : new Map(), from: kept ? s.from : null, gone: policy, progress: String(m.progress) });
+    return;
+  }
+  const units = m.continuity === 'reset' || !s || s.generation !== m.generation ? new Map() : s.units;
+  for (const [k, sha] of m.units || []) { units.delete(String(k)); units.set(String(k), sha); }
+  for (const k of m.gone || []) units.delete(String(k));
+  sources.set(m.source_key, { generation: m.generation, units, from: m.generation, gone: null, progress: String(m.progress) });
+}
+
+/** One unit back from its archive chunks. */
+function unitOf(sourceKey, cs) {
+  const body = cs.map((c) => c.raw).join('');
+  const enc = cs[0].enc;
+  const u = { source_key: sourceKey, unit_key: cs[0].unit_key, enc, bytes: enc === 'b64' ? Buffer.from(body, 'base64').length : Buffer.byteLength(body, 'utf8'),
+    sha256: cs[0].sha256, redacted: !!cs[0].redacted, body };
+  if (cs[0].orig_sha256 && /^[0-9a-f]{64}$/.test(cs[0].orig_sha256)) u.orig_sha256 = cs[0].orig_sha256;
+  return u;
+}
+
+const CHUNK = 1 << 20;
+/** A body in chunks of at most CHUNK UTF-16 units, never splitting a surrogate pair. */
+function chunksOf(body) {
+  const s = String(body ?? '');
+  const out = [];
+  for (let i = 0; i < s.length || !out.length;) {
+    let end = Math.min(i + CHUNK, s.length);
+    if (end < s.length && /[\uD800-\uDBFF]/.test(s[end - 1])) end--;
+    out.push(s.slice(i, end));
+    i = end;
+    if (i >= s.length) break;
+  }
+  return out;
+}
+
+module.exports = { ProfileHouse, KEY_COLUMNS, ENTITY_TABLES, keyOf, storeWhere, storeKeyOf, chunksOf, unitOf, foldManifests, passesOf, foldManifest, oldVer, CHUNK };

package/memhouse/house/profile-rollup.js

@@ -0,0 +1,190 @@
+// memhouse's derived session columns over harness-sessions-sql's profile tables (step B4: the shared
+// rollup). One SQL builder for every writer of a sessions row, the conversion and the capture
+// engine alike, so a session reads the same whichever wrote it.
+//
+// THE READER RULES (each line is a decision)
+//   live rows        FINAL (tombstones drop out), removed = 0; parts whose state (their own
+//                    part_state where they have one, else their message's) is 'reverted' or
+//                    'retracted' are off the session's line and count nowhere
+//   tokens           the house split (sessions-model §2.4, as records-to-rows houseTokens): input
+//                    without cached input (cache_read and cache_write taken out where
+//                    input_includes_cache is true) plus tool_prompt; output with reasoning (added
+//                    where output_includes_reasoning is false); a null flag leaves the number as given
+//   per_record       a response's usage counts once, from its last record: the response-anchored
+//                    row with the greatest n in its usage_id '<anchor>#<n>' (harness-sessions-sql §4.4)
+//   billed once      a fork's usage rows that copy its origin's count only in the origin: the ones a
+//                    verified engine edge on the fork lists in r_copied_usage (profile-edges.js;
+//                    harness-sessions-sql §4.5, billing reads edges only)
+//   messages         text parts; user/assistant by role; agent_messages those of a sub-agent
+//   chars            text_len of the role's parts (a tool result is role 'tool': in neither)
+//   first_prompt     the main agent's first user text part by order_key, 200 characters
+//   started, ended   the parts' first and last ts
+//   models           served model, else requested; '' and '<synthetic>' left out. dominant_model
+//                    the one with the most parts
+//   days             the days of the parts and the calls
+//   usage            per hour × model: text parts, and the usage rows' house tokens by their ts
+//   tools            per tool name from {m}_calls: calls, errors (is_error true)
+//   agents           per sub-agent: type, description and workflow from its parts' extra.agent,
+//                    text parts, first ts
+//   folder, project  the first part's (memhouse's columns on session_parts)
+//
+// A refresh (refreshSessionsSQL) writes a live sessions row again at rev + 1 with these columns
+// recomputed and its own columns as they were: the rollup is memhouse's, outside content_sha's line.
+// A key with no live part or usage row left is retired (is_deleted 1).
+
+'use strict';
+
+const sp = require('./schema-profile');
+
+const KEY = 'host, harness, harness_path, session_id, incarnation';
+
+/**
+ * One row per session key matching `where` (a predicate over harness and session_id), with
+ * memhouse's derived columns as r_*. Reads session_parts, session_usage and calls with FINAL.
+ * A part or call with no time of its own counts at the session's earliest time, or at its first_seen
+ * when the session has none at all (a harness that keeps no times); a usage record with none at its
+ * anchor's message or response, else at the session's latest part: never at 1970, and a dated
+ * session's start, end and days stay its own.
+ */
+function sessionRollupSQL(P, where = '1') {
+  const parts = `SELECT host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation,
+      agent_id AS aid, ifNull(ts, ifNull(min(ts) OVER (PARTITION BY host, harness, harness_path, session_id, incarnation), first_seen)) AS t,
+      message_id AS mid, response_id AS rid, order_key AS ok, toString(role) AS rl, toString(kind) AS kd,
+      if(toString(served_model) != '', toString(served_model), toString(requested_model)) AS md,
+      text AS tx, text_len AS tlen, folder AS fo, project AS pr, extra AS ex
+    FROM ${P.session_parts} FINAL
+    WHERE removed = 0 AND if(part_state != '', part_state, state) NOT IN ('reverted', 'retracted') AND ${where}`;
+  // A per_record response's records: only its last one (the greatest n) counts. A fork's copies of its
+  // origin's usage, listed on its verified engine edges, do not: left out after the last record is
+  // chosen, so a copied last record never lets an earlier one count in its place.
+  const copied = `SELECT host, toString(harness), toString(harness_path), session_id, incarnation, arrayJoin(r_copied_usage)
+      FROM ${P.session_links} FINAL WHERE owner = 'engine' AND removed = 0 AND ifNull(verified, false) AND ${where}`;
+  const usage = `SELECT host, harness, harness_path, session_id, incarnation, t, md, ak, an,
+      toUInt64(greatest(if(ifNull(iic, false), toInt64(i0) - toInt64(cr0) - toInt64(cw0), toInt64(i0)), 0) + tp0) AS ui,
+      o0 + if(isNotNull(oir) AND NOT oir, rs0, 0) AS uo, cr0 AS ucr, cw0 AS ucw, rs0 AS urs
+    FROM (
+      SELECT host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation, ts AS t, toString(model) AS md,
+        toString(granularity) AS g, toString(anchor_kind) AS ak, anchor_id AS an, usage_id AS uid, toUInt64OrZero(splitByChar('#', usage_id)[-1]) AS un,
+        max(un) OVER (PARTITION BY host, harness, harness_path, session_id, incarnation, ak, an) AS umax,
+        ifNull(input, 0) AS i0, ifNull(output, 0) AS o0, ifNull(cache_read, 0) AS cr0, ifNull(cache_write, 0) AS cw0,
+        ifNull(reasoning, 0) AS rs0, ifNull(tool_prompt, 0) AS tp0, input_includes_cache AS iic, output_includes_reasoning AS oir
+      FROM ${P.session_usage} FINAL WHERE removed = 0 AND ${where}
+    ) WHERE (NOT (g = 'per_record' AND ak = 'response') OR un = umax)
+      AND (host, harness, harness_path, session_id, incarnation, uid) NOT IN (${copied})`;
+  // A call with no time takes the session's earliest part time, else the earliest call time, else its first_seen.
+  const calls = `SELECT c.host AS host, c.harness AS harness, c.harness_path AS harness_path, c.session_id AS session_id, c.incarnation AS incarnation,
+      c.tn AS tn, c.ce AS ce, coalesce(c.ts, pm.pmin, c.cmin, c.first_seen) AS t
+    FROM (SELECT host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation,
+        toString(tool_name) AS tn, ifNull(is_error, false) AS ce, ts, first_seen,
+        min(ts) OVER (PARTITION BY host, harness, harness_path, session_id, incarnation) AS cmin
+      FROM ${P.calls} FINAL WHERE removed = 0 AND ${where}) AS c
+    LEFT JOIN (SELECT host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation, CAST(min(ts) AS Nullable(DateTime64(3, 'UTC'))) AS pmin
+      FROM ${P.session_parts} FINAL WHERE removed = 0 AND ${where} GROUP BY host, harness, harness_path, session_id, incarnation) AS pm
+      ON pm.host = c.host AND pm.harness = c.harness AND pm.harness_path = c.harness_path AND pm.session_id = c.session_id AND pm.incarnation = c.incarnation`;
+  // The stores with anything live: a part in any state (one a revert hides is still the session's),
+  // a usage record or a call. A store with none of them is retired by the refresh; any other stays.
+  const present = `SELECT DISTINCT host, toString(harness) AS harness, toString(harness_path) AS harness_path, session_id, incarnation
+      FROM ${P.session_parts} FINAL WHERE removed = 0 AND ${where}
+    UNION DISTINCT SELECT DISTINCT ${KEY} FROM (${usage})
+    UNION DISTINCT SELECT DISTINCT ${KEY} FROM (${calls})`;
+  const hour = (t) => `toStartOfHour(toDateTime(ifNull(${t}, toDateTime64(0, 3, 'UTC')), 'UTC'))`;
+  const on = (a) => KEY.split(', ').map((c) => `${a}.${c} = k.${c}`).join(' AND ');
+  return `SELECT k.host AS host, k.harness AS harness, k.harness_path AS harness_path, k.session_id AS session_id, k.incarnation AS incarnation,
+      coalesce(a.r_started, tl.tl_first, u.u_first) AS r_started, coalesce(a.r_ended, tl.tl_last, u.u_last) AS r_ended, a.r_messages, a.r_user_messages, a.r_assistant_messages, a.r_agent_messages,
+      toUInt32(ifNull(tl.tl_calls, 0)) AS r_tool_calls,
+      ifNull(u.u_in, 0) AS r_in, ifNull(u.u_out, 0) AS r_out, ifNull(u.u_cr, 0) AS r_cr, ifNull(u.u_cw, 0) AS r_cw, ifNull(u.u_rs, 0) AS r_rs,
+      a.r_user_chars, a.r_assistant_chars, a.r_first_prompt, a.r_models, dm.dm_model AS r_dominant,
+      arraySort(arrayDistinct(arrayConcat(a.r_days, ifNull(tl.tl_days, [])))) AS r_days,
+      ifNull(uh.uh_arr, []) AS r_usage, ifNull(tl.tl_arr, []) AS r_tools, ifNull(ag.ag_arr, []) AS r_agents,
+      a.r_folder, a.r_project, toUInt8(1) AS r_live
+    FROM (${present}) AS k
+    LEFT JOIN (
+      SELECT ${KEY}, CAST(min(t) AS Nullable(DateTime64(3, 'UTC'))) AS r_started, CAST(max(t) AS Nullable(DateTime64(3, 'UTC'))) AS r_ended,
+        toUInt32(countIf(kd = 'text')) AS r_messages, toUInt32(countIf(kd = 'text' AND rl = 'user')) AS r_user_messages,
+        toUInt32(countIf(kd = 'text' AND rl = 'assistant')) AS r_assistant_messages, toUInt32(countIf(kd = 'text' AND aid != '')) AS r_agent_messages,
+        sumIf(toUInt64(tlen), rl = 'user') AS r_user_chars, sumIf(toUInt64(tlen), rl = 'assistant') AS r_assistant_chars,
+        substringUTF8(argMinIf(tx, ok, rl = 'user' AND aid = '' AND kd = 'text' AND tx != ''), 1, 200) AS r_first_prompt,
+        arraySort(groupUniqArrayIf(md, md NOT IN ('', '<synthetic>'))) AS r_models,
+        arraySort(groupUniqArray(toDate(t))) AS r_days, argMin(fo, ok) AS r_folder, argMin(pr, ok) AS r_project
+      FROM (${parts}) GROUP BY ${KEY}
+    ) AS a ON ${on('a')}
+    LEFT JOIN (
+      SELECT ${KEY}, sum(ui) AS u_in, sum(uo) AS u_out, sum(ucr) AS u_cr, sum(ucw) AS u_cw, sum(urs) AS u_rs,
+        CAST(min(t) AS Nullable(DateTime64(3, 'UTC'))) AS u_first, CAST(max(t) AS Nullable(DateTime64(3, 'UTC'))) AS u_last FROM (${usage}) GROUP BY ${KEY}
+    ) AS u ON ${on('u')}
+    LEFT JOIN (
+      SELECT ${KEY}, arraySort(x -> (x.1, x.2), groupArray((hr, md, toUInt32(hm), hi, ho, hcr, hcw, hrs))) AS uh_arr
+      FROM (
+        SELECT ${KEY}, hr, md, sum(m) AS hm, sum(i) AS hi, sum(o) AS ho, sum(cr) AS hcr, sum(cw) AS hcw, sum(rs) AS hrs FROM (
+          SELECT ${KEY}, ${hour('t')} AS hr, md, toUInt64(kd = 'text') AS m, toUInt64(0) AS i, toUInt64(0) AS o, toUInt64(0) AS cr, toUInt64(0) AS cw, toUInt64(0) AS rs FROM (${parts})
+          UNION ALL
+          SELECT ${KEY.split(', ').map((c) => `uu.${c} AS ${c}`).join(', ')}, ${hour('coalesce(uu.t, am.mt, ar.rt, pm.pt)')} AS hr, uu.md AS md, toUInt64(0), uu.ui, uu.uo, uu.ucr, uu.ucw, uu.urs
+          FROM (${usage}) AS uu
+          LEFT JOIN (SELECT ${KEY}, mid, CAST(min(t) AS Nullable(DateTime64(3, 'UTC'))) AS mt FROM (${parts}) GROUP BY ${KEY}, mid) AS am
+            ON ${KEY.split(', ').map((c) => `am.${c} = uu.${c}`).join(' AND ')} AND am.mid = if(uu.ak = 'message', uu.an, '')
+          LEFT JOIN (SELECT ${KEY}, rid, CAST(min(t) AS Nullable(DateTime64(3, 'UTC'))) AS rt FROM (${parts}) WHERE rid != '' GROUP BY ${KEY}, rid) AS ar
+            ON ${KEY.split(', ').map((c) => `ar.${c} = uu.${c}`).join(' AND ')} AND ar.rid = if(uu.ak = 'response', uu.an, '')
+          LEFT JOIN (SELECT ${KEY}, CAST(max(t) AS Nullable(DateTime64(3, 'UTC'))) AS pt FROM (${parts}) GROUP BY ${KEY}) AS pm
+            ON ${KEY.split(', ').map((c) => `pm.${c} = uu.${c}`).join(' AND ')}
+        ) GROUP BY ${KEY}, hr, md
+      ) GROUP BY ${KEY}
+    ) AS uh ON ${on('uh')}
+    LEFT JOIN (
+      SELECT ${KEY}, arraySort(x -> x.1, groupArray((tn, n_calls, n_errors))) AS tl_arr, sum(n_calls) AS tl_calls, arraySort(arrayDistinct(groupArrayArray(td))) AS tl_days,
+        CAST(min(tmin) AS Nullable(DateTime64(3, 'UTC'))) AS tl_first, CAST(max(tmax) AS Nullable(DateTime64(3, 'UTC'))) AS tl_last
+      FROM (SELECT ${KEY}, tn, toUInt32(count()) AS n_calls, toUInt32(countIf(ce)) AS n_errors, groupUniqArray(toDate(t)) AS td, min(t) AS tmin, max(t) AS tmax
+        FROM (${calls}) GROUP BY ${KEY}, tn)
+      GROUP BY ${KEY}
+    ) AS tl ON ${on('tl')}
+    LEFT JOIN (
+      SELECT ${KEY}, arraySort(x -> (x.6, x.1), groupArray((aid, g_type, g_desc, g_wf, g_msgs, g_first))) AS ag_arr
+      FROM (
+        SELECT ${KEY}, aid, argMin(JSONExtractString(ex, 'agent', 'type'), ok) AS g_type, argMin(JSONExtractString(ex, 'agent', 'description'), ok) AS g_desc,
+          argMin(JSONExtractString(ex, 'agent', 'workflow'), ok) AS g_wf, toUInt32(countIf(kd = 'text')) AS g_msgs,
+          ifNull(min(t), toDateTime64(0, 3, 'UTC')) AS g_first
+        FROM (${parts}) WHERE aid != '' GROUP BY ${KEY}, aid
+      ) GROUP BY ${KEY}
+    ) AS ag ON ${on('ag')}
+    LEFT JOIN (
+      SELECT ${KEY}, argMax(md, (c, md)) AS dm_model FROM (SELECT ${KEY}, md, count() AS c FROM (${parts}) WHERE md NOT IN ('', '<synthetic>') GROUP BY ${KEY}, md) GROUP BY ${KEY}
+    ) AS dm ON ${on('dm')}`;
+}
+
+/** The sessions columns the rollup fills, with the r_* expression each takes (Nested as arrays). */
+const ROLLUP_COLUMNS = [
+  ['started', 'r.r_started'], ['ended', 'r.r_ended'], ['messages', 'r.r_messages'], ['user_messages', 'r.r_user_messages'],
+  ['assistant_messages', 'r.r_assistant_messages'], ['agent_messages', 'r.r_agent_messages'], ['tool_calls', 'r.r_tool_calls'],
+  ['input_tokens', 'r.r_in'], ['output_tokens', 'r.r_out'], ['cache_read_tokens', 'r.r_cr'], ['cache_write_tokens', 'r.r_cw'], ['reasoning_tokens', 'r.r_rs'],
+  ['user_chars', 'r.r_user_chars'], ['assistant_chars', 'r.r_assistant_chars'], ['first_prompt', 'r.r_first_prompt'], ['models', 'r.r_models'],
+  ['dominant_model', 'r.r_dominant'], ['days', 'r.r_days'],
+  ['`usage.hour`', 'arrayMap(x -> x.1, r.r_usage)'], ['`usage.model`', 'arrayMap(x -> x.2, r.r_usage)'], ['`usage.messages`', 'arrayMap(x -> x.3, r.r_usage)'],
+  ['`usage.input`', 'arrayMap(x -> x.4, r.r_usage)'], ['`usage.output`', 'arrayMap(x -> x.5, r.r_usage)'], ['`usage.cache_read`', 'arrayMap(x -> x.6, r.r_usage)'],
+  ['`usage.cache_write`', 'arrayMap(x -> x.7, r.r_usage)'], ['`usage.reasoning`', 'arrayMap(x -> x.8, r.r_usage)'],
+  ['`tools.name`', 'arrayMap(x -> x.1, r.r_tools)'], ['`tools.calls`', 'arrayMap(x -> x.2, r.r_tools)'], ['`tools.errors`', 'arrayMap(x -> x.3, r.r_tools)'],
+  ['`agents.agent_id`', 'arrayMap(x -> x.1, r.r_agents)'], ['`agents.type`', 'arrayMap(x -> x.2, r.r_agents)'], ['`agents.description`', 'arrayMap(x -> x.3, r.r_agents)'],
+  ['`agents.workflow`', 'arrayMap(x -> x.4, r.r_agents)'], ['`agents.messages`', 'arrayMap(x -> x.5, r.r_agents)'], ['`agents.first_ts`', 'arrayMap(x -> x.6, r.r_agents)'],
+];
+
+/**
+ * Write the live sessions rows matching `where` (a predicate over the session key) again: rev + 1,
+ * the rollup columns recomputed, every other column as stored. Run after the parts, usage and calls
+ * writes it should see.
+ */
+function refreshSessionsSQL(P, where) {
+  const roll = new Map(ROLLUP_COLUMNS.map(([c, e]) => [c.replace(/`/g, ''), e]));
+  const cols = sp.insertColumns(sp.readTemplate(), 'sessions');
+  const q = (c) => `\`${c}\``;
+  const sel = cols.map((c) => {
+    if (roll.has(c)) return `${roll.get(c)} AS ${q(c)}`;
+    if (c === 'rev') return 's.rev + 1 AS rev';
+    if (c === 'is_deleted') return 'toUInt8(ifNull(r.r_live, 0) = 0) AS is_deleted';
+    return `s.${q(c)} AS ${q(c)}`;
+  });
+  return `INSERT INTO ${P.sessions} (${cols.map(q).join(', ')})
+    SELECT ${sel.join(', ')}
+    FROM (SELECT * FROM ${P.sessions} FINAL WHERE ${where}) AS s
+    LEFT JOIN (${sessionRollupSQL(P, where)}) AS r
+      ON ${KEY.split(', ').map((c) => `r.${c} = s.${c}`).join(' AND ')}`;
+}
+
+module.exports = { sessionRollupSQL, refreshSessionsSQL, ROLLUP_COLUMNS, KEY };