memhouse 0.9.0 → 0.11.1

package/memhouse/shipper/ship.js

@@ -14,11 +14,18 @@
 //   - DateTime64 values travel as 'YYYY-MM-DD HH:MM:SS.mmm' UTC strings (plain
 //     format; ISO 'T'/'Z' forms parse unreliably under JSONEachRow). Nullable → null.
 //   - Int64-bound values are integer-coerced (some adapters emit fractional ms).
+//   - INSERT-ONLY. The shipper never deletes and never mutates. A re-parse that shrinks
+//     or diverges from what is stored is written under a NEW epoch, so the superseded
+//     parse survives intact — Claude Code compacts transcripts and deletes them after
+//     cleanupPeriodDays (30 by default), which makes the house the only remaining copy.
+//     memhouse is an accumulator, not a mirror: absence of a session is never a signal,
+//     and no "sync" or "prune" feature may ever be built here.
 //   - Re-shipping is always safe: ReplacingMergeTree(ingested_at) collapses to
-//     latest-wins at FINAL. Keyed (session_id, user_id, origin, seq) on messages and
-//     (session_id, user_id, origin, idx) on tool_calls, so an imported row and a shipped
-//     one at the same seq are two rows, not one. sessions is (session_id, user_id) with
-//     NO origin — one metadata row per session is what every read path assumes.
+//     latest-wins at FINAL. Keyed (session_id, user_id, origin, epoch, seq) on messages
+//     and (…, epoch, idx) on tool_calls, so an imported row and a shipped one at the same
+//     seq are two rows, and so are two parses of the same session. sessions is
+//     (session_id, user_id) with NO origin and NO epoch — one metadata row per session is
+//     what every read path assumes.
 //
 // CLI:  node ship.js                one incremental pass
 //       node ship.js --loop [sec]   repeat every sec seconds (default 300)
@@ -52,7 +59,14 @@ const selfUpdate = require('../self-update');
 // Captured at require time, which is as close to "what this process booted with" as it
 // gets. Taking it later would record whatever an upgrade had already replaced.
 const selfSnap = selfUpdate.snapshot(__filename);
-const { resolveRooms, READ_SETTINGS, ROOM_TYPES } = require('../house/house');
+const {
+  resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN,
+  SUPPORTED_SCHEMAS, keyProblem,
+} = require('../house/house');
+// Rooms plus the house's own record of itself. Every table the template declares, which
+// is what the column healer and the drift warning have to cover — a column added to
+// house_events would otherwise roll out to nobody.
+const ALL_TABLES = [...ROOM_TYPES, ...META_TYPES];
 
 const BATCH_ROWS = 2000;   // insert batch ceiling (binding)
 const TEXT_MAX = 50000;    // messages.text truncation
@@ -263,11 +277,19 @@ function makeClient() {
  *
  * MATERIALIZED and DEFAULT clauses are kept: `user_id String MATERIALIZED currentUser()`
  * has to be added exactly that way or the identity stamp does not happen.
+ *
+ * The room name in this pattern is the LITERAL table name. It used to be
+ * `<type>_{{MEMBER}}`, from the per-member layout, and nobody updated it when the rooms
+ * became plain shared tables — so it matched nothing, and every consumer silently got an
+ * empty column list. That took out three surfaces at once, all of them reporting success:
+ * the column healer in ensureSchema had nothing to add, warnMissingColumns had nothing to
+ * warn about, and doctor's column check printed "✓ columns: every room matches the schema
+ * template" having compared zero columns. A regression test now pins it (misc/unit-test.js).
  */
 function templateColumns(tpl) {
   const out = {};
-  for (const t of ROOM_TYPES) {
-    const re = new RegExp(`CREATE TABLE IF NOT EXISTS ${t}_\\{\\{MEMBER\\}\\}\\s*\\(([\\s\\S]*?)\\n\\)`, 'm');
+  for (const t of ALL_TABLES) {
+    const re = new RegExp(`CREATE TABLE IF NOT EXISTS ${t}\\s*\\(([\\s\\S]*?)\\n\\)`, 'm');
     const m = tpl.match(re);
     if (!m) continue;
     const cols = [];
@@ -308,15 +330,15 @@ async function warnMissingColumns(client, rooms) {
   catch { return; }
   const want = templateColumns(tpl);
   const missing = [];
-  for (const t of ROOM_TYPES) {
+  for (const t of ALL_TABLES) {
     try {
       const rs = await client.query({
         query: 'SELECT name FROM system.columns WHERE database = currentDatabase() AND table = {n:String}',
-        query_params: { n: rooms[t] }, format: 'JSONEachRow',
+        query_params: { n: rooms[`${t}_raw`] }, format: 'JSONEachRow',
       });
       const have = new Set((await rs.json()).map((r) => r.name));
       if (!have.size) continue;
-      for (const c of (want[t] || [])) if (!have.has(c.name)) missing.push(`${rooms[t]}.${c.name}`);
+      for (const c of (want[t] || [])) if (!have.has(c.name)) missing.push(`${rooms[`${t}_raw`]}.${c.name}`);
     } catch { /* unreadable rooms are assertRoomsExist's problem */ }
   }
   if (missing.length) {
@@ -329,11 +351,11 @@ async function warnMissingColumns(client, rooms) {
 async function assertRoomsExist(client, rooms) {
   const rs = await client.query({
     query: `SELECT name FROM system.tables WHERE database = currentDatabase() AND name IN ({n:Array(String)})`,
-    query_params: { n: ROOM_TYPES.map((t) => rooms[t]) },
+    query_params: { n: ROOM_TYPES.map((t) => rooms[`${t}_raw`]) },
     format: 'JSONEachRow',
   });
   const have = new Set((await rs.json()).map((r) => r.name));
-  const missing = ROOM_TYPES.map((t) => rooms[t]).filter((n) => !have.has(n));
+  const missing = ROOM_TYPES.map((t) => rooms[`${t}_raw`]).filter((n) => !have.has(n));
   if (!missing.length) return;
   throw new Error(
     `'${rooms.member}' has no ${missing.join(', ')} in this house.\n`
@@ -343,46 +365,47 @@ async function assertRoomsExist(client, rooms) {
     + '  A house from before 0.4 reaches this too — its rooms have different names.');
 }
 
-async function assertOriginKeyed(client, rooms) {
+/**
+ * Refuse to write into rooms whose sorting key is not the one the shipper's safety
+ * depends on. What each column buys is written out on ROOM_KEYS in ../house/house.js;
+ * this is the enforcement, run at the top of EVERY pass rather than only in
+ * `--ensure-schema`, because a plain `memhouse ship` never calls ensureSchema and a guard
+ * living only there is a guard that never runs on the path that does the damage.
+ */
+async function assertRoomKeys(client, rooms) {
   const wrong = [];
   for (const t of ROOM_TYPES) {
     const rs = await client.query({
       query: `SELECT sorting_key FROM system.tables WHERE database = currentDatabase() AND name = {n:String}`,
-      query_params: { n: rooms[t] }, format: 'JSONEachRow',
+      query_params: { n: rooms[`${t}_raw`] }, format: 'JSONEachRow',
     });
     const row = (await rs.json())[0];
-    const key = (row || {}).sorting_key || '';
-    // A room that EXISTS but has no sorting key is not a MergeTree — a Merge, a View, a Log
-    // engine standing where the shipper expects to DELETE and INSERT. Skipping it here let
-    // the pass sail past its own guard and die later on
-    // `DELETE query is not supported for table …`, which is exactly what this refusal is
-    // meant to prevent. (A room that does not exist at all is assertRoomsExist's job and is
-    // reported there.)
-    if (row && !key) { wrong.push(`${rooms[t]} — not a MergeTree, so it cannot be shipped to`); continue; }
-    if (!key) continue;
-    const keyed = /\borigin\b/.test(key);
-    // sessions is one row per session and must NOT key on origin; the transcript rooms
-    // hold many rows per session and must.
-    const want = t !== 'sessions';
-    if (keyed !== want) {
-      wrong.push(`${rooms[t]} (${key}) — origin ${want ? 'missing from' : 'must not be in'} the key`);
-    }
+    // A room that does not exist at all is assertRoomsExist's job and is reported there.
+    if (!row) continue;
+    const problem = keyProblem(t, row.sorting_key);
+    if (problem) wrong.push(`${rooms[`${t}_raw`]} ${problem}`);
   }
   if (wrong.length) {
     throw new Error(
-      'these rooms have the wrong sorting key, so a ship pass would corrupt them:\n'
+      'this house needs a one-time migration before it can be shipped to. Run:\n'
+      + '\n    memhouse migrate\n'
+      + '\n  (an upgrade from a pre-0.10 memhouse cannot prompt for this itself — the old\n'
+      + '  `update` command predates migrations, so run the line above once, by hand.)\n'
+      + '\n  Why: these rooms have the wrong sorting key, so a ship pass would corrupt them:\n'
       + wrong.map((s) => `    ${s}`).join('\n')
-      + '\n  ORDER BY cannot be altered in place. Rebuild each room, then re-ship:'
-      + '\n    RENAME TABLE <room> TO <room>_old;'
-      + `\n    -- recreate from ${path.join(__dirname, '..', 'house', 'schema.sql.tpl')}`
-      + '\n    INSERT INTO <room> SELECT * FROM <room>_old;'
-      + '\n  (add `, \'ship\' AS origin` to the SELECT only if <room>_old has no origin column)'
-      + '\n  A house with no imported rows can also just be re-shipped from scratch.');
+      + '\n  ORDER BY cannot be altered in place, so migrate copies each room into one with\n'
+      + '  the current key, swaps them atomically, and keeps the old one as <room>_pre_epoch\n'
+      + '  for you to drop. Nothing is deleted.');
   }
 }
 
 async function ensureSchema(client) {
   const rooms = await resolveRooms(client);
+  // BEFORE the CREATEs and ALTERs, not beside the end-of-function asserts. A house whose
+  // record says a newer release moved it forward may have columns this template does not
+  // know; running this template's DDL first could add back what that release removed —
+  // the exact write the guard exists to prevent.
+  await assertWriterSupported(client, rooms);
   const tpl = fs.readFileSync(path.join(__dirname, '..', 'house', 'schema.sql.tpl'), 'utf-8');
   const sql = tpl; // plain shared tables — nothing to render
   const stripped = sql.split('\n').map((l) => l.replace(/--.*$/, '')).join('\n');
@@ -418,10 +441,11 @@ async function ensureSchema(client) {
     console.error(`[memhouse] ${denied} schema statement(s) needed rights you do not hold — continuing with what you can do.`);
     console.error('[memhouse] creating or replacing a ROOM is the house owner\'s job; adding a missing COLUMN is not.');
   }
-  // A house created before origin existed has no such column, and the clear binds it —
-  // an unguarded DELETE there would be the old destructive behaviour, and a guarded one
-  // would error. Add it in place; ReplacingMergeTree backfills the DEFAULT, so every
-  // pre-existing row reads as 'ship', which is what it was.
+  // A house created before origin existed has no such column, and every read and write
+  // scopes by it. Add it in place; ReplacingMergeTree backfills the DEFAULT, so every
+  // pre-existing row reads as 'ship', which is what it was. (Having the COLUMN is not
+  // having it in the sorting KEY — that is assertRoomKeys' refusal and migrate-rooms'
+  // job, because ORDER BY cannot be altered.)
   //
   // EVERY column the template declares, not just `origin`.
   //
@@ -435,60 +459,198 @@ async function ensureSchema(client) {
   // It also made the grant a lie: rooms.js and INSTALL.md both say ALTER ADD COLUMN is
   // granted "for ensureSchema's rollout", and no rollout of anything but `origin` existed.
   const wantCols = templateColumns(tpl);
-  for (const t of ROOM_TYPES) {
+  for (const t of ALL_TABLES) {
     let have = new Set();
     try {
       const rs = await client.query({
         query: 'SELECT name FROM system.columns WHERE database = currentDatabase() AND table = {n:String}',
-        query_params: { n: rooms[t] }, format: 'JSONEachRow',
+        query_params: { n: rooms[`${t}_raw`] }, format: 'JSONEachRow',
       });
       have = new Set((await rs.json()).map((r) => r.name));
     } catch { /* unreadable: the checks above already reported why */ }
+    // No columns means the table is not there — the CREATEs above were denied, or this is
+    // a house where the meta tables were never created. ALTERing it would throw
+    // UNKNOWN_TABLE, which is not a permissions error and so prints, once per column, on
+    // every `--ensure-schema`. Nothing to heal on a table that does not exist.
+    if (!have.size) continue;
     for (const col of (wantCols[t] || [])) {
-      if (have.size && have.has(col.name)) continue;
+      if (have.has(col.name)) continue;
       try {
         await client.command({
-          query: `ALTER TABLE ${rooms[t]} ADD COLUMN IF NOT EXISTS ${col.name} ${col.type}`,
+          query: `ALTER TABLE ${rooms[`${t}_raw`]} ADD COLUMN IF NOT EXISTS ${col.name} ${col.type}`,
           clickhouse_settings: { allow_experimental_full_text_index: 1 },
         });
-        if (have.size) console.log(`[memhouse] added missing column ${rooms[t]}.${col.name}`);
+        console.log(`[memhouse] added missing column ${rooms[`${t}_raw`]}.${col.name}`);
       } catch (e) {
         const m = e && e.message ? e.message : String(e);
         if (!/Not enough privileges|ACCESS_DENIED/i.test(m)) {
-          console.error(`[memhouse] could not add ${rooms[t]}.${col.name}: ${m}`);
+          console.error(`[memhouse] could not add ${rooms[`${t}_raw`]}.${col.name}: ${m}`);
         }
       }
     }
-    try {
-      await client.command({
-        query: `ALTER TABLE ${rooms[t]} ADD COLUMN IF NOT EXISTS origin LowCardinality(String) DEFAULT 'ship'`,
-        // 25.11 refuses ANY alter on a table carrying the messages text indexes unless
-        // this is set — including an ADD COLUMN that has nothing to do with them
-        // (Code: 344, SUPPORT_IS_DISABLED). Without it the backfill fails silently and
-        // the guard ends up missing on precisely the houses that need upgrading. A no-op
-        // on 26.x, and query-scoped, so it needs no server config.
-        clickhouse_settings: { allow_experimental_full_text_index: 1 },
-      });
-    } catch (e) {
-      // No rights to alter is genuinely not fatal — a member on someone else's house
-      // cannot roll a column out and does not need to. But this used to swallow
-      // EVERYTHING, including the 25.11 Code 344 this call carries a setting to avoid, and
-      // `schema ensured` printed regardless. Name anything that is not a permissions
-      // problem, and keep going.
-      const m = e && e.message ? e.message : String(e);
-      if (!/Not enough privileges|ACCESS_DENIED/i.test(m)) {
-        console.error(`[memhouse] could not add the 'origin' column to ${rooms[t]}: ${m}`);
-        console.error('[memhouse] the room still works; a house that predates 0.4.4 needs rebuilding — memhouse doctor');
-      }
-    }
   }
+  // There used to be a second, hardcoded `ADD COLUMN … origin` here, from when the healer
+  // above could only add that one column. It is redundant now that the generic loop reads
+  // the template correctly — and it would have been actively wrong once this loop covered
+  // house_meta and house_events, which have no origin and want none.
 
   await assertRoomsExist(client, rooms);
-  await assertOriginKeyed(client, rooms);
+  await assertRoomKeys(client, rooms);
   await warnMissingColumns(client, rooms);
+  // Re-assert the async-insert pin. A 0.11 member holds ALTER USER on themselves (so
+  // `memhouse passwd` needs no admin) — and that same grant lets them drop the pin
+  // (`ALTER USER <self> SETTINGS NONE`), which on some ClickHouse versions makes their
+  // OWN async inserts land with an empty user_id. It cannot forge another identity
+  // (currentUser() is server-side) and doctor flags any blank user_id, but re-adding the
+  // pin here means an accidental or transient drop self-heals on the next --ensure-schema
+  // (install, update, migrate). Best-effort: a member who cannot ADD SETTING just stays
+  // as they were.
+  try { await client.command({ query: `ALTER USER \`${rooms.user}\` ADD SETTING ${MEMBER_PIN}`, clickhouse_settings: { async_insert: 0 } }); }
+  catch { /* no self-alter grant (pre-0.11), or not permitted — leave it */ }
+  // The rooms are at this schema generation — the assertion above is what makes that a
+  // fact rather than a claim. Recording it here means a house built by `install` is
+  // already stamped, instead of looking un-migrated until its first ship pass.
+  //
+  // MEMHOUSE_NO_RECORD is the invite path: `memhouse invite` runs this only to PROVE the
+  // new member can build the rooms, on a throwaway home. Recording would stamp a phantom
+  // <member>@<ephemeral-host> writer into a house the inviter never ships to — the
+  // invitee mints their real identity on their own first pass.
+  if (process.env.MEMHOUSE_NO_RECORD !== '1') await recordHouseState(client, rooms, hostId());
   return stmts.length;
 }
 
+/**
+ * Keep the house's record of itself current: which schema generation it is at, and which
+ * memhouse version is writing into it.
+ *
+ * Written only when something CHANGED. A row per pass would turn house_events into a
+ * heartbeat log — under `--loop` at the default interval that is 288 rows a day per
+ * machine, and the one question the table exists to answer ("when did this house move,
+ * and who moved it") would be buried in noise.
+ *
+ * Entirely best-effort. A member on someone else's house may hold no rights on these
+ * tables, or the house may predate them; none of that is a reason to stop shipping, so
+ * every failure here is swallowed. The rooms are the product, this is the paperwork.
+ */
+async function recordHouseState(client, rooms, host) {
+  const version = require('../../package.json').version;
+  const writer = `${rooms.user}@${host}`;
+  try {
+    const rs = await client.query({
+      query: `SELECT key, value FROM ${rooms.house_meta} FINAL WHERE key IN ('schema_version', {ck:String}, {cs:String})`,
+      // Keyed per member AND host. Keyed by member alone, two machines of one member
+      // overwrote each other's entry on every pass, so the fleet view could only ever
+      // show the machine that shipped last — the exact machine that needs no attention.
+      query_params: { ck: `client_version:${writer}`, cs: `client_schema:${writer}` }, format: 'JSONEachRow',
+    });
+    const have = new Map((await rs.json()).map((r) => [r.key, String(r.value)]));
+    const events = [];
+    const metas = [];
+    if (have.get('schema_version') !== String(SCHEMA_VERSION)) {
+      // The rooms are at this generation — assertRoomKeys ran before this and refuses to
+      // let a pass reach here otherwise, so recording it is a statement of fact, not a
+      // claim about what someone intends to do.
+      metas.push({ key: 'schema_version', value: String(SCHEMA_VERSION), host });
+      events.push({
+        kind: 'schema', status: 'observed', host,
+        from_version: have.get('schema_version') || '', to_version: String(SCHEMA_VERSION),
+        detail: 'rooms carry the current sorting keys',
+      });
+    }
+    const clientKey = `client_version:${writer}`;
+    // The schema this writer SUPPORTS, beside the marketing version — the fleet view
+    // judges on this, because a version string cannot be compared against a schema
+    // requirement (and a pre-release checkout may not have bumped package.json at all).
+    // Tracked on its own change, not the version's: tying it to a version bump left every
+    // already-recorded writer without one forever.
+    const mySchema = String(Math.max(...SUPPORTED_SCHEMAS));
+    if (have.get(`client_schema:${writer}`) !== mySchema) {
+      metas.push({ key: `client_schema:${writer}`, value: mySchema, host });
+    }
+    if (have.get(clientKey) !== version) {
+      metas.push({ key: clientKey, value: version, host });
+      events.push({
+        kind: 'version', status: 'observed', host,
+        from_version: have.get(clientKey) || '', to_version: version,
+        detail: `${rooms.user} shipping from ${host}`,
+      });
+    }
+    // The heartbeat, EVERY pass — the fleet view's health column. Latest-wins on the key,
+    // so it is one live row per writer however often it fires; version/schema above stay
+    // change-only so house_events remains a record of moves, not a pulse trace.
+    metas.push({ key: `last_ship:${writer}`, value: new Date().toISOString(), host });
+    await client.insert({
+      table: rooms.house_meta_raw, values: metas, format: 'JSONEachRow',
+      clickhouse_settings: { async_insert: 0 },
+    });
+    if (events.length) {
+      await client.insert({
+        table: rooms.house_events_raw, values: events, format: 'JSONEachRow',
+        clickhouse_settings: { async_insert: 0 },
+      });
+    }
+  } catch { /* the house's paperwork is never worth failing a pass over */ }
+}
+
+/**
+ * The forward half of the compatibility story: refuse a house whose RECORDED schema this
+ * release does not support, before any room is touched.
+ *
+ * Distinct from assertRoomKeys, which inspects key SHAPES — a future generation could be
+ * a data transform the keys do not show. This reads what the house says about itself:
+ * house_meta['schema_version'] (what generation the rooms are at) and
+ * house_meta['min_writer_schema'] (the floor `memhouse migrate` sets under writers).
+ * Too new -> the fix is on THIS machine: memhouse update. Below the floor -> same.
+ * (A house OLDER than this release is not an error here — the room checks catch it and
+ * name `memhouse migrate`; this guard must not fire on a pre-0.10 house that has no
+ * record at all.)
+ *
+ * Releases before 0.10.0 never read this — for them the floor is enforced by the
+ * pilot's GRANTs, not by code.
+ */
+async function assertWriterSupported(client, rooms) {
+  let have;
+  try {
+    const rs = await client.query({
+      query: `SELECT key, value FROM ${rooms.house_meta} FINAL WHERE key IN ('schema_version', 'min_writer_schema')`,
+      format: 'JSONEachRow',
+    });
+    have = new Map((await rs.json()).map((r) => [r.key, toInt(r.value)]));
+  } catch (e) {
+    // ONLY a missing table means a pre-0.10 house (the room checks own that case). Any
+    // other failure — an ACCESS_DENIED on house_meta, a timeout — used to fall through
+    // here too, and a writer that merely could not READ the record was treated as if the
+    // record did not exist: on a newer-schema house whose key shapes happen to match,
+    // that bypassed the whole compatibility guard. Not being able to check is a reason
+    // to refuse, never a reason to proceed.
+    const m = e && e.message ? e.message : String(e);
+    if (/UNKNOWN_TABLE|Unknown table expression|doesn't exist|does not exist/i.test(m)) return;
+    throw new Error(`could not read this house's compatibility record (${m.split('\n')[0]})\n`
+      + '  Refusing to write until it is readable — a writer that cannot check the schema\n'
+      + '  generation must not assume it matches.');
+  }
+  const houseSchema = have.get('schema_version') || 0;
+  const floor = have.get('min_writer_schema') || 0;
+  const mine = Math.max(...SUPPORTED_SCHEMAS);
+  // MEMBERSHIP, not a ceiling. `houseSchema <= max` would let a [4]-only writer into a
+  // schema-3 house — and if generation 4 was a data transform, the sorting keys match and
+  // nothing else refuses. A recorded generation this release does not list is unwritable
+  // in either direction; only the remedy differs.
+  if (houseSchema > 0 && !SUPPORTED_SCHEMAS.includes(houseSchema)) {
+    throw new Error(houseSchema > mine
+      ? `this house is at schema ${houseSchema}; this memhouse (${require('../../package.json').version}) supports ${SUPPORTED_SCHEMAS.join(', ')}.\n`
+        + '  A newer release moved the house forward. Writing with this one could corrupt it, so it will not.\n'
+        + '  Update THIS machine:  memhouse update'
+      : `this house is at schema ${houseSchema}; this memhouse supports ${SUPPORTED_SCHEMAS.join(', ')}.\n`
+        + '  Bring the house forward:  memhouse migrate');
+  }
+  if (floor > mine) {
+    throw new Error(
+      `this house requires writers at schema ${floor}+; this memhouse supports ${SUPPORTED_SCHEMAS.join(', ')}.\n`
+      + '  Update THIS machine:  memhouse update');
+  }
+}
+
 // Incremental state: what the house already holds, keyed by session_id. We compare
 // against extra.bubbleCount (the adapter's own cheap count, stored at ship time) —
 // NOT message_count — because parsed-message count and bubbleCount are different
@@ -496,11 +658,11 @@ async function ensureSchema(client) {
 // must be decidable WITHOUT calling getMessages on every chat.
 async function loadExisting(client, rooms) {
   const rs = await client.query({
-    query: `SELECT session_id, last_updated_at, message_count, extra FROM ${rooms.sessions} FINAL WHERE user_id = currentUser()`,
+    query: `SELECT session_id, last_updated_at, message_count, extra FROM ${rooms.sessions_raw} FINAL WHERE user_id = currentUser()`,
     format: 'JSONEachRow',
   });
-  // Actual message rows per session: an interrupted re-ship (crash between the
-  // clear-DELETE and the inserts, or mid-flush on a large transcript) leaves a
+  // Actual message rows per session: an interrupted re-ship (a crash mid-flush on a
+  // large transcript, or between the sessions insert and the messages one) leaves a
   // fresh-looking session row with a missing or PARTIAL transcript — the skip
   // must compare the real row count against the recorded message_count, not
   // merely check that some row exists.
@@ -511,12 +673,40 @@ async function loadExisting(client, rooms) {
   // on every pass — no data lost, but the incremental skip silently stops existing.
   // Measured: adding one origin='import' row to a settled session made it re-ship
   // every pass; removing it froze the session again.
+  //
+  // GROUPED BY EPOCH, and only the newest epoch's rows count. A session whose parse was
+  // superseded still holds every older parse — that is the point of the epoch — so a
+  // plain count would exceed message_count forever, `intact` would never be true, and the
+  // incremental skip would quietly stop existing for exactly the sessions that have been
+  // through a compaction. One row per (session, epoch); epochs are rare, so this stays
+  // the same size as the per-session grouping it replaces.
   const mr = await client.query({
-    query: `SELECT session_id, countIf(origin = 'ship') AS n FROM ${rooms.messages} FINAL WHERE user_id = currentUser() GROUP BY session_id`,
+    query: `SELECT session_id, epoch, count() AS n, max(seq) AS max_seq FROM ${rooms.messages_raw} FINAL
+            WHERE user_id = currentUser() AND origin = 'ship' GROUP BY session_id, epoch`,
     format: 'JSONEachRow',
   });
-  const msgCounts = new Map();
-  for (const r of await mr.json()) msgCounts.set(r.session_id, toInt(r.n));
+  // Kept PER EPOCH, not collapsed to each room's own newest — a room can legitimately
+  // have no rows at the session's current epoch, and reading its highest epoch instead
+  // would answer with the superseded parse. See the tool_calls note below, which is where
+  // that actually bites.
+  //
+  // U+0000 as the key separator, and written as an ESCAPE, deliberately. A session_id is
+  // '<source>:<adapter-local id>' and an adapter-local id can legally contain any
+  // printable character, so only a byte that cannot appear in one is collision-proof.
+  // The escape matters twice over: a literal NUL byte typed here once made grep treat
+  // this file as binary — and one of the three key sites was typed with a plain space
+  // instead, so the tool lookups missed on every call, `tools.n` read 0, every session
+  // with tool calls failed `intact`, and a real 437-session history re-shipped 324
+  // sessions on EVERY pass, forever, with every surface green. All three sites must
+  // build this key identically; the unit test pins them to each other.
+  const at = (m, id, epoch) => m.get(`${id}\u0000${epoch}`) || null;
+  const msgState = new Map();     // (session, epoch) → { n, maxSeq }
+  const msgEpoch = new Map();     // session → newest epoch present
+  for (const r of await mr.json()) {
+    const epoch = toInt(r.epoch);
+    msgState.set(`${r.session_id}\u0000${epoch}`, { n: toInt(r.n), maxSeq: toInt(r.max_seq) });
+    msgEpoch.set(r.session_id, Math.max(epoch, msgEpoch.get(r.session_id) || 0));
+  }
   // The tool_calls room needs the same check, and used to have none. shipSession writes
   // sessions, then messages, then tool_calls — so a pass that fails during the LAST of
   // those three leaves a session whose message count matches perfectly. The next pass
@@ -525,11 +715,17 @@ async function loadExisting(client, rooms) {
   // Reproduced by dropping the room mid-pass; any transient — a ClickHouse restart, a
   // quota rejection, a network blip — reaches the same state.
   const tr = await client.query({
-    query: `SELECT session_id, countIf(origin = 'ship') AS n FROM ${rooms.tool_calls} FINAL WHERE user_id = currentUser() GROUP BY session_id`,
+    query: `SELECT session_id, epoch, count() AS n, max(idx) AS max_idx FROM ${rooms.tool_calls_raw} FINAL
+            WHERE user_id = currentUser() AND origin = 'ship' GROUP BY session_id, epoch`,
     format: 'JSONEachRow',
   });
-  const toolCounts = new Map();
-  for (const r of await tr.json()) toolCounts.set(r.session_id, toInt(r.n));
+  const toolState = new Map();    // (session, epoch) → { n, maxIdx }
+  const toolEpoch = new Map();    // session → newest epoch present
+  for (const r of await tr.json()) {
+    const epoch = toInt(r.epoch);
+    toolState.set(`${r.session_id}\u0000${epoch}`, { n: toInt(r.n), maxIdx: toInt(r.max_idx) });
+    toolEpoch.set(r.session_id, Math.max(epoch, toolEpoch.get(r.session_id) || 0));
+  }
   const map = new Map();
   for (const r of await rs.json()) {
     // DateTime64 comes back as 'YYYY-MM-DD HH:MM:SS.mmm' — re-parse as UTC.
@@ -540,9 +736,30 @@ async function loadExisting(client, rooms) {
     // have tool calls re-ship exactly once and then record it. Self-correcting, and
     // cheaper than a branch that has to be remembered forever.
     const toolCount = toInt(r.extra && r.extra.toolCallCount);
-    const intact = (msgCounts.get(r.session_id) || 0) === count
-      && (toolCounts.get(r.session_id) || 0) === toolCount;
-    map.set(r.session_id, { ms, count, bc, intact });
+    // The session's current epoch is whatever its ROWS say, never what the sessions row
+    // says: sessions is latest-wins with no origin in its key, so an import writing that
+    // row after a bump would hand the next pass a 0 and make it overwrite the current
+    // parse. The column there is for people, not for this.
+    const epoch = Math.max(msgEpoch.get(r.session_id) || 0, toolEpoch.get(r.session_id) || 0);
+    // Everything else is read AT that epoch, and "no rows there" means zero — not "look
+    // at the newest epoch this room happens to have".
+    //
+    // Getting this wrong does not lose data, it makes the shipper churn forever. A parse
+    // that keeps its messages but drops every tool call writes nothing into tool_calls at
+    // the new epoch. Reading that room's own newest epoch then returned the SUPERSEDED
+    // count (say 2) and its maxIdx — so `intact` compared 2 against the recorded 0 and was
+    // false on every pass, and decideEpoch compared `0 - 1 < oldMaxIdx` and bumped again
+    // on every pass. The session re-shipped and gained an epoch forever, growing the house
+    // without end, while every surface reported success.
+    const msgs = at(msgState, r.session_id, epoch) || { n: 0, maxSeq: -1 };
+    const tools = at(toolState, r.session_id, epoch) || { n: 0, maxIdx: -1 };
+    const intact = msgs.n === count && tools.n === toolCount;
+    map.set(r.session_id, {
+      ms, count, bc, intact, epoch,
+      maxSeq: msgs.maxSeq,
+      maxIdx: tools.maxIdx,
+      hasRows: msgEpoch.has(r.session_id) || toolEpoch.has(r.session_id),
+    });
   }
   return map;
 }
@@ -663,20 +880,124 @@ function rowsForChat(chat, host) {
   return { session, msgRows, toolRows };
 }
 
+/**
+ * Which parse epoch this re-ship belongs to, and why.
+ *
+ * THE RULE: the shipper never writes over stored content that differs from what it is
+ * writing. If the new parse can be laid on top of the old one without changing or
+ * orphaning anything, it reuses the epoch and ReplacingMergeTree dedupes it exactly as
+ * before — the common case, free. Otherwise it moves to a new epoch, where the new rows
+ * cannot collide with the old ones, and the old parse stays whole.
+ *
+ * Two things make a parse un-overwritable, and both are ordinary:
+ *
+ *   SHRINK. A re-parse with fewer messages leaves the old higher-seq rows with nothing
+ *   written over them. That stale tail is what the DELETE this replaces was written to
+ *   remove — and the delete could not tell a fixed adapter bug (junk, worth removing)
+ *   from a compacted or user-deleted transcript (irreplaceable), because from outside
+ *   they look identical.
+ *
+ *   DIVERGENCE. Claude Code COMPACTS a session in place: same session, rewritten
+ *   shorter, with different content at low seq numbers. Bumping on shrink alone would
+ *   preserve the tail and still overwrite the rewritten head — the same loss, reached
+ *   through the merge instead of the delete. line_hash is already stored per row, so
+ *   comparing is a lookup, not a re-read of the source.
+ *
+ * Tool calls carry no hash, so they are compared on (tool_name, args) directly. They are
+ * derived from `m._toolCalls`, which can change while the assistant text does not.
+ *
+ * Pure, and exported: this is the decision the whole redesign rests on, and it is unit
+ * tested without a server.
+ *
+ * @param {object|null} stored  { epoch, maxSeq, maxIdx, hashes: Map<seq,string>,
+ *                                tools: Map<idx,{tool_name,args}> } — null when the
+ *                                session is not in the house yet.
+ * @param {object} incoming     { msgRows, toolRows } as built by rowsForChat.
+ * @returns {{ epoch: number, reason: string|null }} reason is null when nothing moved.
+ */
+function decideEpoch(stored, incoming) {
+  if (!stored) return { epoch: 0, reason: null };
+  const epoch = toInt(stored.epoch);
+  const bump = (reason) => ({ epoch: epoch + 1, reason });
+
+  const maxSeq = Number.isFinite(stored.maxSeq) ? stored.maxSeq : -1;
+  const maxIdx = Number.isFinite(stored.maxIdx) ? stored.maxIdx : -1;
+  if (incoming.msgRows.length - 1 < maxSeq) {
+    return bump(`${maxSeq + 1} messages stored, ${incoming.msgRows.length} parsed`);
+  }
+  if (incoming.toolRows.length - 1 < maxIdx) {
+    return bump(`${maxIdx + 1} tool calls stored, ${incoming.toolRows.length} parsed`);
+  }
+
+  // A seq the house does not hold is a gap, not a disagreement — there is nothing there
+  // to destroy, so it is not a reason to fork the session.
+  const hashes = stored.hashes || new Map();
+  for (const row of incoming.msgRows) {
+    if (row.seq > maxSeq) break;
+    const was = hashes.get(row.seq);
+    if (was !== undefined && was !== String(row.line_hash)) {
+      return bump(`message ${row.seq} was rewritten`);
+    }
+  }
+  const tools = stored.tools || new Map();
+  for (const row of incoming.toolRows) {
+    if (row.idx > maxIdx) break;
+    const was = tools.get(row.idx);
+    if (was !== undefined && (was.tool_name !== row.tool_name || was.args !== row.args)) {
+      return bump(`tool call ${row.idx} was rewritten`);
+    }
+  }
+  return { epoch, reason: null };
+}
+
+/**
+ * What the house currently holds for ONE session, at its current epoch — the input
+ * decideEpoch needs to tell an append from a rewrite.
+ *
+ * Read only for sessions actually being re-shipped, which the incremental skip has
+ * already narrowed to a handful; it replaces the two DELETE mutations that used to run
+ * per re-shipped session, so a normal pass does strictly less work than before.
+ *
+ * `toString(line_hash)`: the client is configured with
+ * output_format_json_quote_64bit_integers = 0, so a UInt64 arrives as a JSON number and
+ * everything above 2^53 comes back rounded. Comparing rounded hashes would report
+ * divergence on identical rows and fork a session on every single pass.
+ */
+async function loadStoredParse(client, rooms, id, uid, epoch) {
+  const params = { id, uid, e: epoch };
+  const mr = await client.query({
+    query: `SELECT seq, toString(line_hash) AS line_hash FROM ${rooms.messages_raw} FINAL
+            WHERE session_id = {id:String} AND user_id = {uid:String} AND origin = 'ship' AND epoch = {e:UInt32}`,
+    query_params: params, format: 'JSONEachRow',
+  });
+  const hashes = new Map();
+  for (const r of await mr.json()) hashes.set(toInt(r.seq), String(r.line_hash));
+  const tr = await client.query({
+    query: `SELECT idx, tool_name, args FROM ${rooms.tool_calls_raw} FINAL
+            WHERE session_id = {id:String} AND user_id = {uid:String} AND origin = 'ship' AND epoch = {e:UInt32}`,
+    query_params: params, format: 'JSONEachRow',
+  });
+  const tools = new Map();
+  for (const r of await tr.json()) tools.set(toInt(r.idx), { tool_name: r.tool_name, args: r.args });
+  return { hashes, tools };
+}
+
 // One shipping pass. Incremental unless opts.full: a chat is skipped when the house
 // already has it at least as fresh (last_updated_at) and at least as large
 // (extra.bubbleCount) — both readable without parsing the chat.
 async function runShip(client, opts = {}) {
   const { full = false } = opts;
   const host = hostId();
-  // Always load what the house holds — even with --full. The skip decision uses it
-  // only in incremental mode, but re-shipping a KNOWN session must clear its old
-  // message/tool rows first (a shrunken re-parse leaves stale higher-seq rows
-  // otherwise: ReplacingMergeTree collapses same-key rows only).
+  // Always load what the house holds — even with --full. The skip decision uses it only
+  // in incremental mode, but re-shipping a KNOWN session needs its current epoch and row
+  // shape: writing at the wrong epoch either forks a session that did not change, or
+  // overwrites a stored parse that did.
   const rooms = await resolveRooms(client);
+  await assertWriterSupported(client, rooms);
   await assertRoomsExist(client, rooms);
-  await assertOriginKeyed(client, rooms);
+  await assertRoomKeys(client, rooms);
   await warnMissingColumns(client, rooms);
+  await recordHouseState(client, rooms, host);
   const existing = await loadExisting(client, rooms);
   const chats = getAllChats();
 
@@ -693,7 +1014,7 @@ async function runShip(client, opts = {}) {
   const flush = async (table) => {
     if (!batches[table].length) return;
     await client.insert({
-      table: rooms[table],
+      table: rooms[`${table}_raw`],
       values: batches[table],
       format: 'JSONEachRow',
       clickhouse_settings: { async_insert: 0 }, // binding: user_id stamping breaks otherwise
@@ -706,7 +1027,7 @@ async function runShip(client, opts = {}) {
   };
 
   const seen = new Set(); // adapters must not double-ship a session id within a pass
-  let sessions = 0, skipped = 0, msgRows = 0, toolRows = 0, unreadable = 0;
+  let sessions = 0, skipped = 0, msgRows = 0, toolRows = 0, unreadable = 0, bumped = 0, withheld = 0;
   for (const chat of chats) {
     if (chat.encrypted) continue;
     // Same canonical '<source>:<adapter-local id>' as rowsForChat: dedup and
@@ -734,11 +1055,13 @@ async function runShip(client, opts = {}) {
     }
 
     // rowsForChat returns null only when getMessages() *throws*. An adapter that
-    // swallows its own failure returns [] instead, which would look like a session
-    // that genuinely has no messages — and for a known session the re-ship below
-    // deletes the old transcript before inserting that emptiness. So treat a
-    // failure recorded while reading THIS chat as unreadable too: write nothing,
-    // delete nothing, retry next pass.
+    // swallows its own failure returns [] instead, which would look like a session that
+    // genuinely has no messages — and for a known session that is a maximal shrink, so
+    // the pass would fork it to a new, EMPTY epoch and every read would show nothing
+    // where a transcript used to be. Nothing is destroyed any more, but a house that
+    // reports empty sessions after a locked SQLite file is still wrong. So treat a
+    // failure recorded while reading THIS chat as unreadable: write nothing, retry next
+    // pass.
     const errsBefore = adapterErrorSink.recorded().length;
     const rows = rowsForChat(chat, host);
     const failedHere = adapterErrorSink.recorded().slice(errsBefore).some((e) => e.source === chat.source);
@@ -753,43 +1076,56 @@ async function runShip(client, opts = {}) {
     // the skip predicate entirely, so every pass retries it and re-reports it until
     // the underlying store is readable.
     if (!rows || failedHere) { unreadable++; continue; }
-    if (prev) {
-      // Known session being re-shipped: clear its old rows BEFORE inserting so a
-      // shorter re-parse can't leave stale seq/idx tails. A crash between the
-      // delete and the inserts is repaired by the next pass: the skip predicate
-      // refuses to skip a non-empty session whose message rows are missing.
-      // user_id is BOUND, not `= currentUser()`. A DELETE is a mutation, and a mutation
-      // does not necessarily evaluate currentUser() in the caller's context — it matches
-      // nothing at all, so the delete silently removes zero rows and the stale tail this
-      // code exists to clear survives forever. Measured on ClickHouse 25.11 —
-      // the identical predicate with the literal value deleted 2000 rows where
-      // currentUser() deleted 0. The value is the same identity either way: it is read
-      // from the server over this very connection.
-      // origin='ship' is the third bind, and it is not cosmetic. This clear exists so a
-      // shorter re-parse cannot leave a stale seq tail behind — but scoped to
-      // (session_id, user_id) alone it deletes EVERY row for the session, including rows
-      // the adapters did not write and cannot rewrite. Measured, on a real house: an
-      // import of 135,307 messages lost 27,948 of them to one ship pass, because
-      // memory-house had captured more per session than the adapters emit. The rows the
-      // shipper owns are the only rows it may remove.
-      for (const t of ['messages', 'tool_calls']) {
-        await client.command({
-          query: `DELETE FROM ${rooms[t]} WHERE session_id = {id:String} AND user_id = {uid:String} AND origin = 'ship'`,
-          query_params: { id, uid: rooms.user },
-          clickhouse_settings: { async_insert: 0 },
-        });
+
+    // A KNOWN session that now parses to nothing is withheld, not written.
+    //
+    // Two reasons, and the second is the one that bites. An adapter that swallows its own
+    // failure returns [] rather than throwing, which is indistinguishable from a chat
+    // whose content really did vanish — and the house is the party that still has the
+    // content either way. Worse, writing it would not even be honest about itself: with
+    // zero rows to insert, the new epoch exists nowhere in the room, so the read filter's
+    // max(epoch) would keep answering with the SUPERSEDED epoch and serve the old parse as
+    // if it were current. Withholding leaves the session out of the skip predicate, so
+    // every pass retries it and says so.
+    if (prev && prev.hasRows && !rows.msgRows.length) {
+      withheld++;
+      console.log(`[memhouse] WARNING: ${id} parsed to 0 messages but the house holds ${prev.count} — withholding`);
+      continue;
+    }
+
+    // Which parse this belongs to. A known session whose stored rows the new parse
+    // cannot be laid over — shorter, or rewritten at an overlapping position — moves to
+    // a new epoch instead of overwriting anything. See decideEpoch.
+    //
+    // This is where a DELETE used to be. It removed the session's shipped rows before
+    // re-inserting them, which is the only way to clear a stale seq tail in a
+    // ReplacingMergeTree — and it destroyed content that existed nowhere else, because
+    // Claude Code compacts transcripts and deletes them after cleanupPeriodDays (30 by
+    // default), so a shorter re-parse is at least as likely to mean "the source lost it"
+    // as "the adapter was fixed". The house is supposed to outlive the source.
+    let epoch = 0;
+    if (prev && prev.hasRows) {
+      const stored = await loadStoredParse(client, rooms, id, rooms.user, prev.epoch);
+      const d = decideEpoch({ ...prev, ...stored }, rows);
+      epoch = d.epoch;
+      if (d.reason) {
+        bumped++;
+        console.log(`[memhouse] ${id} → epoch ${epoch} (${d.reason}); the previous parse is kept`);
       }
+    } else if (prev) {
+      epoch = toInt(prev.epoch);
     }
+    rows.session.epoch = epoch;
     await push('sessions', scrub(rows.session));
     sessions++;
-    for (const r of rows.msgRows) { await push('messages', scrub(r)); msgRows++; }
-    for (const r of rows.toolRows) { await push('tool_calls', scrub(r)); toolRows++; }
+    for (const r of rows.msgRows) { r.epoch = epoch; await push('messages', scrub(r)); msgRows++; }
+    for (const r of rows.toolRows) { r.epoch = epoch; await push('tool_calls', scrub(r)); toolRows++; }
   }
   for (const table of Object.keys(batches)) await flush(table);
   // Anything that only failed while reading messages — the sink is reset by the
   // next getAllChats(), so unreported here means never reported at all.
   reportAdapterErrors(warned);
-  return { sessions, skipped, msgRows, toolRows, unreadable };
+  return { sessions, skipped, msgRows, toolRows, unreadable, bumped, withheld };
 }
 
 // Warn once per adapter per pass. `warned` carries across the two call sites so one
@@ -870,7 +1206,10 @@ async function main() {
       let failed = false;
       try {
         const r = await runShip(client, { full });
-        console.log(`[memhouse] shipped ${r.sessions} sessions (${r.skipped} skipped${r.unreadable ? `, ${r.unreadable} unreadable-will-retry` : ''}) → ` +
+        console.log(`[memhouse] shipped ${r.sessions} sessions (${r.skipped} skipped`
+          + `${r.unreadable ? `, ${r.unreadable} unreadable-will-retry` : ''}`
+          + `${r.withheld ? `, ${r.withheld} withheld-empty` : ''}`
+          + `${r.bumped ? `, ${r.bumped} kept an earlier parse` : ''}) → ` +
           `${r.msgRows} msg rows, ${r.toolRows} tool rows in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
       } catch (e) {
         failed = true;
@@ -921,7 +1260,7 @@ async function main() {
   }
 }
 
-module.exports = { runShip, ensureSchema, templateColumns };
+module.exports = { runShip, ensureSchema, templateColumns, decideEpoch };
 
 if (require.main === module) {
   main().catch((e) => {
@@ -936,7 +1275,11 @@ if (require.main === module) {
     // advice to run `ship --ensure-schema`.
     const m = e && e.message ? e.message : String(e);
     if (/Not enough privileges|ACCESS_DENIED/i.test(m) && process.env.MEMHOUSE_QUIET_DENIED === '1') process.exit(1);
-    console.error(`[memhouse] ${/Not enough privileges|ACCESS_DENIED/i.test(m) ? 'refused' : 'fatal'}: ${m}`);
+    // The sorting-key refusal joins the permissions one: both are the shipper DECLINING
+    // to act, with the remediation already in the message. "fatal:" ahead of it made the
+    // most-seen line of the upgrade path — every `--ensure-schema` on a pre-epoch house —
+    // read like a crash.
+    console.error(`[memhouse] ${/Not enough privileges|ACCESS_DENIED|wrong sorting key/i.test(m) ? 'refused' : 'fatal'}: ${m}`);
     process.exit(1);
   });
 }