memhouse 0.8.0 → 0.11.0

package/memhouse/house/schema.sql.tpl

@@ -39,7 +39,13 @@ CREATE TABLE IF NOT EXISTS sessions
     -- messages and tool_calls are the opposite case and DO key on origin -- see there.
     origin LowCardinality(String) DEFAULT 'ship',
     user_id String MATERIALIZED currentUser(),
-    ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
+    ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
+    -- The session's CURRENT parse epoch, for people and for `doctor` — never for reads.
+    -- The read filter is computed from the messages room itself (see there), because an
+    -- IMPORT writes a sessions row too, and this room is latest-wins with no origin in the
+    -- key: an import landing after a shipped epoch bump would hand every reader a 0 and
+    -- hide the current transcript. Observability only.
+    epoch UInt32 DEFAULT 0
 )
 ENGINE = ReplacingMergeTree(ingested_at)
 ORDER BY (session_id, user_id);
@@ -63,18 +69,35 @@ CREATE TABLE IF NOT EXISTS messages
     is_subagent Bool DEFAULT false,
     extra JSON,
     line_hash UInt64,
-    -- Who put this row here. The shipper CLEARS a session before re-inserting it (a
-    -- shorter re-parse must not leave a stale seq tail), and that clear used to remove
-    -- every row for the session regardless of origin -- destroying imported history the
-    -- adapters cannot reproduce. The clear now binds origin='ship', so anything imported
-    -- survives a re-ship of the same session.
-    -- IN THE SORTING KEY, and that is the load-bearing half. Guarding the DELETE alone
-    -- is not enough: ReplacingMergeTree collapses on the sorting key, so an imported row
-    -- and a shipped row sharing (session_id, user_id, seq) are the SAME row and the newer
-    -- ingested_at wins. Measured: 3 imported + 2 shipped rows became 3, and the two
-    -- imported ones the shipper happened to overlap were gone. Same loss as the
-    -- unguarded delete, reached through the merge instead.
+    -- Who put this row here, IN THE SORTING KEY. ReplacingMergeTree collapses on the
+    -- sorting key, so an imported row and a shipped row sharing (session_id, user_id, seq)
+    -- would be the SAME row and the newer ingested_at would win. Measured: 3 imported +
+    -- 2 shipped rows became 3, and the two imported ones the shipper overlapped were gone.
+    -- A whole import of 135,307 messages once lost 27,948 of them this way.
     origin LowCardinality(String) DEFAULT 'ship',
+    -- Which PARSE of the session this row belongs to, also in the sorting key.
+    --
+    -- The shipper used to DELETE a session's rows before re-inserting them, because a
+    -- re-parse that yields fewer messages leaves the old higher-seq rows with nothing
+    -- written over them -- a stale tail, and RMT is a dedupe engine, not a diff engine.
+    -- That delete destroyed content that existed nowhere else: Claude Code deletes
+    -- transcripts after cleanupPeriodDays (30 by default) and COMPACTS them before that,
+    -- rewriting a session shorter and different. A shorter re-parse and a fixed adapter
+    -- bug are indistinguishable from outside, and the house is supposed to outlive the
+    -- source.
+    --
+    -- So the shipper never overwrites diverging content. When a re-parse is shorter than
+    -- what is stored, or any overlapping seq hashes differently, it bumps the epoch and
+    -- writes the new parse under it. The old rows keep their epoch, cannot collide with
+    -- the new ones, and stay complete. An unchanged re-ship reuses the epoch and dedupes
+    -- exactly as before, so the common case costs nothing extra. Insert-only: no DELETE,
+    -- no ALTER DELETE grant, no tombstones.
+    --
+    -- Reads see ONE parse: the read layer resolves this room to the current epoch per
+    -- (session_id, user_id) for origin='ship' rows, and leaves every other origin alone
+    -- (see house.js roomNames). Without that filter the retained rows would over-count
+    -- every rollup -- worse than the stale tail this replaces.
+    epoch UInt32 DEFAULT 0,
     user_id String MATERIALIZED currentUser(),
     ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
     text_ngram String MATERIALIZED lower(text),
@@ -83,7 +106,7 @@ CREATE TABLE IF NOT EXISTS messages
     INDEX idx_text_word  text_word  TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1
 )
 ENGINE = ReplacingMergeTree(ingested_at)
-ORDER BY (session_id, user_id, origin, seq);
+ORDER BY (session_id, user_id, origin, epoch, seq);
 
 CREATE TABLE IF NOT EXISTS tool_calls
 (
@@ -97,26 +120,74 @@ CREATE TABLE IF NOT EXISTS tool_calls
     ts DateTime64(3, 'UTC'),
     project String DEFAULT '',
     folder String DEFAULT '',
-    -- Who put this row here. The shipper CLEARS a session before re-inserting it (a
-    -- shorter re-parse must not leave a stale seq tail), and that clear used to remove
-    -- every row for the session regardless of origin -- destroying imported history the
-    -- adapters cannot reproduce. The clear now binds origin='ship', so anything imported
-    -- survives a re-ship of the same session.
-    -- IN THE SORTING KEY, and that is the load-bearing half. Guarding the DELETE alone
-    -- is not enough: ReplacingMergeTree collapses on the sorting key, so an imported row
-    -- and a shipped row sharing (session_id, user_id, seq) are the SAME row and the newer
-    -- ingested_at wins. Measured: 3 imported + 2 shipped rows became 3, and the two
-    -- imported ones the shipper happened to overlap were gone. Same loss as the
-    -- unguarded delete, reached through the merge instead.
+    -- Who put this row here, and which parse it belongs to -- both in the sorting key,
+    -- for the reasons written out at length on the messages room above. The tool room
+    -- shrinks the same way a transcript does: a re-parse with fewer assistant turns emits
+    -- fewer tool calls, and the old higher-idx rows have nothing written over them.
     origin LowCardinality(String) DEFAULT 'ship',
+    epoch UInt32 DEFAULT 0,
     user_id String MATERIALIZED currentUser(),
     ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
 )
 ENGINE = ReplacingMergeTree(ingested_at)
-ORDER BY (session_id, user_id, origin, idx);
+ORDER BY (session_id, user_id, origin, epoch, idx);
 
 -- NO sessions_v HERE, deliberately. The session rollup is a SAVED QUERY substituted
 -- with these room names and run under the caller's own credential (see rooms.js
 -- sessionsRollup). A stored view would need a name inside the `^sessions_` namespace
 -- the Merge rooms select on, a fourth grant, and a fourth object to provision and roll
 -- forward — for nothing the query does not already do.
+
+-- ─────────────────────────────────────────────────────────────────────────────────
+-- The house's own record of itself. Not rooms: nothing about a conversation lives
+-- here, and the shipper's guards do not require them (a member on someone else's
+-- house may hold no rights to create them, and shipping must not depend on it).
+--
+-- The house needs these because it now has STATE THE CLIENT CANNOT DERIVE. Sorting
+-- keys cannot be altered, so a schema change is a rebuild — and a rebuild has a
+-- middle, an actor, and an outcome. "Which version is this house at, is a migration
+-- half-done, and who did what" was previously answerable only by reading DDL and
+-- guessing.
+--
+-- Live facts are NOT copied here. Which rooms exist, which users hold which grants,
+-- how big each room is: `system.tables`, `system.columns` and `system.users` are
+-- authoritative and always current, and a mirror of them would be a second truth that
+-- goes stale. What is recorded is what those cannot say: intent, sequence, outcome.
+
+CREATE TABLE IF NOT EXISTS house_meta
+(
+    -- House-wide keys only ('schema_version', 'house_created_at'). Anything per-member
+    -- or per-machine belongs in house_events, which is append-only — this room is
+    -- latest-wins, so two members writing one key would overwrite each other.
+    key String,
+    value String,
+    updated_at DateTime64(3, 'UTC') DEFAULT now64(3),
+    updated_by String MATERIALIZED currentUser(),
+    host LowCardinality(String) DEFAULT ''
+)
+ENGINE = ReplacingMergeTree(updated_at)
+ORDER BY (key);
+
+CREATE TABLE IF NOT EXISTS house_events
+(
+    event_at DateTime64(3, 'UTC') DEFAULT now64(3),
+    -- 'migration' — a room rebuild, one row per transition (pending → applied|failed)
+    -- 'version'   — a memhouse version seen writing to this house, when it changes
+    -- 'schema'    — the house's schema version being set or moved
+    kind LowCardinality(String),
+    id String DEFAULT '',                      -- migration id, e.g. '0100-epoch-key'
+    status LowCardinality(String) DEFAULT '',  -- pending | applied | failed | observed
+    from_version String DEFAULT '',
+    to_version String DEFAULT '',
+    actor String MATERIALIZED currentUser(),
+    host LowCardinality(String) DEFAULT '',
+    rows_before UInt64 DEFAULT 0,
+    rows_after UInt64 DEFAULT 0,
+    detail String DEFAULT ''
+)
+-- Plain MergeTree, and nothing ever updates a row: the history IS the table, and the
+-- current state of a migration is argMax(status, event_at) over its id. A migration
+-- that failed and was retried should read as exactly that, not as a single row whose
+-- earlier attempts were overwritten.
+ENGINE = MergeTree
+ORDER BY (event_at, kind, id);

package/memhouse/self-update.js

@@ -6,10 +6,10 @@
 // of the CLI you just typed, not the version the daemon is running. On macminim a shipper
 // ran 1d16h out of a directory that had been MOVED — the fix was live, the machine was not.
 //
-// npm's own `postinstall` is the obvious hook and the wrong one. Install scripts are
-// blocked by default (`--allow-scripts`), which is the very condition this product already
-// has to work around, and a package install that restarts a user's daemons is a surprise
-// even when it works.
+// npm's own `postinstall` is the obvious hook and the wrong one. npm >= 12 blocks install
+// scripts by default, so it would not run at all on a stock install — the same default
+// that cost this product its SQLite binding until `node:sqlite` removed the need for one —
+// and a package install that restarts a user's daemons is a surprise even when it works.
 //
 // So the daemon notices for itself. Once per loop pass it compares what is on disk against
 // what it booted with, and when they differ it hands over:

package/memhouse/service.js

@@ -71,9 +71,14 @@ Wants=network-online.target
 
 [Service]
 Type=simple
+# Restart=always, not on-failure: the shipper EXITS 0 ON PURPOSE when an upgrade replaces
+# the files underneath it (self-update.js, supervised path) and counts on the supervisor
+# to start the new version. Under on-failure that handover left the service dead after
+# every npm upgrade — found on testbed, service inactive(dead) status=0/SUCCESS.
+# RestartSec plus self-update's re-exec chain cap guard against restart loops.
 ${envLines(env)}
 ExecStart=${[node, script, ...args].map(execToken).join(' ')}
-Restart=on-failure
+Restart=always
 RestartSec=30
 StandardOutput=append:${path.join(logDir, logName)}
 StandardError=append:${path.join(logDir, logName)}
@@ -104,7 +109,7 @@ ${envEntries}
   </dict>
   <key>RunAtLoad</key><true/>
   <key>KeepAlive</key>
-  <dict><key>SuccessfulExit</key><false/></dict>
+  <true/>
   <key>StandardOutPath</key><string>${xml(path.join(logDir, logName))}</string>
   <key>StandardErrorPath</key><string>${xml(path.join(logDir, logName))}</string>
 </dict>