memhouse 0.18.1 → 0.18.3

package/memhouse/shipper/passlog.js

@@ -0,0 +1,37 @@
+// What the shipper's log says about its LAST pass.
+//
+// A daemon whose every pass fails is "running" by pid. A drill member's shipper failed
+// auth for twelve minutes while `status` said "shipper: running" and `doctor` was green,
+// and she — and her agent — read an empty house as "still catching up". The log beside
+// it had the answer on every pass. This reads it. Pure; the CLI reads the file.
+//
+// Lines have no timestamps, so "last" is textual order: the later of the last success
+// line and the last failure line wins.
+
+const OK = /\[memhouse\] shipped (\d+) sessions[^\n]*?→ (\d+) msg rows/;
+const FAIL = /\[memhouse\] pass failed: ([^\n]*)/;
+
+/** @returns {{outcome:'ok'|'failed'|null, sessions?:number, rows?:number, why?:string, auth?:boolean}} */
+function lastPass(logText) {
+  const text = String(logText || '');
+  let lastOk = -1; let lastFail = -1; let okM = null; let failM = null;
+  for (const m of text.matchAll(new RegExp(OK.source, 'g'))) { lastOk = m.index; okM = m; }
+  for (const m of text.matchAll(new RegExp(FAIL.source, 'g'))) { lastFail = m.index; failM = m; }
+  if (lastOk < 0 && lastFail < 0) return { outcome: null };
+  if (lastFail > lastOk) {
+    const why = failM[1].trim();
+    return { outcome: 'failed', why, auth: /Authentication failed|ACCESS_DENIED|Not enough privileges/i.test(why) };
+  }
+  return { outcome: 'ok', sessions: Number(okM[1]), rows: Number(okM[2]) };
+}
+
+/** One line for a person: null when there is nothing to say (no passes yet, or the last one was fine). */
+function describeFailure(lp) {
+  if (!lp || lp.outcome !== 'failed') return null;
+  const why = lp.why.length > 90 ? `${lp.why.slice(0, 87)}…` : lp.why;
+  return lp.auth
+    ? `the last pass FAILED — ${why} — restart it: memhouse stop && memhouse start`
+    : `the last pass FAILED — ${why} — see: memhouse doctor`;
+}
+
+module.exports = { lastPass, describeFailure };

package/memhouse/shipper/ship.js

@@ -62,7 +62,9 @@ const selfUpdate = require('../self-update');
 const selfSnap = selfUpdate.snapshot(__filename);
 const {
   resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN, createStatement,
-  SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor } = require('../house/house');
+  SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor,
+  statCreateStatements, statRefreshStatements, statTableNames,
+} = require('../house/house');
 // Rooms plus the house's own record of itself. Every table the template declares, which
 // is what the column healer and the drift warning have to cover — a column added to
 // events would otherwise roll out to nobody.
@@ -463,6 +465,24 @@ async function ensureSchema(client) {
       throw e;
     }
   }
+  // The precomputed rollups — additive, so they belong here and not in a migration (see
+  // migrate.js's rule). Three plain tables the shipper fills after every pass; the read
+  // layer probes for them at startup and falls back to the inline rollup when they are
+  // absent or empty, so a denied CREATE here degrades to slow, never to wrong. Why they
+  // are shipper-filled and not materialized views: house.js, above statBodies.
+  for (const q of statCreateStatements(rooms)) {
+    try {
+      await client.command({ query: q, clickhouse_settings: { async_insert: 0 } });
+    } catch (e) {
+      const m = e && e.message ? e.message : String(e);
+      if (/Not enough privileges|ACCESS_DENIED/i.test(m)) { denied++; continue; }
+      if (/ALREADY_EXISTS|already exists/i.test(m)) { continue; }
+      throw e;
+    }
+  }
+  // First generation now, so install and update are fast at once instead of after the
+  // next pass. Best effort: a denied CREATE above means these fail too, quietly.
+  await refreshStats(client, rooms);
   if (denied) {
     // In a SHARED house this is the designed state, not a shortfall: the operator creates
     // each member's rooms during `invite` precisely so the member cannot — a member who
@@ -903,8 +923,11 @@ function rowsForChat(chat, host) {
       text,
       project,
       folder,
-      is_subagent: text.startsWith('[subagent]'),
-      extra: {},
+      is_subagent: text.startsWith('[subagent]') || Boolean(m._agent),
+      // Which subagent (Claude Code fork) a folded turn belongs to — id, the description
+      // the parent gave it, its type, and the turn's position within that subagent — so
+      // one fork's transcript is `WHERE extra.agent.id = '…' ORDER BY seq`. Not hashed.
+      extra: m._agent ? { agent: { id: String(m._agent.id || ''), description: String(m._agent.description || ''), type: String(m._agent.type || ''), turn: Number(m._agent.turn) || 0, ...(m._agent.workflow ? { workflow: String(m._agent.workflow) } : {}) } } : {},
     };
     // Hash the content fields only (not ts — it is synthesized and may drift).
     row.line_hash = lineHash({
@@ -1051,6 +1074,38 @@ async function loadStoredParse(client, rooms, id, uid, epoch) {
   return { hashes, tools };
 }
 
+/**
+ * Recompute the three stat tables — one generation, one INSERT each, all stamped with the
+ * same refreshed_at. Runs at the end of every ship pass, which is exactly the cadence a
+ * refreshable view would have had, and after --ensure-schema. Best effort: a house whose
+ * tables were never created (a denied CREATE) fails every INSERT here, and the read layer
+ * is already on the inline rollup.
+ */
+async function refreshStats(client, rooms) {
+  // One timestamp for all three, taken once — three now64() calls would be three
+  // generations, and the read layer would pair session_stats from one with model_stats
+  // from another.
+  let at;
+  try {
+    const rs = await client.query({ query: 'SELECT toString(now64(3)) AS t', format: 'JSONEachRow' });
+    at = `toDateTime64('${(await rs.json())[0].t}', 3, 'UTC')`;
+  } catch { return 0; }
+  let done = 0;
+  for (const q of statRefreshStatements(rooms, at)) {
+    try {
+      await client.command({ query: q, clickhouse_settings: { ...READ_SETTINGS, async_insert: 0 } });
+      done++;
+    } catch (e) {
+      const m = e && e.message ? e.message : String(e);
+      if (!/Not enough privileges|ACCESS_DENIED|UNKNOWN_TABLE|doesn't exist|does not exist/i.test(m)) {
+        console.error(`[memhouse] stats refresh: ${m.split('\n')[0]}`);
+      }
+      return done;
+    }
+  }
+  return done;
+}
+
 // One shipping pass. Incremental unless opts.full: a chat is skipped when the house
 // already has it at least as fresh (last_updated_at) and at least as large
 // (extra.bubbleCount) — both readable without parsing the chat.
@@ -1198,6 +1253,9 @@ async function runShip(client, opts = {}) {
     for (const r of rows.toolRows) { r.epoch = epoch; await push('tool_calls', scrub(r)); toolRows++; }
   }
   for (const table of Object.keys(batches)) await flush(table);
+  // The dashboard's tables, recomputed from what this pass just wrote. Skipped on a pass
+  // that shipped nothing: the previous generation is still exact.
+  if (sessions > 0) await refreshStats(client, rooms);
   // Anything that only failed while reading messages — the sink is reset by the
   // next getAllChats(), so unreported here means never reported at all.
   reportAdapterErrors(warned);
@@ -1251,7 +1309,7 @@ async function printStats(client) {
 
 async function main() {
   const argv = process.argv.slice(2);
-  const client = makeClient();
+  let client = makeClient();
   try {
     if (argv.includes('--ensure-schema')) {
       const n = await ensureSchema(client);
@@ -1289,6 +1347,7 @@ async function main() {
           `${r.msgRows} msg rows, ${r.toolRows} tool rows in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
       } catch (e) {
         failed = true;
+        var adoptedCredential = false;
         // A connection failure arrives as an AggregateError whose own `message` is empty,
         // so this printed "[memhouse] pass failed:" and nothing else, after 35 lines of
         // driver internals. Dig out a cause and name the fix; `status` and `doctor`
@@ -1299,7 +1358,22 @@ async function main() {
         if (/ECONNREFUSED|fetch failed|ENOTFOUND|EAI_AGAIN|socket hang up/i.test(why + causes.join(' '))) {
           console.error(`[memhouse] ${process.env.MEMHOUSE_URL} did not answer — is the house running?`);
         } else if (/Authentication failed|ACCESS_DENIED|Not enough privileges/i.test(why)) {
-          console.error(`[memhouse] the credential in ${process.env.MEMHOUSE_HOME || '~/.memhouse'}/env was rejected by ${process.env.MEMHOUSE_URL}`);
+          // This process was spawned with one credential; the env FILE may hold a newer one
+          // — `install --env` rotates the invite password and rewrites the file, and
+          // `memhouse passwd` can run under a live daemon. Retrying the dead copy for
+          // 300s a pass while the file beside it is right is what a drill member watched
+          // for twelve minutes. Adopt the file, rebuild the client, retry now.
+          const envPath = require('path').join(process.env.MEMHOUSE_HOME || require('path').join(require('os').homedir(), '.memhouse'), 'env');
+          let drift = { changed: [] };
+          try { drift = require('../envfile').credentialDrift(require('fs').readFileSync(envPath, 'utf-8'), process.env); } catch { /* no file: nothing to adopt */ }
+          if (drift.changed.length) {
+            for (const k of drift.changed) process.env[k] = drift.values[k];
+            client = makeClient();
+            adoptedCredential = true;
+            console.error(`[memhouse] ${envPath} holds a newer ${drift.changed.map((k) => k.replace('MEMHOUSE_', '').toLowerCase()).join('/')} — adopting it and retrying now`);
+          } else {
+            console.error(`[memhouse] the credential this shipper holds was rejected by ${process.env.MEMHOUSE_URL} (and ${envPath} says the same)`);
+          }
         } else if (/does not exist|UNKNOWN_TABLE|UNKNOWN_DATABASE/i.test(why)) {
           console.error('[memhouse] the house is missing a room this pass needed');
         }
@@ -1314,7 +1388,7 @@ async function main() {
       }
       full = false; // --full applies to the first pass only; loop passes stay incremental
       if (loop) {
-        const waitMs = failed ? Math.min(retryMs, intervalSec * 1000) : intervalSec * 1000;
+        const waitMs = failed ? (adoptedCredential ? 1000 : Math.min(retryMs, intervalSec * 1000)) : intervalSec * 1000;
         if (failed) {
           console.error(`[memhouse] retrying in ${Math.round(waitMs / 1000)}s`);
           retryMs = Math.min(retryMs * 2, intervalSec * 1000);
@@ -1336,7 +1410,7 @@ async function main() {
   }
 }
 
-module.exports = { runShip, ensureSchema, templateColumns, decideEpoch };
+module.exports = { runShip, ensureSchema, refreshStats, templateColumns, decideEpoch };
 
 if (require.main === module) {
   main().catch((e) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memhouse",
-  "version": "0.18.1",
+  "version": "0.18.3",
   "description": "memhouse \u2014 agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, kernel-agency install, and the agentlytics dashboard",
   "main": "bin/memhouse.js",
   "bin": {
@@ -28,7 +28,8 @@
     "dev": "cd ui && npm run dev",
     "test": "node misc/syntax-check.js && node misc/unit-test.js",
     "test:house": "node misc/house-test.js",
-    "prepack": "cd ui && npm install --no-audit --no-fund && npm run build"
+    "prepack": "cd ui && npm install --no-audit --no-fund && npm run build",
+    "prepublishOnly": "node misc/publish-guard.js"
   },
   "keywords": [
     "memory",