memhouse 0.18.9 → 0.18.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/AGENTS.md +194 -0
  2. package/CHANGELOG.md +1097 -0
  3. package/README.md +54 -423
  4. package/SECURITY.md +68 -0
  5. package/bin/memhouse.js +299 -133
  6. package/docs/README.md +46 -0
  7. package/docs/guides/admin-password.md +95 -0
  8. package/docs/guides/configuration.md +109 -0
  9. package/docs/guides/daemons.md +53 -0
  10. package/docs/guides/install.md +128 -0
  11. package/docs/guides/invite.md +87 -0
  12. package/docs/guides/operate.md +112 -0
  13. package/docs/guides/search-and-resume.md +95 -0
  14. package/docs/guides/share.md +102 -0
  15. package/docs/guides/storage.md +93 -0
  16. package/docs/guides/troubleshooting.md +78 -0
  17. package/docs/guides/uninstall.md +37 -0
  18. package/docs/guides/update.md +68 -0
  19. package/docs/releases/README.md +11 -0
  20. package/docs/releases/v0.18.10.md +226 -0
  21. package/docs/releases/v0.18.11.md +191 -0
  22. package/docs/tutorials/01-first-house.md +132 -0
  23. package/docs/tutorials/02-join-a-house.md +91 -0
  24. package/docs/tutorials/03-share-your-memory.md +96 -0
  25. package/editors/claude.js +68 -10
  26. package/examples/01-local-one-machine/README.md +99 -0
  27. package/examples/01-local-one-machine/run.sh +61 -0
  28. package/examples/02-existing-clickhouse/README.md +121 -0
  29. package/examples/02-existing-clickhouse/run.sh +71 -0
  30. package/examples/03-team-house/README.md +129 -0
  31. package/examples/03-team-house/run.sh +124 -0
  32. package/examples/04-self-hosted-team/.env.example +22 -0
  33. package/examples/04-self-hosted-team/README.md +187 -0
  34. package/examples/04-self-hosted-team/compose.yml +158 -0
  35. package/examples/04-self-hosted-team/smoke.sh +113 -0
  36. package/examples/README.md +42 -0
  37. package/examples/fixtures/README.md +24 -0
  38. package/examples/fixtures/make-sessions.js +63 -0
  39. package/memhouse/DESIGN.md +9 -2
  40. package/memhouse/admin-secret.js +81 -0
  41. package/memhouse/delivery/plugin/.claude-plugin/plugin.json +1 -1
  42. package/memhouse/delivery/plugin/reference/HOUSE.md +18 -6
  43. package/memhouse/delivery/plugin/skills/access/SKILL.md +1 -1
  44. package/memhouse/delivery/plugin/skills/admin/SKILL.md +1 -1
  45. package/memhouse/delivery/plugin/skills/house/SKILL.md +1 -1
  46. package/memhouse/delivery/plugin/skills/recall/SKILL.md +34 -14
  47. package/memhouse/delivery/plugin/skills/sql/SKILL.md +1 -1
  48. package/memhouse/deploy.js +4 -2
  49. package/memhouse/flags.js +32 -5
  50. package/memhouse/host.js +11 -6
  51. package/memhouse/house/HOUSE.md +1 -1
  52. package/memhouse/house/house.js +27 -1
  53. package/memhouse/invite-target.js +37 -0
  54. package/memhouse/provision.js +12 -3
  55. package/memhouse/server/queries.js +9 -4
  56. package/memhouse/service.js +5 -0
  57. package/memhouse/shipper/ship.js +148 -31
  58. package/package.json +10 -2
  59. package/public/THIRD_PARTY_NOTICES.txt +2114 -0
  60. package/ui/vite.config.js +4 -1
package/memhouse/host.js CHANGED
@@ -14,13 +14,18 @@
14
14
  // * One machine splits. Rename the box and every row shipped afterwards lands under a
15
15
  // new host, so the machine's own history appears to end and a stranger's to begin.
16
16
  //
17
- // A random fingerprint, written once and kept, fixes both: nothing derives it, so nothing
18
- // can collide with it, and nothing about the machine changing can move it.
17
+ // So the fingerprint comes from the OS's own machine id (IOPlatformUUID, /etc/machine-id,
18
+ // MachineGuid), hashed with a fixed salt (machineFingerprint), never from the hostname: two
19
+ // machines cannot share it and a rename cannot move it. Since 0.18.6. Before that it was
20
+ // random, which fixed both of the above but split a machine whenever MEMHOUSE_HOME was
21
+ // wiped: a re-imaged laptop came back as a stranger. Where no machine id can be read it is
22
+ // still random.
19
23
  //
20
- // It lives in MEMHOUSE_HOME and is deliberately NOT removed by a plain `memhouse
21
- // uninstall` — reinstalling on the same machine should continue that machine's history,
22
- // not start a second one beside it. `uninstall --full-removal` drops it, and then the
23
- // machine genuinely is a new host next time; that is the honest meaning of full removal.
24
+ // It is written once to MEMHOUSE_HOME/host.json and an existing file always wins, so no
25
+ // install changes identity under it. `uninstall --full-removal` drops the file, and the
26
+ // next install derives the same fingerprint again from the same machine. The id also
27
+ // carries the hostname the machine had at first install, so it comes back the same unless
28
+ // the machine was renamed in between, or has no readable machine id.
24
29
 
25
30
  const fs = require('fs');
26
31
  const os = require('os');
@@ -17,7 +17,7 @@ Two columns say where every row came from:
17
17
  | Column | Meaning | Comes from |
18
18
  |---|---|---|
19
19
  | `user_id` | who wrote it | `String MATERIALIZED currentUser()` — stamped by the server, unforgeable by clients. `async_insert = 0 CONST` is pinned on each user (`ADD SETTING`, which merges — a bare `SETTINGS` clause replaces the user's whole list) because an async flush stores the stamp as the empty string (measured on 25.11.9.34) |
20
- | `host` | which machine | the install's fingerprint (`../host.js`) — random, minted once, so two machines with the same hostname cannot collide and renaming one cannot split its history |
20
+ | `host` | which machine | the machine's fingerprint (`../host.js`) — derived from the OS machine id (random only where none can be read), so two machines with the same hostname cannot collide and renaming one cannot split its history |
21
21
 
22
22
  `WHERE user_id = 'alice'` is one person. `WHERE host = 'macbook-4127a95b'` is one
23
23
  machine. No filter is the whole house — which is what a team dashboard wants.
@@ -8,7 +8,7 @@
8
8
  // clients (async_insert is pinned to 0
9
9
  // on the user so the stamp cannot be
10
10
  // skipped)
11
- // host LowCardinality(String) — the machine's fingerprint, minted once
11
+ // host LowCardinality(String) — the machine's fingerprint, derived from its OS machine id
12
12
  // per install (../host.js)
13
13
  //
14
14
  // That pair IS the provenance model.
@@ -412,6 +412,31 @@ function statRefreshStatements(rooms, at = 'now64(3)') {
412
412
  `INSERT INTO ${names[k]} SELECT ${at} AS refreshed_at, * FROM (${bodies[k].body})`);
413
413
  }
414
414
 
415
+ /**
416
+ * When the shipper refreshes the stat tables (G11). After every pass that shipped something,
417
+ * as before — and ALSO when the newest generation is older than STATS_MAX_AGE_MS, even if
418
+ * nothing shipped. The tables carry a one-day TTL, so a house whose machines went quiet for
419
+ * a day used to lose every generation and the dashboard went empty while the rooms still
420
+ * held everything. `ageMs` null means "could not tell" (no table, no rights): do not refresh
421
+ * on age then; the read layer is already on the inline rollup.
422
+ */
423
+ const STATS_MAX_AGE_MS = 60 * 60 * 1000;
424
+ function statsNeedRefresh({ shipped, ageMs, maxAgeMs = STATS_MAX_AGE_MS }) {
425
+ if (shipped > 0) return true;
426
+ return ageMs !== null && ageMs !== undefined && ageMs > maxAgeMs;
427
+ }
428
+
429
+ /**
430
+ * How long a reader may keep its resolved rooms (G11). resolveRooms() decides once whether
431
+ * the stat tables are usable or the inline rollup must answer; a long-running dashboard that
432
+ * never asked again kept reading a stat table after its TTL had emptied it. Re-resolving
433
+ * every few minutes costs one small round trip and lets it follow the house either way.
434
+ */
435
+ const ROOMS_TTL_MS = 5 * 60 * 1000;
436
+ function roomsCacheStale(resolvedAt, now = Date.now(), ttlMs = ROOMS_TTL_MS) {
437
+ return !resolvedAt || now - resolvedAt > ttlMs;
438
+ }
439
+
415
440
  /** The read-side source for one stat table: its newest generation, and nothing else. */
416
441
  function latestGeneration(table) {
417
442
  return `(SELECT * FROM ${table} WHERE refreshed_at = (SELECT max(refreshed_at) FROM ${table}))`;
@@ -598,4 +623,5 @@ module.exports = {
598
623
  sessionsRollup, currentParse, createStatement, roomNames, currentUser, resolveRooms,
599
624
  SESSION_STATS, MODEL_STATS, TOOL_STATS, statTableNames, hasSessionStats, statTables,
600
625
  statCreateStatements, statRefreshStatements, latestGeneration,
626
+ STATS_MAX_AGE_MS, statsNeedRefresh, ROOMS_TTL_MS, roomsCacheStale,
601
627
  };
@@ -0,0 +1,37 @@
1
+ // Where `invite` may send a credential — and the one place that decides it (G4).
2
+ //
3
+ // `--url` is the address the INVITEE will reach the house at. It goes into the invite file;
4
+ // it is typed on a command line, so it can be a typo, a tunnel, or somebody else's server.
5
+ // A credential this install KEEPS — the member's own, or an admin saved with --keep-admin —
6
+ // belongs to the house it was saved for, and is only ever sent there. Before 0.18.11 the
7
+ // stored credential was sent to `--url`: resolveConfig() lets the flag outrank the env file,
8
+ // so `cfg.url` WAS `--url`, and the reachability probe, the grants read and the provisioning
9
+ // all authenticated against whatever `--url` named.
10
+ //
11
+ // A credential given for THIS command — `--admin-user`, or an exported MEMHOUSE_ADMIN_USER,
12
+ // with a password from a file, stdin, the environment or a prompt — was chosen by the person
13
+ // who also typed `--url`, so it provisions there, as it always has.
14
+ //
15
+ // Pure, so it is unit tested.
16
+
17
+ /**
18
+ * @param {object} o
19
+ * @param {string} o.url the invitee-facing --url
20
+ * @param {string|null} o.storedUrl the house this install is configured for (env or env file)
21
+ * @param {'flag'|'env'|'file'|'member'} o.adminUserSource
22
+ * where the credential's USER came from: --admin-user, an exported MEMHOUSE_ADMIN_USER,
23
+ * the env file's MEMHOUSE_ADMIN_USER, or (no admin at all) the member credential
24
+ * @param {string|null} [o.passwordSource] resolveAdminPassword's `source` ('stored' = the env file)
25
+ * @returns {{ url: string, stored: boolean } | { error: string }}
26
+ */
27
+ function provisionTarget({ url, storedUrl, adminUserSource, passwordSource = null }) {
28
+ const stored = adminUserSource === 'file' || adminUserSource === 'member' || passwordSource === 'stored';
29
+ if (!stored) return { url, stored: false };
30
+ if (!storedUrl) {
31
+ return { error: 'this install keeps no house URL to send its stored credential to — '
32
+ + 'give the admin for this command instead (--admin-user, password from MEMHOUSE_ADMIN_PASSWORD or --admin-password-file)' };
33
+ }
34
+ return { url: storedUrl, stored: true };
35
+ }
36
+
37
+ module.exports = { provisionTarget };
@@ -54,13 +54,22 @@ function sqlString(v) {
54
54
  *
55
55
  * @returns {Array<{sql:string, why:string, optional:boolean}>}
56
56
  */
57
- function plan({ db, member, password = null }) {
57
+ // What a PRINTED plan carries where the password goes. Deliberately not a quoted string:
58
+ // run as printed, `IDENTIFIED BY <member-password>` does not parse, so nobody can create a
59
+ // member whose password is the placeholder text. A printed plan never carries a real
60
+ // password: printed output ends up in terminals, scrollback and agent transcripts, and
61
+ // memhouse ships transcripts into the house (G23).
62
+ const PASSWORD_PLACEHOLDER = '<member-password>';
63
+
64
+ function plan({ db, member, password = null, placeholder = false }) {
58
65
  assertUsableName(db, 'house');
59
66
  assertUsableName(member, 'member');
60
67
  const steps = [
61
68
  { sql: `CREATE DATABASE IF NOT EXISTS ${db}`, why: 'the house', optional: false },
62
69
  ];
63
- if (password !== null) {
70
+ if (placeholder) {
71
+ steps.push({ sql: `CREATE USER ${member} IDENTIFIED BY ${PASSWORD_PLACEHOLDER}`, why: `the member — replace ${PASSWORD_PLACEHOLDER} with a quoted password you generate, e.g. '…' from \`openssl rand -hex 16\`; as printed this line does not parse, on purpose`, optional: false });
72
+ } else if (password !== null) {
64
73
  // No IF NOT EXISTS. The live path checks first and refuses an existing user; the printed
65
74
  // plan used to no-op here, apply the grants, and hand out a password the server never saw —
66
75
  // found on a realm that mints its own users. Now it fails loudly on line one instead.
@@ -100,4 +109,4 @@ function render(steps, { db, member }) {
100
109
  return out.join('\n');
101
110
  }
102
111
 
103
- module.exports = { ROOM_PRIVILEGES, roomGrant, plan, render };
112
+ module.exports = { ROOM_PRIVILEGES, PASSWORD_PLACEHOLDER, roomGrant, plan, render };
@@ -18,7 +18,7 @@
18
18
 
19
19
  const { createClient, ClickHouseLogLevel } = require('@clickhouse/client');
20
20
  const { calculateCost, normalizeModelName } = require('../../pricing');
21
- const { resolveRooms, READ_SETTINGS } = require('../house/house');
21
+ const { resolveRooms, READ_SETTINGS, roomsCacheStale } = require('../house/house');
22
22
 
23
23
  const config = {
24
24
  url: process.env.MEMHOUSE_URL || 'http://localhost:8123',
@@ -90,11 +90,16 @@ const TZ = Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
90
90
  // braces, no type.
91
91
  const ROOM_TOKEN = /\{\{([a-z_]+)\}\}/g;
92
92
  let roomsPromise = null;
93
+ let roomsAt = 0;
93
94
 
94
95
  function rooms() {
95
- // Resolved once per process. resolveRooms asks the server for currentUser(), so it is a
96
- // round trip, and every dashboard request would otherwise pay for it.
97
- if (!roomsPromise) {
96
+ // Cached, because resolveRooms asks the server for currentUser() and which stat tables
97
+ // are usable, and every dashboard request would otherwise pay that round trip. But not
98
+ // for the life of the process: whether the stat tables can answer changes — their TTL
99
+ // empties them on a quiet house, a refresh fills them again — and a dashboard that
100
+ // decided once kept reading an emptied table and showed nothing (G11).
101
+ if (!roomsPromise || roomsCacheStale(roomsAt)) {
102
+ roomsAt = Date.now();
98
103
  roomsPromise = resolveRooms(getClient()).catch((e) => { roomsPromise = null; throw e; });
99
104
  }
100
105
  return roomsPromise;
@@ -190,6 +190,11 @@ function install({ shipJs, envFile, logDir, interval = 300, home = null, force =
190
190
  // another member's rooms — which either fails on permissions or, if it can read them,
191
191
  // quietly ships into the wrong place.
192
192
  const env = parseEnvFile(envFile);
193
+ // The shipper never uses the admin credential, and the unit inlines every key it is
194
+ // given — a kept admin password would land in a second file and in the daemon's
195
+ // environment for its whole life.
196
+ delete env.MEMHOUSE_ADMIN_USER;
197
+ delete env.MEMHOUSE_ADMIN_PASSWORD;
193
198
  // MEMHOUSE_HOME is not in the env file — it is where the env file itself lives — and a
194
199
  // service installed under a custom home must keep using it, or it reads a different
195
200
  // config after a reboot than the one just written.
@@ -66,7 +66,7 @@ const selfSnap = selfUpdate.snapshot(__filename);
66
66
  const {
67
67
  resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN, createStatement,
68
68
  SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor,
69
- statCreateStatements, statRefreshStatements, statTableNames,
69
+ statCreateStatements, statRefreshStatements, statTableNames, statsNeedRefresh,
70
70
  } = require('../house/house');
71
71
  // Rooms plus the house's own record of itself. Every table the template declares, which
72
72
  // is what the column healer and the drift warning have to cover — a column added to
@@ -109,7 +109,7 @@ function chTs(ms) {
109
109
 
110
110
  // Whether the adapter gave this message its own timestamp, or the shipper has to
111
111
  // interpolate one. The distinction matters beyond accuracy: an interpolated ts is a
112
- // function of `seq / (total - 1)`, so EVERY already-stored row's ts moves when the
112
+ // function of `pos / (total - 1)` (position in the parse), so EVERY already-stored row's ts moves when the
113
113
  // session grows. That is what forces a session with no per-message timestamps to
114
114
  // re-ship whole rather than tail-only. See rowsForChat's `tsInterpolated` and tailSafe.
115
115
  function hasOwnTs(msg) {
@@ -127,12 +127,12 @@ function hasOwnTs(msg) {
127
127
  // and the search skill reported those to the user as fact.
128
128
  //
129
129
  // Fall back to interpolating across [createdAt, lastUpdatedAt] for the adapters that
130
- // genuinely have nothing per message — monotonic-by-seq, so sessions_v started/ended
130
+ // genuinely have nothing per message — monotonic by position, so sessions_v started/ended
131
131
  // still line up with the session bounds (same approximation as agency/ingest.js;
132
132
  // documented in DESIGN.md). Real timestamps are NOT necessarily monotonic by seq — a
133
133
  // folded subagent transcript is appended after its parent's turns but ran during them.
134
134
  // Nothing depends on that: ordering is by `seq`, and started/ended are min/max.
135
- function messageTs(chat, seq, total, msg) {
135
+ function messageTs(chat, pos, total, msg) {
136
136
  const at = Number(msg && msg._ts);
137
137
  // Seconds or milliseconds, depending on the format: goose stores seconds, Claude Code
138
138
  // and opencode milliseconds. Real epoch-ms is > 1e12 and real epoch-seconds ~1.7e9, so
@@ -141,7 +141,9 @@ function messageTs(chat, seq, total, msg) {
141
141
  const start = chat.createdAt || chat.lastUpdatedAt || Date.now();
142
142
  const end = chat.lastUpdatedAt || chat.createdAt || start;
143
143
  if (total <= 1) return chTs(start);
144
- return chTs(start + Math.round((end - start) * (seq / (total - 1))));
144
+ // `pos` is the message's position in the parse, NOT its seq: a folded subagent's seq
145
+ // is sparse (see subagentSeq) and would put its interpolated time far past the session.
146
+ return chTs(start + Math.round((end - start) * (pos / (total - 1))));
145
147
  }
146
148
 
147
149
  /**
@@ -875,6 +877,42 @@ async function loadExisting(client, rooms) {
875
877
  // m._toolCalls: matches '[tool-call: Name(' and '[tool-call: Name]' forms.
876
878
  const TOOL_CALL_RE = /\[tool-call: ([^(\]]+)/g;
877
879
 
880
+ // Where a folded subagent's rows are numbered. A session's own turns keep the dense
881
+ // 0, 1, 2, … they always had; subagent number `slot` (see getMessages in
882
+ // editors/claude.js) owns the block
883
+ //
884
+ // seq = SUBAGENT_SEQ_BASE + slot * SUBAGENT_SEQ_STRIDE + turn
885
+ //
886
+ // and its tool calls the same block of `idx`, counted within that subagent. So a row's
887
+ // seq depends only on the subagent it belongs to and its position inside it — never on
888
+ // how long the parent is, how long any other subagent is, or how many there are.
889
+ //
890
+ // It used to be the row's position in the whole parse, with subagents folded in after the
891
+ // parent. One new parent turn then moved every subagent row down by one; decideEpoch saw
892
+ // stored rows whose line_hash (which covers seq) no longer matched, forked the session to
893
+ // a new epoch, and re-sent it whole — on every pass, for every session that had ever
894
+ // spawned a subagent. Measured on a real house: 1.07 M message rows in 24 h against 10.2 M
895
+ // in total, most of them full copies of five sessions, one stored 60 times in a day.
896
+ //
897
+ // Decimal so a seq reads at a glance (1,003,000,042 = subagent 3, turn 42), and sized for
898
+ // UInt32: 1e9 parent turns, 32,949 subagents of 100,000 turns each (the largest sessions
899
+ // today hold ~33 k parent lines and 35 subagents). A parse that does not fit falls back to
900
+ // positional numbering, which is correct and merely re-ships whole, as before.
901
+ const SUBAGENT_SEQ_BASE = 1000000000;
902
+ const SUBAGENT_SEQ_STRIDE = 100000;
903
+ const SUBAGENT_SLOTS = Math.floor((2 ** 32 - SUBAGENT_SEQ_BASE) / SUBAGENT_SEQ_STRIDE);
904
+
905
+ // The block a folded subagent turn belongs to, or null for the session's own turns and for
906
+ // adapters that fold nothing.
907
+ function subagentSlot(m) {
908
+ const a = m && m._agent;
909
+ return a && Number.isInteger(a.slot) && Number.isInteger(a.turn) ? a.slot : null;
910
+ }
911
+ function subagentSeq(slot, n) {
912
+ if (slot < 0 || slot >= SUBAGENT_SLOTS || n < 0 || n >= SUBAGENT_SEQ_STRIDE) return null;
913
+ return SUBAGENT_SEQ_BASE + slot * SUBAGENT_SEQ_STRIDE + n;
914
+ }
915
+
878
916
  // Build the typed rows for one chat. Returns null when the chat is unreadable
879
917
  // (adapter threw — e.g. a partially-written or locked session file): the caller
880
918
  // must write NOTHING for it, so the next incremental pass retries. A chat that
@@ -891,6 +929,13 @@ function rowsForChat(chat, host) {
891
929
  adapterErrorSink.record(chat.source, e, chat.composerId);
892
930
  return null;
893
931
  }
932
+ // The adapter asked to wait a pass: what it read is not yet the session's settled shape
933
+ // (a subagent with no timestamped line — see editors/claude.js). Not an error; the next
934
+ // pass reads it again.
935
+ if (messages._defer) {
936
+ console.log(`[memhouse] ${chat.source}:${chat.composerId} held for one pass — ${messages._defer}`);
937
+ return null;
938
+ }
894
939
 
895
940
  const source = chat.source;
896
941
  // Canonical globally-unique session id: '<source>:<adapter-local id>'.
@@ -918,19 +963,47 @@ function rowsForChat(chat, host) {
918
963
  extra: { bubbleCount: chat.bubbleCount || 0 },
919
964
  };
920
965
 
966
+ // Stable numbering first; positional if the parse does not fit it (see subagentSeq).
967
+ const built = buildRows(chat, messages, { id, source, host, folder, project }, true)
968
+ || buildRows(chat, messages, { id, source, host, folder, project }, false);
969
+ // What a complete ship of this session looks like, recorded ON the session row so the
970
+ // next pass can tell "finished" from "got part way". message_count already carries the
971
+ // message half; without the tool half a pass that dies between the messages insert and
972
+ // the tool_calls insert leaves a session that looks finished forever. See loadExisting.
973
+ session.extra.toolCallCount = built.toolRows.length;
974
+ return { session, ...built };
975
+ }
976
+
977
+ // The message and tool_call rows for one parse. `stable` numbers folded subagent rows by
978
+ // their slot (subagentSeq) and returns null if any of them does not fit; otherwise every
979
+ // row is numbered by its position, the pre-0.18.10 scheme. For a session with no subagents
980
+ // the two are identical.
981
+ function buildRows(chat, messages, { id, source, host, folder, project }, stable) {
982
+ const total = messages.length;
921
983
  const msgRows = [];
922
984
  const toolRows = [];
923
- let idx = 0; // session-wide tool-call index
985
+ let seqNext = 0; // the session's own turns, dense from 0
986
+ let idx = 0; // the session's own tool calls, dense from 0
987
+ const subIdx = new Map(); // slot → tool calls seen so far in that subagent
924
988
  // Set as soon as one message has no timestamp of its own. One is enough: the
925
989
  // interpolation denominator is the SESSION's length, so a single interpolated row
926
990
  // means this session's stored timestamps drift the next time it grows.
927
991
  let tsInterpolated = false;
928
- for (let seq = 0; seq < total; seq++) {
929
- const m = messages[seq];
992
+ for (let pos = 0; pos < total; pos++) {
993
+ const m = messages[pos];
994
+ const slot = stable ? subagentSlot(m) : null;
995
+ let seq;
996
+ if (slot === null) {
997
+ if (stable && seqNext >= SUBAGENT_SEQ_BASE) return null;
998
+ seq = seqNext++;
999
+ } else {
1000
+ seq = subagentSeq(slot, m._agent.turn);
1001
+ if (seq === null) return null;
1002
+ }
930
1003
  if (!hasOwnTs(m)) tsInterpolated = true;
931
1004
  const raw = typeof m.content === 'string' ? m.content : JSON.stringify(m.content) ?? '';
932
1005
  const text = truncate(raw, TEXT_MAX);
933
- const ts = messageTs(chat, seq, total, m);
1006
+ const ts = messageTs(chat, pos, total, m);
934
1007
  const row = {
935
1008
  session_id: id,
936
1009
  seq,
@@ -973,10 +1046,18 @@ function rowsForChat(chat, host) {
973
1046
  }
974
1047
  for (const tc of calls) {
975
1048
  const args = JSON.stringify(tc.args);
1049
+ let at;
1050
+ if (slot === null) at = idx++;
1051
+ else {
1052
+ const n = subIdx.get(slot) || 0;
1053
+ subIdx.set(slot, n + 1);
1054
+ at = subagentSeq(slot, n);
1055
+ if (at === null) return null;
1056
+ }
976
1057
  toolRows.push({
977
1058
  session_id: id,
978
1059
  seq,
979
- idx: idx++,
1060
+ idx: at,
980
1061
  source,
981
1062
  host,
982
1063
  tool_name: tc.name,
@@ -987,12 +1068,7 @@ function rowsForChat(chat, host) {
987
1068
  });
988
1069
  }
989
1070
  }
990
- // What a complete ship of this session looks like, recorded ON the session row so the
991
- // next pass can tell "finished" from "got part way". message_count already carries the
992
- // message half; without the tool half a pass that dies between the messages insert and
993
- // the tool_calls insert leaves a session that looks finished forever. See loadExisting.
994
- session.extra.toolCallCount = toolRows.length;
995
- return { session, msgRows, toolRows, tsInterpolated };
1071
+ return { msgRows, toolRows, tsInterpolated };
996
1072
  }
997
1073
 
998
1074
  /**
@@ -1037,26 +1113,43 @@ function decideEpoch(stored, incoming) {
1037
1113
 
1038
1114
  const maxSeq = Number.isFinite(stored.maxSeq) ? stored.maxSeq : -1;
1039
1115
  const maxIdx = Number.isFinite(stored.maxIdx) ? stored.maxIdx : -1;
1040
- if (incoming.msgRows.length - 1 < maxSeq) {
1041
- return bump(`${maxSeq + 1} messages stored, ${incoming.msgRows.length} parsed`);
1116
+ const hashes = stored.hashes || new Map();
1117
+ const tools = stored.tools || new Map();
1118
+
1119
+ // SHRINK is "the house holds a row this parse does not produce", checked per key. It
1120
+ // used to be `parsed - 1 < maxSeq`, which is the same question only while seq is dense
1121
+ // 0..N-1; a folded subagent's rows are numbered from a sparse block (subagentSeq), so a
1122
+ // vanished subagent or a stale pre-0.18.10 numbering leaves stored keys inside the range
1123
+ // that nothing overwrites. The max comparison stays for a caller holding no hashes.
1124
+ const orphans = (have, rows, key, max) => {
1125
+ const want = new Set(rows.map((r) => r[key]));
1126
+ let missing = 0;
1127
+ for (const k of have.keys()) if (!want.has(k)) missing++;
1128
+ const inMax = rows.reduce((a, r) => Math.max(a, r[key]), -1);
1129
+ // How many the house holds, for the log line. `max + 1` is a count only while keys are
1130
+ // dense; with subagent blocks max is ~1e9, so it said "1000000123 messages stored".
1131
+ return { missing, beyond: max > inMax, stored: have.size || max + 1 };
1132
+ };
1133
+ const mo = orphans(hashes, incoming.msgRows, 'seq', maxSeq);
1134
+ if (mo.missing || mo.beyond) {
1135
+ return bump(`${mo.stored} messages stored, ${incoming.msgRows.length} parsed${mo.missing ? `, ${mo.missing} of them not in the new parse` : ''}`);
1042
1136
  }
1043
- if (incoming.toolRows.length - 1 < maxIdx) {
1044
- return bump(`${maxIdx + 1} tool calls stored, ${incoming.toolRows.length} parsed`);
1137
+ const to = orphans(tools, incoming.toolRows, 'idx', maxIdx);
1138
+ if (to.missing || to.beyond) {
1139
+ return bump(`${to.stored} tool calls stored, ${incoming.toolRows.length} parsed${to.missing ? `, ${to.missing} of them not in the new parse` : ''}`);
1045
1140
  }
1046
1141
 
1047
1142
  // A seq the house does not hold is a gap, not a disagreement — there is nothing there
1048
1143
  // to destroy, so it is not a reason to fork the session.
1049
- const hashes = stored.hashes || new Map();
1050
1144
  for (const row of incoming.msgRows) {
1051
- if (row.seq > maxSeq) break;
1145
+ if (row.seq > maxSeq) continue;
1052
1146
  const was = hashes.get(row.seq);
1053
1147
  if (was !== undefined && was !== String(row.line_hash)) {
1054
1148
  return bump(`message ${row.seq} was rewritten`);
1055
1149
  }
1056
1150
  }
1057
- const tools = stored.tools || new Map();
1058
1151
  for (const row of incoming.toolRows) {
1059
- if (row.idx > maxIdx) break;
1152
+ if (row.idx > maxIdx) continue;
1060
1153
  const was = tools.get(row.idx);
1061
1154
  if (was !== undefined && (was.tool_name !== row.tool_name || was.args !== row.args)) {
1062
1155
  return bump(`tool call ${row.idx} was rewritten`);
@@ -1072,7 +1165,7 @@ function decideEpoch(stored, incoming) {
1072
1165
  * token counts, text. Three shipped columns sit outside it and are not derived from
1073
1166
  * anything inside it, so a row can be stale in a way no hash comparison can see:
1074
1167
  *
1075
- * ts. Interpolated across [createdAt, lastUpdatedAt] as `seq / (total - 1)` for the
1168
+ * ts. Interpolated across [createdAt, lastUpdatedAt] as `pos / (total - 1)` (position in the parse, not seq) for the
1076
1169
  * adapters that record no per-message time. The denominator is the session's length,
1077
1170
  * so every stored row's ts moves the moment the session grows. Adapters that DO supply
1078
1171
  * `_ts` (Claude Code, opencode, goose, …) are exempt — their timestamps are facts, not
@@ -1084,8 +1177,11 @@ function decideEpoch(stored, incoming) {
1084
1177
  * folder / project. Denormalized onto every message and tool_call row; a moved or
1085
1178
  * renamed project has to reach the rows that carry it.
1086
1179
  *
1087
- * `source` cannot change (it is the session_id prefix), `is_subagent` is derived from
1088
- * `text`, and messages' `extra` is always empty — so those three need no check.
1180
+ * `source` cannot change (it is the session_id prefix) and `is_subagent` is derived from
1181
+ * `text`, so neither needs a check. Messages' `extra` carries a folded subagent's
1182
+ * `agent` stamp; its id and turn are fixed by the row's seq, and a description the parent
1183
+ * supplies only later (a background fork named by its task notification) is not worth
1184
+ * re-shipping a session for.
1089
1185
  *
1090
1186
  * When this returns false the session ships whole, exactly as it always has. Pure.
1091
1187
  */
@@ -1169,6 +1265,24 @@ async function loadStoredParse(client, rooms, id, uid, epoch) {
1169
1265
  return { hashes, tools };
1170
1266
  }
1171
1267
 
1268
+ /**
1269
+ * How old the newest stats generation is, in ms. Infinity when the table is empty (a TTL
1270
+ * emptied it, or no refresh ever landed); null when it cannot be read at all, so a house
1271
+ * without the tables is not refreshed on every idle pass. Cheap: count() is metadata and
1272
+ * refreshed_at leads the sorting key.
1273
+ */
1274
+ async function statsAgeMs(client, rooms) {
1275
+ try {
1276
+ const t = statTableNames(rooms.member).session_stats;
1277
+ const rs = await client.query({
1278
+ query: `SELECT count() AS n, dateDiff('second', max(refreshed_at), now64(3)) AS age FROM ${t}`,
1279
+ format: 'JSONEachRow',
1280
+ });
1281
+ const [r] = await rs.json();
1282
+ return Number(r.n) > 0 ? Number(r.age) * 1000 : Infinity;
1283
+ } catch { return null; }
1284
+ }
1285
+
1172
1286
  /**
1173
1287
  * Recompute the three stat tables — one generation, one INSERT each, all stamped with the
1174
1288
  * same refreshed_at. Runs at the end of every ship pass, which is exactly the cadence a
@@ -1358,9 +1472,12 @@ async function runShip(client, opts = {}) {
1358
1472
  for (const r of send.toolRows) { r.epoch = epoch; await push('tool_calls', scrub(r)); toolRows++; }
1359
1473
  }
1360
1474
  for (const table of Object.keys(batches)) await flush(table);
1361
- // The dashboard's tables, recomputed from what this pass just wrote. Skipped on a pass
1362
- // that shipped nothing: the previous generation is still exact.
1363
- if (sessions > 0) await refreshStats(client, rooms);
1475
+ // The dashboard's tables, recomputed from what this pass just wrote. A pass that shipped
1476
+ // nothing leaves an exact generation alone — unless it is getting old: the tables carry a
1477
+ // one-day TTL, and a quiet house used to age out of its dashboard entirely (G11).
1478
+ if (statsNeedRefresh({ shipped: sessions, ageMs: sessions > 0 ? 0 : await statsAgeMs(client, rooms) })) {
1479
+ await refreshStats(client, rooms);
1480
+ }
1364
1481
  // Anything that only failed while reading messages — the sink is reset by the
1365
1482
  // next getAllChats(), so unreported here means never reported at all.
1366
1483
  reportAdapterErrors(warned);
@@ -1541,7 +1658,7 @@ async function main() {
1541
1658
  }
1542
1659
  }
1543
1660
 
1544
- module.exports = { runShip, ensureSchema, refreshStats, templateColumns, decideEpoch, tailRows, tailSafe };
1661
+ module.exports = { runShip, ensureSchema, refreshStats, templateColumns, decideEpoch, tailRows, tailSafe, rowsForChat, buildRows, SUBAGENT_SEQ_BASE, SUBAGENT_SEQ_STRIDE };
1545
1662
 
1546
1663
  if (require.main === module) {
1547
1664
  main().catch((e) => {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "memhouse",
3
- "version": "0.18.9",
3
+ "version": "0.18.11",
4
4
  "description": "memhouse \u2014 agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, kernel-agency install, and the agentlytics dashboard",
5
5
  "main": "bin/memhouse.js",
6
6
  "bin": {
@@ -20,7 +20,15 @@
20
20
  "ui/vite.config.js",
21
21
  "ui/eslint.config.js",
22
22
  "public/",
23
- "README.md"
23
+ "README.md",
24
+ "docs/README.md",
25
+ "docs/tutorials/",
26
+ "docs/guides/",
27
+ "docs/releases/",
28
+ "examples/",
29
+ "AGENTS.md",
30
+ "SECURITY.md",
31
+ "CHANGELOG.md"
24
32
  ],
25
33
  "scripts": {
26
34
  "start": "node bin/memhouse.js",