memhouse 0.4.4 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +244 -114
- package/bin/memhouse.js +1177 -231
- package/editors/base.js +6 -0
- package/editors/claude.js +44 -9
- package/editors/goose.js +4 -0
- package/editors/index.js +0 -0
- package/editors/opencode.js +3 -0
- package/{mem-house → memhouse}/COMPETITION.md +5 -5
- package/{mem-house → memhouse}/DESIGN.md +18 -12
- package/memhouse/RESUME-AND-UPDATE.md +185 -0
- package/{mem-house → memhouse}/delivery/AGENT-INSTALL-PROMPT.md +1 -1
- package/{mem-house → memhouse}/delivery/AGENT-INSTALL.md +9 -7
- package/{mem-house → memhouse}/delivery/PROMPT.md +8 -8
- package/{mem-house → memhouse}/delivery/kernel-install.md +3 -3
- package/{mem-house → memhouse}/delivery/plugin/.claude-plugin/plugin.json +1 -1
- package/{mem-house/delivery/skills/memhouse-search → memhouse/delivery/plugin/skills/search}/SKILL.md +60 -18
- package/memhouse/delivery/plugin/skills/sessions/SKILL.md +177 -0
- package/memhouse/delivery/plugin/skills/sql/SKILL.md +206 -0
- package/memhouse/host.js +101 -0
- package/{mem-house → memhouse}/per-member/INSTALL.md +59 -14
- package/{mem-house → memhouse}/per-member/SCHEMA.md +47 -0
- package/{mem-house → memhouse}/per-member/SHARING.md +47 -0
- package/memhouse/per-member/provision.js +297 -0
- package/memhouse/per-member/rooms.js +252 -0
- package/{mem-house → memhouse}/per-member/schema-member.sql.tpl +18 -14
- package/{mem-house → memhouse}/per-member/schema-merge.sql.tpl +3 -3
- package/memhouse/resume.js +150 -0
- package/memhouse/self-update.js +150 -0
- package/{mem-house → memhouse}/server/queries.js +80 -34
- package/memhouse/server/server.js +451 -0
- package/{mem-house → memhouse}/service.js +48 -29
- package/memhouse/shipper/ship.js +949 -0
- package/package.json +5 -20
- package/public/assets/{index-CtH-5i95.js → index-CnWzrjF3.js} +10 -7
- package/public/index.html +2 -2
- package/ui/index.html +1 -1
- package/ui/src/App.jsx +1 -1
- package/ui/src/lib/api.js +29 -13
- package/ui/src/pages/Dashboard.jsx +8 -5
- package/ui/src/pages/Sessions.jsx +3 -1
- package/ui/src/pages/SqlViewer.jsx +6 -23
- package/cache.js +0 -850
- package/ch-ingest.js +0 -329
- package/ch-schema.js +0 -152
- package/clickhouse.js +0 -87
- package/index.js +0 -425
- package/mcp-server.js +0 -294
- package/mem-house/delivery/install.sh +0 -183
- package/mem-house/delivery/skills/memhouse-sessions/SKILL.md +0 -125
- package/mem-house/delivery/skills/memhouse-sql/SKILL.md +0 -146
- package/mem-house/per-member/provision.js +0 -142
- package/mem-house/per-member/rooms.js +0 -132
- package/mem-house/server/server.js +0 -228
- package/mem-house/shipper/ship.js +0 -566
- package/relay-client.js +0 -306
- package/relay-server.js +0 -577
- package/server.js +0 -860
- package/share-image.js +0 -367
- /package/{mem-house → memhouse}/deploy.js +0 -0
- /package/{mem-house → memhouse}/envfile.js +0 -0
- /package/{mem-house → memhouse}/per-member/PLAN.md +0 -0
- /package/{mem-house → memhouse}/per-member/PROVISIONING.md +0 -0
- /package/{mem-house → memhouse}/per-member/README.md +0 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// Room-name resolution.
|
|
2
|
+
//
|
|
3
|
+
// Every member owns their rooms, named TYPE-FIRST: `sessions_<member>`,
|
|
4
|
+
// `messages_<member>`, `tool_calls_<member>`. Type-first is what lets the Merge rooms
|
|
5
|
+
// anchor on a fixed room type (`^sessions_`) and never match themselves.
|
|
6
|
+
//
|
|
7
|
+
// THIS IS THE ONLY LAYOUT. There was a shared one — three rooms named `sessions`,
|
|
8
|
+
// `messages`, `tool_calls`, with row policies narrowing each caller to their own rows —
|
|
9
|
+
// and it is gone. A shared read is a Merge room plus a GRANT, which is strictly less
|
|
10
|
+
// machinery than a policy that has to be right on every table and every read path; and
|
|
11
|
+
// isolation by absent grant fails closed, where a row policy fails open the moment one is
|
|
12
|
+
// missing.
|
|
13
|
+
//
|
|
14
|
+
// The member is never supplied by the caller — it is read back from the server with
|
|
15
|
+
// currentUser(), the same value the rooms' user_id column is stamped with. A client that
|
|
16
|
+
// could name its own member could write into someone else's rooms.
|
|
17
|
+
//
|
|
18
|
+
// `sessions_v` IS A SAVED QUERY, NOT A ROOM. It was briefly a stored view, which cost a
|
|
19
|
+
// name inside the `^sessions_` namespace the Merge rooms select on (silently merged into
|
|
20
|
+
// `all_sessions`, doubling every count), a fourth grant per member, and a fourth object to
|
|
21
|
+
// provision, roll forward and drop. A saved query is substituted with the caller's own
|
|
22
|
+
// room names and runs under the caller's credential, so it inherits their grants and
|
|
23
|
+
// raises no view-ownership question at all.
|
|
24
|
+
|
|
25
|
+
const ROOM_TYPES = ['sessions', 'messages', 'tool_calls'];
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* What a member is granted on their own rooms. ONE definition, because there are two paths
|
|
29
|
+
* that provision a house — provision.js and `install --print-sql` — and when only the first
|
|
30
|
+
* was narrowed the second went on emitting `GRANT ALL` for a release. GRANT ALL includes
|
|
31
|
+
* CREATE TABLE **on the member's own room name**, so a member owns the name rather than the
|
|
32
|
+
* data and can replace the room with a Merge over everyone's, doubling other members' rows
|
|
33
|
+
* in the team room.
|
|
34
|
+
*
|
|
35
|
+
* ALTER UPDATE and ALTER DELETE both: `DELETE FROM` is a lightweight delete implemented as
|
|
36
|
+
* `ALTER TABLE … UPDATE _row_exists = 0`, and which of the two a server demands varies by
|
|
37
|
+
* version. ALTER ADD COLUMN for ensureSchema's rollout. Nothing that defines an object.
|
|
38
|
+
*/
|
|
39
|
+
const MEMBER_PRIVS = 'SELECT, INSERT, ALTER UPDATE, ALTER DELETE, ALTER ADD COLUMN, OPTIMIZE';
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Resource ceilings for a member, as a settings profile.
|
|
43
|
+
*
|
|
44
|
+
* Here for the same reason MEMBER_PRIVS is here: there are two paths that provision a
|
|
45
|
+
* house — provision.js and `install --print-sql` — and a change made to one of them and
|
|
46
|
+
* not the other has now happened eight times. Anything a member is GIVEN belongs in this
|
|
47
|
+
* file, so both paths read the same definition.
|
|
48
|
+
*
|
|
49
|
+
* MIN..MAX, not CONST and not MAX alone.
|
|
50
|
+
*
|
|
51
|
+
* A plain default is advisory — a member writes `SETTINGS max_memory_usage = …` on their
|
|
52
|
+
* query and overrides it. MAX alone is not enough either: in ClickHouse **0 means
|
|
53
|
+
* unlimited** for these, and 0 satisfies any MAX, so `SETTINGS max_memory_usage = 0`
|
|
54
|
+
* removed the ceiling entirely. MIN 1 closes that.
|
|
55
|
+
*
|
|
56
|
+
* CONST closes it too and was tried first — but CONST refuses EVERY change, including the
|
|
57
|
+
* server's own. `rawQuery` in memhouse/server/queries.js pins max_result_rows and
|
|
58
|
+
* max_execution_time to protect the dashboard from an unbounded query, and under CONST
|
|
59
|
+
* those pins were refused with Code 452, so /api/query returned an error for every input
|
|
60
|
+
* on any house provisioned after that change. A ceiling has to stop a member RAISING the
|
|
61
|
+
* limit while still letting anyone LOWER it.
|
|
62
|
+
*
|
|
63
|
+
* max_memory_usage is per QUERY; max_memory_usage_for_user bounds the member across
|
|
64
|
+
* concurrent queries, without which N parallel queries x the per-query ceiling is
|
|
65
|
+
* unbounded.
|
|
66
|
+
*
|
|
67
|
+
* Limits are generous on purpose: the point is to stop one member exhausting a shared
|
|
68
|
+
* server, not to make an honest full re-ship fail — a 20M-row scan runs inside them.
|
|
69
|
+
*
|
|
70
|
+
* KNOWN GAP, deliberately not closed here: ALTER UPDATE, ALTER DELETE and OPTIMIZE are in
|
|
71
|
+
* MEMBER_PRIVS and execute in the background merge pool, where these per-query settings do
|
|
72
|
+
* not apply. A member looping `OPTIMIZE … FINAL` on a large room can still load the box.
|
|
73
|
+
* Bounding that needs server-level merge-pool configuration, which is the operator's, not
|
|
74
|
+
* something a per-member profile can express.
|
|
75
|
+
*/
|
|
76
|
+
/**
|
|
77
|
+
* The global-install command that will actually work HERE.
|
|
78
|
+
*
|
|
79
|
+
* Every site that printed `npm install -g memhouse --allow-scripts=better-sqlite3` printed
|
|
80
|
+
* a line that fails with EACCES on any distro-packaged Node, because the global prefix is
|
|
81
|
+
* root-owned — measured on a clean Ubuntu machine, exit 243, including as step 1 of
|
|
82
|
+
* `prompt --install`, which advertises itself as rendered for this machine. The README
|
|
83
|
+
* learned this; eight code sites had not.
|
|
84
|
+
*
|
|
85
|
+
* `sudo` is right for exactly one of the two causes and makes the other worse, and what
|
|
86
|
+
* tells them apart is who owns the prefix — so look, rather than guess.
|
|
87
|
+
*/
|
|
88
|
+
function installCommand() {
|
|
89
|
+
const base = 'npm install -g memhouse --allow-scripts=better-sqlite3';
|
|
90
|
+
try {
|
|
91
|
+
const { execFileSync } = require('child_process');
|
|
92
|
+
const prefix = execFileSync('npm', ['prefix', '-g'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
93
|
+
const st = require('fs').statSync(prefix);
|
|
94
|
+
if (st.uid !== process.getuid()) return `sudo ${base}`;
|
|
95
|
+
} catch { /* npm not on PATH, or an unreadable prefix: fall through to the plain form */ }
|
|
96
|
+
return base;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const MEMBER_PROFILE = 'memhouse_member';
|
|
100
|
+
const MEMBER_PROFILE_SETTINGS = [
|
|
101
|
+
'max_memory_usage = 8000000000 MIN 1 MAX 8000000000',
|
|
102
|
+
'max_memory_usage_for_user = 16000000000 MIN 1 MAX 16000000000',
|
|
103
|
+
'max_execution_time = 600 MIN 1 MAX 600',
|
|
104
|
+
'max_threads = 16 MIN 1 MAX 16',
|
|
105
|
+
'max_result_rows = 10000000 MIN 1 MAX 10000000',
|
|
106
|
+
// Provenance, pinned server-side rather than trusted to every client.
|
|
107
|
+
//
|
|
108
|
+
// `user_id MATERIALIZED currentUser()` is what records who wrote a row — but a
|
|
109
|
+
// MATERIALIZED column is computed during the INSERT, and an ASYNC insert flushes
|
|
110
|
+
// outside that context: measured on 25.11.9.34, `async_insert=1` stores user_id as the
|
|
111
|
+
// EMPTY STRING while a sync insert on the same table stamps correctly. An unattributed
|
|
112
|
+
// row is invisible to every identity-bound path at once, including its own owner's
|
|
113
|
+
// `reset`.
|
|
114
|
+
//
|
|
115
|
+
// The shipper already passes async_insert=0 on every insert, so memhouse itself was
|
|
116
|
+
// never the risk. The risk is everything ELSE holding a member credential — an agent
|
|
117
|
+
// following the /memhouse:sql skill, a migration script, a psql-habit one-liner. CONST
|
|
118
|
+
// makes the house refuse (SETTING_CONSTRAINT_VIOLATION) instead of silently accepting
|
|
119
|
+
// rows nobody owns, which is the difference between a guarantee and a convention.
|
|
120
|
+
// `doctor` still checks for blank user_id — that stays, as the detector for rows
|
|
121
|
+
// written before this profile existed.
|
|
122
|
+
'async_insert = 0 CONST',
|
|
123
|
+
].join(', ');
|
|
124
|
+
|
|
125
|
+
// ClickHouse usernames are permissive; room names are not. Refuse anything that would
|
|
126
|
+
// need quoting or could change how a Merge regex or a name-splitter reads.
|
|
127
|
+
function assertUsableMember(member) {
|
|
128
|
+
if (typeof member !== 'string' || !/^[A-Za-z][A-Za-z0-9_]*$/.test(member)) {
|
|
129
|
+
throw new Error(`cannot build room names for user '${member}': expected [A-Za-z][A-Za-z0-9_]*`);
|
|
130
|
+
}
|
|
131
|
+
// The reservation lives HERE, not only in the CLI, because provision.js is the admin
|
|
132
|
+
// entry point INSTALL.md sends you to for a second member — and it went as far as
|
|
133
|
+
// creating sessions_root/messages_root/tool_calls_root before dying on the grant,
|
|
134
|
+
// leaving three orphan rooms nothing removes. Case-insensitive: Root and ROOT are the
|
|
135
|
+
// same account. `default` is deliberately NOT reserved — a poor member name, but a real
|
|
136
|
+
// one that existing houses are built on.
|
|
137
|
+
if (member.toLowerCase() === 'root') {
|
|
138
|
+
throw new Error("'root' cannot be a member: it is a container artefact, not a person");
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The session rollup, as SQL text rather than a name.
|
|
144
|
+
*
|
|
145
|
+
* Two properties are load bearing: the join is a LEFT JOIN, so a session with no messages
|
|
146
|
+
* still appears with zero aggregates instead of vanishing from every count; and
|
|
147
|
+
* `join_use_nulls = 1` makes the unmatched message columns NULL so `coalesce` yields true
|
|
148
|
+
* zeros rather than counting the placeholder row.
|
|
149
|
+
*
|
|
150
|
+
* BOTH ARE NOW CARRIED BY THE TEXT ITSELF — `FINAL` on each room and a trailing
|
|
151
|
+
* `SETTINGS join_use_nulls = 1`. It used to depend on the caller passing `final=1` and
|
|
152
|
+
* `join_use_nulls=1`, on the belief that a subquery cannot carry its own SETTINGS. It can
|
|
153
|
+
* (verified on 26.7 and 25.11), and the belief was expensive: every consumer that forgot
|
|
154
|
+
* the settings silently counted each message once per undeleted ReplacingMergeTree
|
|
155
|
+
* version. Measured right after a `ship --full` — 2x, and 3x three ships later. It does
|
|
156
|
+
* not self-heal; the error grows with each pass until a merge happens to collapse the
|
|
157
|
+
* parts. `memhouse sessions-query > q.sql` lost the warning entirely, because it was
|
|
158
|
+
* printed on stderr as a SQL comment that never reached the SQL.
|
|
159
|
+
*
|
|
160
|
+
* Passing final=1 as well is harmless, so READ_SETTINGS stays as it is for direct room
|
|
161
|
+
* reads, which still need it.
|
|
162
|
+
*/
|
|
163
|
+
function sessionsRollup({ sessions, messages }) {
|
|
164
|
+
return `(
|
|
165
|
+
SELECT
|
|
166
|
+
s.session_id AS session_id,
|
|
167
|
+
any(s.source) AS source,
|
|
168
|
+
any(s.host) AS host,
|
|
169
|
+
any(s.name) AS name,
|
|
170
|
+
any(s.mode) AS mode,
|
|
171
|
+
any(s.folder) AS folder,
|
|
172
|
+
any(s.project) AS project,
|
|
173
|
+
any(s.git_branch) AS git_branch,
|
|
174
|
+
s.user_id AS user_id,
|
|
175
|
+
any(s.created_at) AS created_at,
|
|
176
|
+
any(s.last_updated_at) AS last_updated_at,
|
|
177
|
+
min(m.ts) AS started,
|
|
178
|
+
max(m.ts) AS ended,
|
|
179
|
+
coalesce(dateDiff('second', min(m.ts), max(m.ts)), 0) AS duration_sec,
|
|
180
|
+
count(m.seq) AS total_msgs,
|
|
181
|
+
countIf(m.role = 'user') AS user_msgs,
|
|
182
|
+
countIf(m.role = 'assistant') AS assistant_msgs,
|
|
183
|
+
countIf(m.is_subagent) AS subagent_msgs,
|
|
184
|
+
groupUniqArrayIf(m.model, m.model NOT IN ('', '<synthetic>')) AS models,
|
|
185
|
+
coalesce(sum(m.input_tokens), 0) AS input_tokens,
|
|
186
|
+
coalesce(sum(m.output_tokens), 0) AS output_tokens,
|
|
187
|
+
coalesce(sum(m.cache_read_tokens), 0) AS cache_read_tokens,
|
|
188
|
+
coalesce(sum(m.cache_write_tokens), 0) AS cache_write_tokens,
|
|
189
|
+
coalesce(sumIf(length(m.text), m.role = 'user'), 0) AS user_chars,
|
|
190
|
+
coalesce(sumIf(length(m.text), m.role = 'assistant'), 0) AS assistant_chars,
|
|
191
|
+
coalesce(substring(argMinIf(m.text, m.seq, m.role = 'user' AND m.text != ''), 1, 200), '') AS first_prompt
|
|
192
|
+
FROM ${sessions} AS s FINAL
|
|
193
|
+
LEFT JOIN ${messages} AS m FINAL ON m.session_id = s.session_id AND m.user_id = s.user_id
|
|
194
|
+
GROUP BY s.session_id, s.user_id
|
|
195
|
+
SETTINGS join_use_nulls = 1
|
|
196
|
+
)`;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* The settings every read needs. `final` collapses ReplacingMergeTree versions;
|
|
201
|
+
* `join_use_nulls` is what the rollup's coalesce depends on.
|
|
202
|
+
*/
|
|
203
|
+
const READ_SETTINGS = { final: 1, join_use_nulls: 1 };
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Names, given the member the server reported. Shared by every caller so the two
|
|
207
|
+
* transports in the tree (@clickhouse/client and the CLI's raw fetch) cannot drift into
|
|
208
|
+
* two different naming rules.
|
|
209
|
+
*
|
|
210
|
+
* `sessions_v` is the rollup as SQL text, usable in the same `FROM ... AS c` position a
|
|
211
|
+
* view name would occupy — which is what lets the read layer treat it as just another
|
|
212
|
+
* resolved name.
|
|
213
|
+
*/
|
|
214
|
+
function roomNames(member, user = member) {
|
|
215
|
+
assertUsableMember(member);
|
|
216
|
+
const out = { member, user };
|
|
217
|
+
for (const t of ROOM_TYPES) out[t] = `${t}_${member}`;
|
|
218
|
+
out.sessions_v = sessionsRollup(out);
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function currentUser(client) {
|
|
223
|
+
const rs = await client.query({ query: 'SELECT currentUser() AS u', format: 'JSONEachRow' });
|
|
224
|
+
const rows = await rs.json();
|
|
225
|
+
const u = rows[0] && rows[0].u;
|
|
226
|
+
if (!u) throw new Error('could not determine currentUser()');
|
|
227
|
+
return u;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Resolve the rooms for whoever this client is connected as.
|
|
232
|
+
* Returns { sessions, messages, tool_calls, sessions_v, member, user }.
|
|
233
|
+
*
|
|
234
|
+
* `user` is the identity the server reports. Writers need the value itself, not just the
|
|
235
|
+
* name it produces — see ship.js's delete, which must BIND the user rather than call
|
|
236
|
+
* currentUser() inside a mutation, where it is not evaluated in the caller's context.
|
|
237
|
+
*/
|
|
238
|
+
async function resolveRooms(client) {
|
|
239
|
+
const user = await currentUser(client);
|
|
240
|
+
return roomNames(user, user);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** The three Merge rooms — team-wide read paths, owner-managed. */
|
|
244
|
+
function mergeRooms() {
|
|
245
|
+
return { sessions: 'all_sessions', messages: 'all_messages', tool_calls: 'all_tool_calls' };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
module.exports = {
|
|
249
|
+
MEMBER_PRIVS, installCommand, MEMBER_PROFILE, MEMBER_PROFILE_SETTINGS,
|
|
250
|
+
ROOM_TYPES, READ_SETTINGS, assertUsableMember,
|
|
251
|
+
sessionsRollup, roomNames, currentUser, resolveRooms, mergeRooms,
|
|
252
|
+
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
-- mem — one member's rooms. Applied per member
|
|
1
|
+
-- mem — one member's rooms. Applied per member — {{MEMBER}} is the ClickHouse username.
|
|
2
2
|
--
|
|
3
3
|
-- Room naming is TYPE-FIRST (sessions_<m>, not <m>_sessions) so the Merge rooms
|
|
4
4
|
-- can anchor on a fixed room type and never match themselves.
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
-- it keeps provenance across a share and keeps the Merge rooms meaningful. Writers must
|
|
8
8
|
-- use async_insert=0 or the stamp does not happen.
|
|
9
9
|
--
|
|
10
|
-
-- ClickHouse >= 26.2 natively
|
|
10
|
+
-- ClickHouse >= 26.2 natively — 25.11 works because the shipper passes
|
|
11
11
|
-- allow_experimental_full_text_index=1 per-query when applying this file.
|
|
12
12
|
|
|
13
13
|
CREATE TABLE IF NOT EXISTS sessions_{{MEMBER}}
|
|
@@ -25,23 +25,27 @@ CREATE TABLE IF NOT EXISTS sessions_{{MEMBER}}
|
|
|
25
25
|
message_count UInt32 DEFAULT 0,
|
|
26
26
|
path String DEFAULT '',
|
|
27
27
|
extra JSON,
|
|
28
|
-
-- Who put this row here.
|
|
29
|
-
--
|
|
30
|
-
--
|
|
31
|
-
--
|
|
32
|
-
--
|
|
33
|
-
--
|
|
34
|
-
--
|
|
35
|
-
--
|
|
36
|
-
--
|
|
37
|
-
--
|
|
38
|
-
--
|
|
28
|
+
-- Who put this row here. Carried on every room so a reader can tell shipped rows
|
|
29
|
+
-- from imported ones, and so the shipper's clear can bind origin='ship'.
|
|
30
|
+
--
|
|
31
|
+
-- NOTE the sorting key below does NOT include origin, and sessions is the one room
|
|
32
|
+
-- where that is correct. A session has exactly ONE metadata row: title, counts,
|
|
33
|
+
-- bounds. If origin were in the key, an imported session and its shipped twin would
|
|
34
|
+
-- be two rows, and everything that reads sessions as one-row-per-session breaks --
|
|
35
|
+
-- sessions_v joins messages twice and over-reports (measured on a real house: one
|
|
36
|
+
-- duplicate row inflated the totals by 1,764 messages and 655M tokens), and the
|
|
37
|
+
-- incremental skip cannot decide which row is current. Collapsing them is what we
|
|
38
|
+
-- want: the newer ingested_at wins, which is the shipper's row for any session that
|
|
39
|
+
-- still exists on disk. Nothing is lost -- a session the adapters no longer see is
|
|
40
|
+
-- never cleared and never re-inserted, so its imported row stands untouched.
|
|
41
|
+
--
|
|
42
|
+
-- messages and tool_calls are the opposite case and DO key on origin -- see there.
|
|
39
43
|
origin LowCardinality(String) DEFAULT 'ship',
|
|
40
44
|
user_id String MATERIALIZED currentUser(),
|
|
41
45
|
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
|
|
42
46
|
)
|
|
43
47
|
ENGINE = ReplacingMergeTree(ingested_at)
|
|
44
|
-
ORDER BY (session_id, user_id
|
|
48
|
+
ORDER BY (session_id, user_id);
|
|
45
49
|
|
|
46
50
|
CREATE TABLE IF NOT EXISTS messages_{{MEMBER}}
|
|
47
51
|
(
|
|
@@ -6,13 +6,13 @@
|
|
|
6
6
|
--
|
|
7
7
|
-- Three measured properties this relies on (ClickHouse 26.7.1):
|
|
8
8
|
-- * fails closed on permissions — a caller sees only the underlying rooms they hold
|
|
9
|
-
-- grants for, with no leak and no error, so these can be granted broadly
|
|
10
|
-
-- * auto-discovers rooms created after them, so member onboarding needs no DDL here
|
|
9
|
+
-- grants for, with no leak and no error, so these can be granted broadly —
|
|
10
|
+
-- * auto-discovers rooms created after them, so member onboarding needs no DDL here —
|
|
11
11
|
-- * tolerates schema drift, returning a column's default for rooms lacking it — which
|
|
12
12
|
-- is what makes schema rollout progressive. Merge REJECTS mutations, so a new column
|
|
13
13
|
-- is added with ADD COLUMN, on the Merge room FIRST.
|
|
14
14
|
--
|
|
15
|
-
-- {{TEMPLATE_MEMBER}} is any existing member
|
|
15
|
+
-- {{TEMPLATE_MEMBER}} is any existing member — the Merge room borrows its column list.
|
|
16
16
|
|
|
17
17
|
CREATE TABLE IF NOT EXISTS all_sessions AS sessions_{{TEMPLATE_MEMBER}}
|
|
18
18
|
ENGINE = Merge(currentDatabase(), '^sessions_');
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// Resume: hand the pilot back into a session memhouse only ever read.
|
|
2
|
+
//
|
|
3
|
+
// Nothing here needs a schema change. A stored session already carries everything a
|
|
4
|
+
// resume needs: `session_id` is canonical `<source>:<native-id>` (ship.js:575) and
|
|
5
|
+
// `folder` is the directory the session ran in. Resume is a read-time computation over
|
|
6
|
+
// two columns.
|
|
7
|
+
//
|
|
8
|
+
// Two rules, both learned the hard way in this repo:
|
|
9
|
+
//
|
|
10
|
+
// 1. PRINT the command, never run it. A wrong cwd or a stale id does not fail — it
|
|
11
|
+
// starts a NEW session, and the transcript the pilot wanted is still lost while the
|
|
12
|
+
// tool reports success. The pilot pastes it, sees what it does, and owns the result.
|
|
13
|
+
//
|
|
14
|
+
// 2. Never write a flag from memory. Round 10 exists because a field name was asserted
|
|
15
|
+
// from a failed search rather than read from the source. Every entry below was read
|
|
16
|
+
// out of that CLI's own `--help` on a machine where it is installed, and the date and
|
|
17
|
+
// version are recorded with it. An editor whose flag has not been read that way is
|
|
18
|
+
// ABSENT from this table — an absent entry prints an honest refusal, a guessed entry
|
|
19
|
+
// prints a command that silently does the wrong thing.
|
|
20
|
+
//
|
|
21
|
+
// To add one: install the CLI, run `<cli> --help` (and `<cli> resume --help` if it is a
|
|
22
|
+
// subcommand), copy the exact syntax, and record what you read it from.
|
|
23
|
+
|
|
24
|
+
// Keyed by the string that is actually STORED in the `source` column, which is NOT the
|
|
25
|
+
// adapter's module `name`: editors/claude.js is `const name = 'claude'` and emits
|
|
26
|
+
// `source: 'claude-code'` on every chat it returns. This table was written against the
|
|
27
|
+
// module names first and would have refused to resume a single Claude Code session — 508 of
|
|
28
|
+
// them on the machine it was tested on. Read the values out of a real house
|
|
29
|
+
// (`SELECT DISTINCT source FROM sessions_<you>`), never off the adapter's name.
|
|
30
|
+
const RESUMERS = {
|
|
31
|
+
// claude --help, 2026-08-12: "-r, --resume [value] Resume a conversation by session ID"
|
|
32
|
+
'claude-code': {
|
|
33
|
+
cli: 'claude',
|
|
34
|
+
argv: (id) => ['--resume', id],
|
|
35
|
+
verified: '2026-08-12',
|
|
36
|
+
},
|
|
37
|
+
// codex resume --help, 2026-08-12: "Usage: codex resume [OPTIONS] [SESSION_ID] [PROMPT]"
|
|
38
|
+
// — positional, and "UUIDs take precedence if it parses".
|
|
39
|
+
codex: {
|
|
40
|
+
cli: 'codex',
|
|
41
|
+
argv: (id) => ['resume', id],
|
|
42
|
+
verified: '2026-08-12',
|
|
43
|
+
},
|
|
44
|
+
// opencode --help, 2026-08-12: "-s, --session session id to continue [string]"
|
|
45
|
+
opencode: {
|
|
46
|
+
cli: 'opencode',
|
|
47
|
+
argv: (id) => ['--session', id],
|
|
48
|
+
verified: '2026-08-12',
|
|
49
|
+
},
|
|
50
|
+
// NOT here, deliberately, and each for a different reason:
|
|
51
|
+
//
|
|
52
|
+
// goose, gemini-cli, cursor-agent — plausibly resumable, but not installed on the
|
|
53
|
+
// machine this table was written on, so the flag could only have been guessed. Install
|
|
54
|
+
// one, read its --help, add it.
|
|
55
|
+
//
|
|
56
|
+
// everything in GUI_ONLY — no CLI entry point takes a session id there. Opening the
|
|
57
|
+
// folder is not resuming the session and must not be dressed up as it.
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
// Surfaces with no CLI to resume into AT ALL, separated from "the flag has not been read
|
|
61
|
+
// yet" so the refusal can say which of the two it is — the same reason `discover` separates
|
|
62
|
+
// a skipped adapter from an editor you do not have.
|
|
63
|
+
//
|
|
64
|
+
// Stored `source` values again, and two corrections already: `copilot.js` emits
|
|
65
|
+
// `copilot-cli` and IS a command line, so it does not belong here; and windsurf never emits
|
|
66
|
+
// 'windsurf' at all — it emits its VARIANTS id, `devin` or `devin-next` — so the name this
|
|
67
|
+
// list first carried matched nothing and every Devin session fell through to the wrong
|
|
68
|
+
// refusal.
|
|
69
|
+
//
|
|
70
|
+
// Membership means: no command line takes a session id here, so no amount of checking will
|
|
71
|
+
// produce one. Anything merely UNCHECKED belongs in UNVERIFIED — asserting "there is
|
|
72
|
+
// nothing to resume into" about a tool nobody has looked at is the same invented fact as
|
|
73
|
+
// guessing a flag.
|
|
74
|
+
const GUI_ONLY = new Set([
|
|
75
|
+
'cursor', 'vscode', 'zed', 'kiro', 'copilot-jetbrains', 'antigravity',
|
|
76
|
+
'devin', 'devin-next',
|
|
77
|
+
]);
|
|
78
|
+
|
|
79
|
+
// A plausible CLI that nobody has verified. Listed explicitly instead of left to fall
|
|
80
|
+
// through a default, so the completeness test can prove every source the adapters can emit
|
|
81
|
+
// lands in exactly one bucket — which is what would have caught both corrections above
|
|
82
|
+
// before they shipped.
|
|
83
|
+
const UNVERIFIED = new Set([
|
|
84
|
+
'goose', 'gemini-cli', 'cursor-agent', 'copilot-cli', 'codebuff', 'commandcode', 'gsd',
|
|
85
|
+
]);
|
|
86
|
+
|
|
87
|
+
// A session_id is `<source>:<native-id>`. Split on the FIRST colon only: several editors
|
|
88
|
+
// use colons inside their own ids, and slicing on the last one hands the CLI a truncated id
|
|
89
|
+
// that resolves to nothing.
|
|
90
|
+
function splitSessionId(sessionId) {
|
|
91
|
+
const i = String(sessionId).indexOf(':');
|
|
92
|
+
if (i <= 0) return null;
|
|
93
|
+
return { source: sessionId.slice(0, i), nativeId: sessionId.slice(i + 1) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Shell-quote for the line we PRINT. This never feeds an exec — it is text a human pastes
|
|
97
|
+
// — but a session folder with a space in it must still paste correctly, and an id is
|
|
98
|
+
// server-supplied data that has no business being read as shell syntax.
|
|
99
|
+
function shq(s) {
|
|
100
|
+
return /^[A-Za-z0-9_@%+=:,./-]+$/.test(s) ? s : `'${String(s).replace(/'/g, `'\\''`)}'`;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Resolve a stored session row into something the pilot can paste.
|
|
105
|
+
*
|
|
106
|
+
* @param {{session_id: string, source?: string, folder?: string, origin?: string}} row
|
|
107
|
+
* @returns {{ok: boolean, source: string, nativeId: string|null, folder: string,
|
|
108
|
+
* command: string|null, reason: string|null}}
|
|
109
|
+
*/
|
|
110
|
+
function resumeFor(row) {
|
|
111
|
+
const parts = splitSessionId(row.session_id);
|
|
112
|
+
// `source` is its own column, but the id prefix is what the CLI half of the pair is keyed
|
|
113
|
+
// on; prefer the column and fall back to the prefix so a row from an older house still
|
|
114
|
+
// resolves.
|
|
115
|
+
const source = row.source || (parts && parts.source) || '';
|
|
116
|
+
const nativeId = parts ? parts.nativeId : null;
|
|
117
|
+
const folder = row.folder || '';
|
|
118
|
+
const base = { source, nativeId, folder, command: null, ok: false, reason: null };
|
|
119
|
+
|
|
120
|
+
if (!nativeId) {
|
|
121
|
+
return { ...base, reason: `'${row.session_id}' is not a canonical <source>:<id> session id` };
|
|
122
|
+
}
|
|
123
|
+
// Imported rows come from an older house, another product, or a machine that no longer
|
|
124
|
+
// exists — the whole point of the `origin` guard the shipper carries. There is no local
|
|
125
|
+
// session store behind them, so no resume command can be right, and this is checked
|
|
126
|
+
// BEFORE the source table: 502 of the 1,304 sessions on the machine this was built on are
|
|
127
|
+
// imported `claude-ai` rows, and a source-only check would have offered to resume them.
|
|
128
|
+
if (row.origin && row.origin !== 'ship') {
|
|
129
|
+
return {
|
|
130
|
+
...base,
|
|
131
|
+
reason: `this session was imported (origin='${row.origin}'), not read from a local editor — there is nothing on this machine to resume into`,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
const r = RESUMERS[source];
|
|
135
|
+
if (!r) {
|
|
136
|
+
return {
|
|
137
|
+
...base,
|
|
138
|
+
reason: GUI_ONLY.has(source)
|
|
139
|
+
? `${source} is a GUI editor — it has no CLI that takes a session id, so there is nothing to resume into`
|
|
140
|
+
: `no verified resume command for ${source} — see memhouse/resume.js for how to add one`,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
const cmd = [r.cli, ...r.argv(nativeId)].map(shq).join(' ');
|
|
144
|
+
// The cd is half the command. Claude Code and Codex both scope their session lists by
|
|
145
|
+
// working directory, so the same id run from the wrong folder opens the picker or a new
|
|
146
|
+
// session instead of the transcript that was asked for.
|
|
147
|
+
return { ...base, ok: true, command: folder ? `cd ${shq(folder)} && ${cmd}` : cmd };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
module.exports = { RESUMERS, GUI_ONLY, UNVERIFIED, resumeFor, splitSessionId };
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// A daemon that keeps executing the code it booted with.
|
|
2
|
+
//
|
|
3
|
+
// `npm i -g memhouse@latest` replaces the files under the global prefix and does not touch
|
|
4
|
+
// the running processes, so the shipper keeps parsing with the old adapters and the
|
|
5
|
+
// dashboard keeps serving the old bundle. Nothing reports it: `status` prints the version
|
|
6
|
+
// of the CLI you just typed, not the version the daemon is running. On macminim a shipper
|
|
7
|
+
// ran 1d16h out of a directory that had been MOVED — the fix was live, the machine was not.
|
|
8
|
+
//
|
|
9
|
+
// npm's own `postinstall` is the obvious hook and the wrong one. Install scripts are
|
|
10
|
+
// blocked by default (`--allow-scripts`), which is the very condition this product already
|
|
11
|
+
// has to work around, and a package install that restarts a user's daemons is a surprise
|
|
12
|
+
// even when it works.
|
|
13
|
+
//
|
|
14
|
+
// So the daemon notices for itself. Once per loop pass it compares what is on disk against
|
|
15
|
+
// what it booted with, and when they differ it hands over:
|
|
16
|
+
//
|
|
17
|
+
// supervised (systemd/launchd set MEMHOUSE_SUPERVISED=1) — exit 0 and let the supervisor
|
|
18
|
+
// start a fresh process. Re-execing under a supervisor fights it: the unit's own
|
|
19
|
+
// restart would then race the child we spawned.
|
|
20
|
+
//
|
|
21
|
+
// unsupervised (`memhouse start` pidfiles) — re-exec ourselves detached, rewrite the
|
|
22
|
+
// pidfile, exit. Nothing else would ever restart it, and exiting silently would stop
|
|
23
|
+
// collecting memory, which is worse than running one pass of stale code.
|
|
24
|
+
//
|
|
25
|
+
// Guards, because a daemon that restarts itself wrongly is worse than one that is stale:
|
|
26
|
+
// a re-exec chain is counted in the environment and capped, and there is a floor on how
|
|
27
|
+
// often a restart may happen at all.
|
|
28
|
+
|
|
29
|
+
const fs = require('fs');
|
|
30
|
+
const path = require('path');
|
|
31
|
+
const { spawn } = require('child_process');
|
|
32
|
+
|
|
33
|
+
const CHAIN_VAR = 'MEMHOUSE_REEXEC_CHAIN';
|
|
34
|
+
const MAX_CHAIN = 3;
|
|
35
|
+
const MIN_INTERVAL_MS = 60_000;
|
|
36
|
+
|
|
37
|
+
function runDir() {
|
|
38
|
+
const home = process.env.MEMHOUSE_HOME || path.join(require('os').homedir(), '.memhouse');
|
|
39
|
+
return path.join(home, 'run');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Read the version off DISK, never `require()`. require caches the first read for the
|
|
43
|
+
// lifetime of the process, so a required package.json reports the version this daemon
|
|
44
|
+
// booted with no matter how many times it is re-read — which is exactly the value we are
|
|
45
|
+
// trying to detect a change in.
|
|
46
|
+
function diskVersion(repoRoot) {
|
|
47
|
+
try { return JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf-8')).version || null; }
|
|
48
|
+
catch { return null; }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* What this process is running, captured at boot.
|
|
53
|
+
* @param {string} entry absolute path of the daemon's own entry script
|
|
54
|
+
* @param {string} [repoRoot] where package.json lives; defaults two levels up from the
|
|
55
|
+
* entry's directory, which is where both daemons sit (memhouse/<part>/<entry>.js).
|
|
56
|
+
* Pass it explicitly rather than relying on that depth if you move an entry.
|
|
57
|
+
*/
|
|
58
|
+
function snapshot(entry, repoRoot = path.join(path.dirname(entry), '..', '..')) {
|
|
59
|
+
let real = null, mtimeMs = null;
|
|
60
|
+
try { real = fs.realpathSync(entry); mtimeMs = fs.statSync(real).mtimeMs; } catch { /* reported as drift */ }
|
|
61
|
+
return { entry, repoRoot, version: diskVersion(repoRoot), real, mtimeMs, at: Date.now() };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Has the installation underneath us changed?
|
|
66
|
+
* @returns {string|null} a human-readable reason, or null when nothing moved
|
|
67
|
+
*/
|
|
68
|
+
function driftReason(snap) {
|
|
69
|
+
const now = diskVersion(snap.repoRoot);
|
|
70
|
+
if (snap.version && now && now !== snap.version) return `version changed ${snap.version} → ${now}`;
|
|
71
|
+
let real = null, mtimeMs = null;
|
|
72
|
+
try { real = fs.realpathSync(snap.entry); mtimeMs = fs.statSync(real).mtimeMs; } catch { real = null; }
|
|
73
|
+
// The entry is gone: the directory this daemon runs out of was deleted, moved, or
|
|
74
|
+
// replaced. Reported, but NOT restartable — see maybeRestart, which refuses to re-exec a
|
|
75
|
+
// path that no longer resolves.
|
|
76
|
+
if (!real) return 'the directory this daemon runs from no longer exists';
|
|
77
|
+
if (snap.real && real !== snap.real) return `entry moved ${snap.real} → ${real}`;
|
|
78
|
+
// mtime alone, with the version unchanged, is a checkout being rebuilt or edited in
|
|
79
|
+
// place. Worth acting on for the same reason the version is: the running process is not
|
|
80
|
+
// the code on disk.
|
|
81
|
+
if (snap.mtimeMs && mtimeMs && mtimeMs !== snap.mtimeMs) return 'the entry script changed on disk';
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function chainCount() {
|
|
86
|
+
const n = Number(process.env[CHAIN_VAR]);
|
|
87
|
+
return Number.isFinite(n) && n > 0 ? n : 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Only ever rewrite a pidfile that names US. A daemon started by hand has no pidfile, and a
|
|
91
|
+
// pidfile naming some other process belongs to that process — overwriting it would point
|
|
92
|
+
// `memhouse stop` at ours and leave theirs running forever.
|
|
93
|
+
function adoptPidfile(name, newPid, log) {
|
|
94
|
+
const file = path.join(runDir(), name + '.pid');
|
|
95
|
+
try {
|
|
96
|
+
if (Number(fs.readFileSync(file, 'utf-8').trim()) !== process.pid) return;
|
|
97
|
+
fs.writeFileSync(file, String(newPid));
|
|
98
|
+
} catch { log(`[memhouse] could not update ${file} — 'memhouse stop' may not find the new process`); }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Act on drift. Returns false when nothing was done, and does not return at all when it
|
|
103
|
+
* hands over (the process exits).
|
|
104
|
+
*
|
|
105
|
+
* @param {object} o
|
|
106
|
+
* @param {ReturnType<snapshot>} o.snap what we booted with
|
|
107
|
+
* @param {string} o.name daemon name, matching its pidfile ('shipper' | 'dashboard')
|
|
108
|
+
* @param {(s: string) => void} [o.log]
|
|
109
|
+
*/
|
|
110
|
+
function maybeRestart({ snap, name, log = console.error }) {
|
|
111
|
+
const reason = driftReason(snap);
|
|
112
|
+
if (!reason) return false;
|
|
113
|
+
if (Date.now() - snap.at < MIN_INTERVAL_MS) return false;
|
|
114
|
+
|
|
115
|
+
const supervised = process.env.MEMHOUSE_SUPERVISED === '1';
|
|
116
|
+
log(`[memhouse] ${reason} — this ${name} is running code that is no longer installed`);
|
|
117
|
+
|
|
118
|
+
if (supervised) {
|
|
119
|
+
log('[memhouse] exiting so the service manager starts the new version');
|
|
120
|
+
process.exit(0);
|
|
121
|
+
}
|
|
122
|
+
// An entry that does not resolve cannot be re-executed, and there is no supervisor to
|
|
123
|
+
// resolve a fresh one. Keep shipping with what is loaded and say so every pass: a stale
|
|
124
|
+
// daemon still collects sessions, a dead one collects nothing.
|
|
125
|
+
let real = null;
|
|
126
|
+
try { real = fs.realpathSync(snap.entry); } catch { /* handled next */ }
|
|
127
|
+
if (!real) {
|
|
128
|
+
log(`[memhouse] cannot restart — nothing to exec. Fix it with: memhouse stop && memhouse start`);
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
if (chainCount() >= MAX_CHAIN) {
|
|
132
|
+
log(`[memhouse] already restarted ${MAX_CHAIN} times in this chain — not restarting again. Fix it with: memhouse stop && memhouse start`);
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const child = spawn(process.execPath, [real, ...process.argv.slice(2)], {
|
|
137
|
+
env: { ...process.env, [CHAIN_VAR]: String(chainCount() + 1) },
|
|
138
|
+
detached: true,
|
|
139
|
+
// The daemon's stdout/stderr are already the log file `memhouse start` opened, and a
|
|
140
|
+
// detached child inherits those descriptors — so the new process keeps writing to the
|
|
141
|
+
// same log with no second file to find.
|
|
142
|
+
stdio: 'inherit',
|
|
143
|
+
});
|
|
144
|
+
child.unref();
|
|
145
|
+
adoptPidfile(name, child.pid, log);
|
|
146
|
+
log(`[memhouse] restarted as pid ${child.pid}`);
|
|
147
|
+
process.exit(0);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
module.exports = { snapshot, driftReason, maybeRestart, CHAIN_VAR, MAX_CHAIN, MIN_INTERVAL_MS };
|