memhouse 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -30
- package/bin/memhouse.js +555 -202
- package/editors/antigravity.js +2 -0
- package/editors/claude.js +3 -1
- package/editors/codebuff.js +2 -0
- package/editors/codex.js +4 -1
- package/editors/commandcode.js +2 -1
- package/editors/copilot-jetbrains.js +2 -1
- package/editors/copilot.js +2 -1
- package/editors/cursor-agent.js +2 -1
- package/editors/cursor.js +2 -0
- package/editors/gemini.js +2 -1
- package/editors/goose.js +2 -1
- package/editors/index.js +8 -1
- package/editors/kiro.js +3 -2
- package/editors/opencode.js +2 -1
- package/editors/scope.js +126 -0
- package/editors/vscode.js +2 -0
- package/editors/zed.js +2 -1
- package/memhouse/DESIGN.md +31 -8
- package/memhouse/binding.js +81 -0
- package/memhouse/channel.js +50 -0
- package/memhouse/delivery/AGENT-INSTALL-PROMPT.md +2 -1
- package/memhouse/delivery/AGENT-INSTALL.md +1 -1
- package/memhouse/delivery/INVITATION.md.tpl +147 -0
- package/memhouse/delivery/PROMPT.md +6 -5
- package/memhouse/delivery/kernel-install.md +4 -2
- package/memhouse/delivery/plugin/.claude-plugin/plugin.json +1 -1
- package/memhouse/delivery/plugin/reference/HOUSE.md +38 -12
- package/memhouse/delivery/plugin/skills/access/SKILL.md +9 -9
- package/memhouse/delivery/plugin/skills/admin/SKILL.md +1 -1
- package/memhouse/delivery/plugin/skills/house/SKILL.md +18 -2
- package/memhouse/flags.js +5 -4
- package/memhouse/house/HOUSE.md +25 -11
- package/memhouse/house/house.js +110 -27
- package/memhouse/house/migrate.js +7 -6
- package/memhouse/house/relocate.js +2 -2
- package/memhouse/house/schema.sql.tpl +3 -3
- package/memhouse/invitefile.js +38 -0
- package/memhouse/provision.js +100 -0
- package/memhouse/shipper/ship.js +87 -28
- package/package.json +1 -1
package/memhouse/house/house.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
// The house.
|
|
2
2
|
//
|
|
3
|
-
// A house is a ClickHouse DATABASE
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// their own credential, and two columns say who and where every row came from:
|
|
3
|
+
// A house is a ClickHouse DATABASE, `mem` by default. Its rooms are tables — `sessions`,
|
|
4
|
+
// `messages`, `tool_calls` — one set per member, named for them, and two columns say who
|
|
5
|
+
// and where every row came from:
|
|
7
6
|
//
|
|
8
7
|
// user_id String MATERIALIZED currentUser() — stamped by the server, unforgeable by
|
|
9
8
|
// clients (async_insert is pinned to 0
|
|
@@ -12,19 +11,30 @@
|
|
|
12
11
|
// host LowCardinality(String) — the machine's fingerprint, minted once
|
|
13
12
|
// per install (../host.js)
|
|
14
13
|
//
|
|
15
|
-
// That pair IS the provenance model.
|
|
16
|
-
// both write; `WHERE user_id = 'alice'` is one person, `WHERE host = '…'` is one machine.
|
|
17
|
-
// The model is collaborative — housemates trust each other with the house — and the
|
|
18
|
-
// boundary between houses is the database: joining someone's ClickHouse takes a database
|
|
19
|
-
// and a credential (`GRANT ALL ON team_a.* TO alice`), nothing else, because ALL on your
|
|
20
|
-
// own database reaches nothing outside it.
|
|
14
|
+
// That pair IS the provenance model.
|
|
21
15
|
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
16
|
+
// ONE LAYOUT. A house is a database — `mem` unless somebody has a reason — and every
|
|
17
|
+
// member's rooms in it carry the member's name: `mem.alice_messages`, `mem.polat_messages`.
|
|
18
|
+
// A member holds one grant, on the wildcard `mem.<name>_*`, and nothing else in the
|
|
19
|
+
// database. That is the whole access model, and roomNames() below is the only place a
|
|
20
|
+
// table name is produced.
|
|
21
|
+
//
|
|
22
|
+
// alone on a laptop mem.polat_* a house of one
|
|
23
|
+
// a team on one server mem.polat_*, mem.alice_*
|
|
24
|
+
// an agency on a kernel the same, in the kernel's `mem`
|
|
25
|
+
//
|
|
26
|
+
// Nobody ever holds ALL ON <db>.*. That grant is dynamic — it covers rooms created later —
|
|
27
|
+
// and it is what turned two members in one database into a leak: either could read and
|
|
28
|
+
// re-grant the other's transcripts, measured. The wildcard reaches nothing outside the
|
|
29
|
+
// member's own name: measured, a member cannot create, read, list or re-grant a
|
|
30
|
+
// housemate's rooms, and can read their own grant back with SHOW GRANTS in one line.
|
|
31
|
+
//
|
|
32
|
+
// THIS FILE HAS SAID THREE OTHER THINGS. Shared tables with row policies (0.4.0: policies
|
|
33
|
+
// are permissive and OR'd, a catch-all fails open). Suffixed per-member rooms with Merge
|
|
34
|
+
// rooms over them (0.8.0: removed because clients had to resolve names first and
|
|
35
|
+
// isolation "solved a problem the product did not have" — the second half was wrong).
|
|
36
|
+
// Then a house per member with `ALL ON db.*`, plus a prefixed variant beside it, plus the
|
|
37
|
+
// machinery to keep the two from meeting. All three were special cases of this one.
|
|
28
38
|
//
|
|
29
39
|
// `sessions_v` IS A SAVED QUERY, NOT AN OBJECT — substituted into the same `FROM … AS c`
|
|
30
40
|
// position a view name would occupy, running under the caller's own credential. It was
|
|
@@ -38,7 +48,11 @@ const ROOM_TYPES = ['sessions', 'messages', 'tool_calls'];
|
|
|
38
48
|
* them, the shipper's guards do not require them, and a member who cannot create them
|
|
39
49
|
* still ships normally.
|
|
40
50
|
*/
|
|
41
|
-
|
|
51
|
+
// `<member>_meta` / `<member>_events` since the one-layout: every row in them is about ONE
|
|
52
|
+
// member (their schema version, their machines, their last ship, their shares), so the old
|
|
53
|
+
// `house_` prefix named the wrong unit. A pre-one-layout house still has `house_meta` /
|
|
54
|
+
// `house_events`; the shipper's legacy guard names the rename.
|
|
55
|
+
const META_TYPES = ['meta', 'events'];
|
|
42
56
|
|
|
43
57
|
/**
|
|
44
58
|
* What generation of the schema a house is at. Bumped only when existing rooms have to be
|
|
@@ -75,7 +89,7 @@ const SUPPORTED_SCHEMAS = [2];
|
|
|
75
89
|
|
|
76
90
|
/**
|
|
77
91
|
* The floor a house may set under its writers, recorded by `memhouse migrate` in
|
|
78
|
-
*
|
|
92
|
+
* meta['min_writer_schema']. Today it equals SCHEMA_VERSION; a future
|
|
79
93
|
* back-compatible generation can hold it one step lower for a grace window.
|
|
80
94
|
*/
|
|
81
95
|
const MIN_WRITER_SCHEMA = 2;
|
|
@@ -233,15 +247,19 @@ const ROOM_MATERIALIZED = {
|
|
|
233
247
|
tool_calls: ['user_id'],
|
|
234
248
|
};
|
|
235
249
|
|
|
236
|
-
function currentParse(table, epochSource = table) {
|
|
250
|
+
function currentParse(table, epochSource = table, physical = null) {
|
|
251
|
+
// `table` names the ROOM (for the materialized-column lookup); `physical` is what goes
|
|
252
|
+
// into the FROM, which differs whenever a prefix is in play.
|
|
237
253
|
const extra = (ROOM_MATERIALIZED[table] || ['user_id']).join(', ');
|
|
254
|
+
const from = physical || table;
|
|
255
|
+
const epochFrom = physical ? physical.replace(new RegExp(`${table}$`), epochSource) : epochSource;
|
|
238
256
|
return `(
|
|
239
257
|
SELECT *, ${extra}
|
|
240
|
-
FROM ${
|
|
258
|
+
FROM ${from} FINAL
|
|
241
259
|
WHERE origin != 'ship'
|
|
242
260
|
OR (session_id, user_id, epoch) IN (
|
|
243
261
|
SELECT session_id, user_id, max(epoch)
|
|
244
|
-
FROM ${
|
|
262
|
+
FROM ${epochFrom}
|
|
245
263
|
WHERE origin = 'ship'
|
|
246
264
|
GROUP BY session_id, user_id)
|
|
247
265
|
)`;
|
|
@@ -302,24 +320,54 @@ const READ_SETTINGS = { final: 1, join_use_nulls: 1 };
|
|
|
302
320
|
* name it produces. See ship.js's delete, which must BIND the user rather than call
|
|
303
321
|
* currentUser() inside a mutation, where it is not evaluated in the caller's context.
|
|
304
322
|
*/
|
|
323
|
+
/**
|
|
324
|
+
* The table a room type lands in. THE naming rule, and the only copy of it.
|
|
325
|
+
*
|
|
326
|
+
* ONE LAYOUT. Every member's rooms carry the member's own name: `mem.alice_messages`,
|
|
327
|
+
* and `mem.polat_messages` even when polat is the only one in there. A house with one
|
|
328
|
+
* member is a house of one, not a different kind of house — so there is no second case
|
|
329
|
+
* here, no prefix to configure, and nothing for an invite file to carry.
|
|
330
|
+
*
|
|
331
|
+
* Depends on nothing but the member, so a caller holding only a config can name a table
|
|
332
|
+
* without a server round-trip. That matters: when the rule lived only inside roomNames(),
|
|
333
|
+
* callers that could not afford `SELECT currentUser()` spelled tables by hand instead —
|
|
334
|
+
* eight of them, all swallowing the error — and a house silently had no metadata plane.
|
|
335
|
+
*/
|
|
336
|
+
function physicalRoom(type, user) {
|
|
337
|
+
if (!user) throw new Error(`physicalRoom(${type}): a room belongs to a member, and none was given`);
|
|
338
|
+
return `${user}_${type}`;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** The wildcard every grant to a member is scoped to: `<member>_*`. */
|
|
342
|
+
function roomPattern(user) {
|
|
343
|
+
if (!user) throw new Error('roomPattern: a member is required');
|
|
344
|
+
return `${user}_*`;
|
|
345
|
+
}
|
|
346
|
+
|
|
305
347
|
function roomNames(user) {
|
|
306
|
-
|
|
348
|
+
if (!user) throw new Error('roomNames: a member is required');
|
|
349
|
+
const out = { member: user, user, pattern: roomPattern(user) };
|
|
350
|
+
// Every room name in the codebase comes from physicalRoom() above — through this
|
|
351
|
+
// function where a caller has the rooms, directly where it has only a member name.
|
|
352
|
+
const phys = (t) => physicalRoom(t, user);
|
|
353
|
+
out.physical = {};
|
|
354
|
+
for (const t of [...ROOM_TYPES, ...META_TYPES]) out.physical[t] = phys(t);
|
|
307
355
|
// The raw table names — for INSERT, for DDL, and for the shipper's own bookkeeping
|
|
308
356
|
// reads, which have to see every epoch to decide which one to write next.
|
|
309
|
-
for (const t of ROOM_TYPES) out[`${t}_raw`] = t;
|
|
357
|
+
for (const t of ROOM_TYPES) out[`${t}_raw`] = phys(t);
|
|
310
358
|
// What everything else gets. `sessions` is unfiltered: it is one metadata row per
|
|
311
359
|
// session by construction, latest-wins, and carries no epoch anyone may read.
|
|
312
|
-
out.sessions = 'sessions';
|
|
313
|
-
out.messages = currentParse('messages');
|
|
360
|
+
out.sessions = phys('sessions');
|
|
361
|
+
out.messages = currentParse('messages', 'messages', phys('messages'));
|
|
314
362
|
// tool_calls takes its epoch from MESSAGES, not from itself. A parse that produces
|
|
315
363
|
// messages but NO tool calls is ordinary — a compaction can remove every assistant turn
|
|
316
364
|
// that called something — and it writes zero rows into this room at the new epoch. Asked
|
|
317
365
|
// for its own max(epoch), the room would answer with the SUPERSEDED epoch and serve the
|
|
318
366
|
// old parse's tool calls beside the new parse's messages. Both rooms are written by the
|
|
319
367
|
// same pass at the same epoch, so messages is the authority for both.
|
|
320
|
-
out.tool_calls = currentParse('tool_calls', 'messages');
|
|
368
|
+
out.tool_calls = currentParse('tool_calls', 'messages', phys('tool_calls'));
|
|
321
369
|
// The house's own record of itself — plain names, nothing to filter.
|
|
322
|
-
for (const t of META_TYPES) { out[t] = t; out[`${t}_raw`] = t; }
|
|
370
|
+
for (const t of META_TYPES) { out[t] = phys(t); out[`${t}_raw`] = phys(t); }
|
|
323
371
|
out.sessions_v = sessionsRollup(out);
|
|
324
372
|
return out;
|
|
325
373
|
}
|
|
@@ -337,7 +385,42 @@ async function resolveRooms(client) {
|
|
|
337
385
|
return roomNames(await currentUser(client));
|
|
338
386
|
}
|
|
339
387
|
|
|
388
|
+
/**
|
|
389
|
+
* The text-index DDL for servers before the tokenizer grammar changed. 25.8 spells a
|
|
390
|
+
* tokenizer as a quoted name with options — `text(tokenizer = 'ngram', ngram_size = 3)`,
|
|
391
|
+
* `text(tokenizer = 'default')` — and rejects the function form with "Expected literal";
|
|
392
|
+
* 26.x spells it as a function — `ngrams(3)`, `splitByNonAlpha` — and rejects the quoted
|
|
393
|
+
* names as "Unknown tokenizer". Neither parses the other. The template carries the
|
|
394
|
+
* current grammar; this rewrites a statement to the older one when a server refuses it.
|
|
395
|
+
* `default` in the old grammar is the non-alphanumeric splitter, which is what
|
|
396
|
+
* `splitByNonAlpha` names in the new one, so the indexes built are the same.
|
|
397
|
+
*/
|
|
398
|
+
function legacyTextIndexDialect(sql) {
|
|
399
|
+
return sql
|
|
400
|
+
.replace(/TYPE text\(tokenizer = ngrams\((\d+)\)\)/g, "TYPE text(tokenizer = 'ngram', ngram_size = $1)")
|
|
401
|
+
.replace(/TYPE text\(tokenizer = splitByNonAlpha\)/g, "TYPE text(tokenizer = 'default')");
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Which text-index grammar a server speaks, from `SELECT version()`. Measured on real
|
|
406
|
+
* builds: 25.8 and 25.9 want the quoted-name form, 25.10 onwards the function form, and
|
|
407
|
+
* each rejects the other. Anything unparseable is treated as current; the shipper still
|
|
408
|
+
* falls back on a grammar refusal, so a wrong guess costs one round-trip, not the install.
|
|
409
|
+
*/
|
|
410
|
+
function textIndexDialectFor(version) {
|
|
411
|
+
const m = /^(\d+)\.(\d+)/.exec(String(version || '').trim());
|
|
412
|
+
if (!m) return 'modern';
|
|
413
|
+
const major = Number(m[1]); const minor = Number(m[2]);
|
|
414
|
+
return (major < 25 || (major === 25 && minor < 10)) ? 'legacy' : 'modern';
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/** True when a server's refusal is the grammar, not a privilege or a real mistake. */
|
|
418
|
+
function isTextIndexGrammarRefusal(message) {
|
|
419
|
+
return /Expected literal|supports only 'default'|Unknown tokenizer/.test(String(message || ''));
|
|
420
|
+
}
|
|
421
|
+
|
|
340
422
|
module.exports = {
|
|
423
|
+
physicalRoom, roomPattern, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor,
|
|
341
424
|
ROOM_TYPES, META_TYPES, SCHEMA_VERSION, SUPPORTED_SCHEMAS, MIN_WRITER_SCHEMA,
|
|
342
425
|
MIGRATIONS, ROOM_KEYS, keyProblem,
|
|
343
426
|
READ_SETTINGS, MEMBER_PIN,
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
// the old table until the rename; a late-writes pass copies anything newer than the
|
|
40
40
|
// snapshot, and ReplacingMergeTree makes that idempotent.
|
|
41
41
|
//
|
|
42
|
-
// EVERYTHING IS RECORDED. One
|
|
43
|
-
// after, per object; schema_version in
|
|
42
|
+
// EVERYTHING IS RECORDED. One events pending row before work, applied/failed
|
|
43
|
+
// after, per object; schema_version in meta only after a migration completes.
|
|
44
44
|
|
|
45
45
|
const fs = require('fs');
|
|
46
46
|
const path = require('path');
|
|
@@ -59,7 +59,7 @@ const { ROOM_TYPES, keyProblem, createStatement } = require('./house');
|
|
|
59
59
|
* toVersion the SCHEMA_VERSION a house is at once this has run
|
|
60
60
|
* detect(q, ctx) -> what still needs doing (any truthy non-empty value), or
|
|
61
61
|
* null/[] when the house is already past this migration.
|
|
62
|
-
* THE ROOMS ARE THE TRUTH, never
|
|
62
|
+
* THE ROOMS ARE THE TRUTH, never meta: a hand-migrated
|
|
63
63
|
* house has no record, and a restored backup can carry a
|
|
64
64
|
* record newer than its tables.
|
|
65
65
|
* plan(found) -> lines of human text for the confirm prompt
|
|
@@ -81,7 +81,7 @@ function listMigrations() {
|
|
|
81
81
|
const seen = new Set();
|
|
82
82
|
let lastVersion = 0;
|
|
83
83
|
for (const m of out) {
|
|
84
|
-
// The id is spliced into SQL (
|
|
84
|
+
// The id is spliced into SQL (events predicates) and into kept-table suffixes.
|
|
85
85
|
// It is repo-controlled, but "repo-controlled" is one compromised dependency away
|
|
86
86
|
// from "attacker-controlled" — validate like any other identifier.
|
|
87
87
|
if (!/^[a-z0-9][a-z0-9-]*$/.test(m.id)) throw new Error(`migration id '${m.id}' must match [a-z0-9-]+`);
|
|
@@ -116,7 +116,7 @@ async function detectPending(q, ctx, { component = null } = {}) {
|
|
|
116
116
|
* Refuse when another actor's run is unfinished. An `applied` newer than the last
|
|
117
117
|
* `pending` means done; a dangling `pending` means someone is (or was) mid-copy, and two
|
|
118
118
|
* concurrent rebuilds of one room end with one of them renaming the other's work.
|
|
119
|
-
* Best-effort: a house without
|
|
119
|
+
* Best-effort: a house without events cannot answer, and that is not a reason to
|
|
120
120
|
* block the migration that will create it.
|
|
121
121
|
*/
|
|
122
122
|
async function unfinishedBy(q, ctx, migrationId) {
|
|
@@ -124,7 +124,8 @@ async function unfinishedBy(q, ctx, migrationId) {
|
|
|
124
124
|
const rows = await q.rows(
|
|
125
125
|
`SELECT argMax(status, event_at) AS status, argMax(actor, event_at) AS actor,
|
|
126
126
|
formatDateTime(max(event_at), '%Y-%m-%d %H:%i') AS at
|
|
127
|
-
FROM
|
|
127
|
+
FROM ${(ctx.rooms && ctx.rooms.physical && ctx.rooms.physical.events) || 'events'}
|
|
128
|
+
WHERE kind = 'migration' AND id = '${migrationId}'`);
|
|
128
129
|
const r = rows[0];
|
|
129
130
|
if (r && r.status === 'pending' && r.actor && r.actor !== ctx.member) return r;
|
|
130
131
|
} catch { /* no events table yet */ }
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
// not insertable even with insert_allow_materialized_columns.
|
|
13
13
|
const DERIVED = ['text_ngram', 'text_word'];
|
|
14
14
|
|
|
15
|
-
//
|
|
16
|
-
// Everything else in
|
|
15
|
+
// meta keys that are DURABLE facts of the house — worth carrying to the new host.
|
|
16
|
+
// Everything else in meta is a per-host heartbeat (client_version, client_schema,
|
|
17
17
|
// last_ship:<writer>) that the new shipper rewrites for itself on its first run; carrying
|
|
18
18
|
// a stale one would misreport which machine last shipped until that run lands.
|
|
19
19
|
function isDurableMetaKey(key) {
|
|
@@ -154,10 +154,10 @@ ORDER BY (session_id, user_id, origin, epoch, idx);
|
|
|
154
154
|
-- authoritative and always current, and a mirror of them would be a second truth that
|
|
155
155
|
-- goes stale. What is recorded is what those cannot say: intent, sequence, outcome.
|
|
156
156
|
|
|
157
|
-
CREATE TABLE IF NOT EXISTS
|
|
157
|
+
CREATE TABLE IF NOT EXISTS meta
|
|
158
158
|
(
|
|
159
159
|
-- House-wide keys only ('schema_version', 'house_created_at'). Anything per-member
|
|
160
|
-
-- or per-machine belongs in
|
|
160
|
+
-- or per-machine belongs in events, which is append-only — this room is
|
|
161
161
|
-- latest-wins, so two members writing one key would overwrite each other.
|
|
162
162
|
key String,
|
|
163
163
|
value String,
|
|
@@ -168,7 +168,7 @@ CREATE TABLE IF NOT EXISTS house_meta
|
|
|
168
168
|
ENGINE = ReplacingMergeTree(updated_at)
|
|
169
169
|
ORDER BY (key);
|
|
170
170
|
|
|
171
|
-
CREATE TABLE IF NOT EXISTS
|
|
171
|
+
CREATE TABLE IF NOT EXISTS events
|
|
172
172
|
(
|
|
173
173
|
event_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
|
174
174
|
-- 'migration' — a room rebuild, one row per transition (pending → applied|failed)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// An invite file found where the invitee is standing.
|
|
2
|
+
//
|
|
3
|
+
// The guide says "from the directory holding the file, run memhouse install --env <file>".
|
|
4
|
+
// A person who just downloaded `invite-alice.env` and typed `memhouse` should not be shown
|
|
5
|
+
// the help screen; they should be asked whether to join. This module finds candidate files
|
|
6
|
+
// and reads what they promise; the CLI decides whether to ask, and install does the work.
|
|
7
|
+
// Nothing here prints a value from the file — the URL, user and database are safe to show,
|
|
8
|
+
// the password never is.
|
|
9
|
+
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const envfile = require('./envfile');
|
|
13
|
+
|
|
14
|
+
const REQUIRED = ['MEMHOUSE_URL', 'MEMHOUSE_USER', 'MEMHOUSE_PASSWORD', 'MEMHOUSE_DB'];
|
|
15
|
+
|
|
16
|
+
/** `invite-<name>.env` files in the given directories, first directory first. */
|
|
17
|
+
function findInvites(dirs) {
|
|
18
|
+
const out = [];
|
|
19
|
+
for (const dir of dirs) {
|
|
20
|
+
let names = [];
|
|
21
|
+
try { names = fs.readdirSync(dir); } catch { continue; }
|
|
22
|
+
for (const n of names.sort()) if (/^invite-[A-Za-z0-9_]+\.env$/.test(n)) out.push({ file: path.join(dir, n), dir });
|
|
23
|
+
}
|
|
24
|
+
return out;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** What the file promises, without the secret. */
|
|
28
|
+
function describeInvite(text) {
|
|
29
|
+
const parsed = envfile.parse(String(text || ''));
|
|
30
|
+
const missing = REQUIRED.filter((k) => !parsed[k]);
|
|
31
|
+
return {
|
|
32
|
+
user: parsed.MEMHOUSE_USER || null, url: parsed.MEMHOUSE_URL || null, db: parsed.MEMHOUSE_DB || null,
|
|
33
|
+
channel: parsed.MEMHOUSE_CHANNEL || null, isInvite: parsed.MEMHOUSE_INVITE === '1',
|
|
34
|
+
complete: missing.length === 0, missing,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
module.exports = { findInvites, describeInvite, REQUIRED };
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// Provisioning a member — THE plan, whether memhouse executes it or a DBA does.
|
|
2
|
+
//
|
|
3
|
+
// There used to be three copies of this: the live path (adminBootstrap), the printed one
|
|
4
|
+
// (--print-sql), and install --admin-user. They drifted, and the printed copy handed a
|
|
5
|
+
// non-admin the one configuration the live path refuses — found in a drill, not by a
|
|
6
|
+
// test. A plan that is data cannot drift from itself: the live path executes it,
|
|
7
|
+
// --print-sql prints it, and the unit tests assert on it.
|
|
8
|
+
//
|
|
9
|
+
// The plan is a member and a database, and its centre is one statement:
|
|
10
|
+
//
|
|
11
|
+
// GRANT <room privileges> ON <db>.<member>_* TO <member> WITH GRANT OPTION
|
|
12
|
+
//
|
|
13
|
+
// A wildcard on the member's own name. It covers rooms that do not exist yet, so the
|
|
14
|
+
// member's shipper creates their rooms itself, a schema rebuild can CREATE and EXCHANGE
|
|
15
|
+
// inside the pattern, and a share can name the whole set or one table. It reaches nothing
|
|
16
|
+
// outside the pattern — measured on 26.7: a member cannot create, read, list, drop or
|
|
17
|
+
// re-grant a housemate's rooms — and a colleague can read it back with SHOW GRANTS in one
|
|
18
|
+
// line, which is the property the whole product rests on.
|
|
19
|
+
//
|
|
20
|
+
// Nobody is ever granted ALL ON <db>.*. That grant is dynamic (it covers rooms created
|
|
21
|
+
// later), and it is what turned two members in one database into a leak.
|
|
22
|
+
|
|
23
|
+
const { MEMBER_PIN, assertUsableName, roomPattern } = require('./house/house');
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* What a member may do inside their own pattern. Each entry earns its place:
|
|
27
|
+
* SELECT, INSERT the shipper writes, everything else reads
|
|
28
|
+
* ALTER, OPTIMIZE ensureSchema adds columns in place; doctor compacts
|
|
29
|
+
* CREATE TABLE, DROP TABLE their own rooms on first ship; a rebuild on migrate
|
|
30
|
+
* ROW POLICY * `share --only` scopes a grantee to a project or a session
|
|
31
|
+
*/
|
|
32
|
+
const ROOM_PRIVILEGES = [
|
|
33
|
+
'SELECT', 'INSERT', 'ALTER', 'OPTIMIZE',
|
|
34
|
+
'CREATE TABLE', 'DROP TABLE',
|
|
35
|
+
'CREATE ROW POLICY', 'ALTER ROW POLICY', 'DROP ROW POLICY', 'SHOW ROW POLICIES',
|
|
36
|
+
];
|
|
37
|
+
|
|
38
|
+
function roomGrant(db, member) {
|
|
39
|
+
return `GRANT ${ROOM_PRIVILEGES.join(', ')} ON ${db}.${roomPattern(member)} TO ${member} WITH GRANT OPTION`;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function sqlString(v) {
|
|
43
|
+
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The statements that make `member` a member of house `db`, in order.
|
|
48
|
+
*
|
|
49
|
+
* `optional` marks steps that need ACCESS MANAGEMENT the admin may not hold. The live
|
|
50
|
+
* path skips those on refusal and says what the member loses; a refused required step
|
|
51
|
+
* aborts. `password` may be null for a member that already exists (an invite re-run, an
|
|
52
|
+
* install against an account the DBA made) — then the CREATE USER step is omitted and the
|
|
53
|
+
* rest still applies.
|
|
54
|
+
*
|
|
55
|
+
* @returns {Array<{sql:string, why:string, optional:boolean}>}
|
|
56
|
+
*/
|
|
57
|
+
function plan({ db, member, password = null }) {
|
|
58
|
+
assertUsableName(db, 'house');
|
|
59
|
+
assertUsableName(member, 'member');
|
|
60
|
+
const steps = [
|
|
61
|
+
{ sql: `CREATE DATABASE IF NOT EXISTS ${db}`, why: 'the house', optional: false },
|
|
62
|
+
];
|
|
63
|
+
if (password !== null) {
|
|
64
|
+
steps.push({ sql: `CREATE USER IF NOT EXISTS ${member} IDENTIFIED BY ${sqlString(password)}`, why: 'the member', optional: false });
|
|
65
|
+
}
|
|
66
|
+
steps.push(
|
|
67
|
+
{ sql: roomGrant(db, member), why: `their rooms — ${db}.${roomPattern(member)} — and nothing else in the house`, optional: false },
|
|
68
|
+
{ sql: `ALTER USER ${member} ADD SETTING ${MEMBER_PIN}`, why: 'user_id is stamped on every row; an async insert would skip the stamp', optional: false },
|
|
69
|
+
{ sql: `GRANT ALTER USER ON ${member} TO ${member}`, why: '`memhouse passwd` without an admin (their own account only)', optional: true },
|
|
70
|
+
{ sql: `GRANT SHOW USERS ON *.* TO ${member}`, why: 'see who to share with — names only, never data', optional: true },
|
|
71
|
+
{ sql: `GRANT REMOTE ON *.* TO ${member}`, why: '`memhouse relocate` pulls the old house over remoteSecure()', optional: true },
|
|
72
|
+
);
|
|
73
|
+
return steps;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The plan as SQL a person runs — clickhouse-client, the play UI, curl. One statement per
|
|
78
|
+
* line so it can be split on ';' by anything, comments above each saying why.
|
|
79
|
+
*/
|
|
80
|
+
function render(steps, { db, member }) {
|
|
81
|
+
const out = [
|
|
82
|
+
`-- memhouse: make '${member}' a member of house '${db}'.`,
|
|
83
|
+
'-- Run as a user with ACCESS MANAGEMENT (a stock \'default\' with access_management=1 will do).',
|
|
84
|
+
'--',
|
|
85
|
+
`-- One grant, on ${db}.${roomPattern(member)}: their rooms and nothing else in the database.`,
|
|
86
|
+
'-- Housemates keep their own rooms beside these and cannot read them. Do NOT widen it to',
|
|
87
|
+
`-- ${db}.* — that reaches every housemate's rooms, present and future.`,
|
|
88
|
+
'--',
|
|
89
|
+
`-- Their rooms are created by their own first \`memhouse ship\`; nothing to pre-build.`,
|
|
90
|
+
'',
|
|
91
|
+
];
|
|
92
|
+
for (const s of steps) {
|
|
93
|
+
out.push(`-- ${s.why}${s.optional ? ' (optional: needs ACCESS MANAGEMENT; skip if refused)' : ''}`);
|
|
94
|
+
out.push(`${s.sql};`);
|
|
95
|
+
out.push('');
|
|
96
|
+
}
|
|
97
|
+
return out.join('\n');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
module.exports = { ROOM_PRIVILEGES, roomGrant, plan, render };
|
package/memhouse/shipper/ship.js
CHANGED
|
@@ -61,12 +61,11 @@ const selfUpdate = require('../self-update');
|
|
|
61
61
|
// gets. Taking it later would record whatever an upgrade had already replaced.
|
|
62
62
|
const selfSnap = selfUpdate.snapshot(__filename);
|
|
63
63
|
const {
|
|
64
|
-
resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN,
|
|
65
|
-
SUPPORTED_SCHEMAS, keyProblem,
|
|
66
|
-
} = require('../house/house');
|
|
64
|
+
resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN, createStatement,
|
|
65
|
+
SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor } = require('../house/house');
|
|
67
66
|
// Rooms plus the house's own record of itself. Every table the template declares, which
|
|
68
67
|
// is what the column healer and the drift warning have to cover — a column added to
|
|
69
|
-
//
|
|
68
|
+
// events would otherwise roll out to nobody.
|
|
70
69
|
const ALL_TABLES = [...ROOM_TYPES, ...META_TYPES];
|
|
71
70
|
|
|
72
71
|
const BATCH_ROWS = 2000; // insert batch ceiling by ROW COUNT
|
|
@@ -410,17 +409,25 @@ async function assertRoomKeys(client, rooms) {
|
|
|
410
409
|
}
|
|
411
410
|
}
|
|
412
411
|
|
|
412
|
+
// The shipper reads the prefix straight from its environment: it is spawned with the
|
|
413
|
+
// resolved MEMHOUSE_* env by the CLI, and every room name it touches comes from
|
|
414
|
+
// resolveRooms, so this is the only line that needs to know.
|
|
415
|
+
|
|
413
416
|
async function ensureSchema(client) {
|
|
414
417
|
const rooms = await resolveRooms(client);
|
|
415
418
|
// BEFORE the CREATEs and ALTERs, not beside the end-of-function asserts. A house whose
|
|
416
419
|
// record says a newer release moved it forward may have columns this template does not
|
|
417
420
|
// know; running this template's DDL first could add back what that release removed —
|
|
418
421
|
// the exact write the guard exists to prevent.
|
|
422
|
+
await assertNotLegacyLayout(client, rooms);
|
|
419
423
|
await assertWriterSupported(client, rooms);
|
|
420
424
|
const tpl = fs.readFileSync(path.join(__dirname, '..', 'house', 'schema.sql.tpl'), 'utf-8');
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
425
|
+
// Rendered per room rather than taken as one blob, because a prefixed house names its
|
|
426
|
+
// rooms `<prefix>_messages` and the template says `messages`. With no prefix the two
|
|
427
|
+
// are identical, so this is the same statements in the same order as before.
|
|
428
|
+
const stmts = [...ROOM_TYPES, ...META_TYPES]
|
|
429
|
+
.map((t) => createStatement(tpl, t, rooms.physical ? rooms.physical[t] : t)
|
|
430
|
+
.replace('CREATE TABLE ', 'CREATE TABLE IF NOT EXISTS '));
|
|
424
431
|
// Two failure shapes are survivable here and both are skipped rather than fatal:
|
|
425
432
|
//
|
|
426
433
|
// - a permission denial. A read-only credential can still run --ensure-schema for
|
|
@@ -431,16 +438,24 @@ async function ensureSchema(client) {
|
|
|
431
438
|
// the loser can get TABLE_ALREADY_EXISTS or a metadata-file collision. The table
|
|
432
439
|
// exists either way, which is the outcome this function wants.
|
|
433
440
|
let denied = 0;
|
|
434
|
-
|
|
441
|
+
// The text-index grammar changed at 25.10 and neither side parses the other (measured:
|
|
442
|
+
// 25.8/25.9 take the quoted form, 25.10+ the function form). Pick by the server's own
|
|
443
|
+
// version, then keep one fallback: if the chosen form is refused on grammar alone —
|
|
444
|
+
// never on privilege — send the other. A wrong guess costs a round-trip, not the install.
|
|
445
|
+
const version = await serverVersion(client);
|
|
446
|
+
const dialect = textIndexDialectFor(version);
|
|
447
|
+
if (dialect === 'legacy') console.error(`[memhouse] ClickHouse ${version}: text indexes in the pre-25.10 grammar`);
|
|
448
|
+
const create = (q) => client.command({ query: q, clickhouse_settings: { async_insert: 0, allow_experimental_full_text_index: 1 } });
|
|
449
|
+
for (const modern of stmts) {
|
|
450
|
+
const legacy = legacyTextIndexDialect(modern);
|
|
451
|
+
const [first, second] = dialect === 'legacy' ? [legacy, modern] : [modern, legacy];
|
|
435
452
|
try {
|
|
436
|
-
await
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
clickhouse_settings: { async_insert: 0, allow_experimental_full_text_index: 1 },
|
|
443
|
-
});
|
|
453
|
+
try { await create(first); }
|
|
454
|
+
catch (e) {
|
|
455
|
+
const m = e && e.message ? e.message : String(e);
|
|
456
|
+
if (!(first !== second && isTextIndexGrammarRefusal(m))) throw e;
|
|
457
|
+
await create(second);
|
|
458
|
+
}
|
|
444
459
|
} catch (e) {
|
|
445
460
|
const m = e && e.message ? e.message : String(e);
|
|
446
461
|
if (/Not enough privileges|ACCESS_DENIED/i.test(m)) { denied++; continue; }
|
|
@@ -449,8 +464,16 @@ async function ensureSchema(client) {
|
|
|
449
464
|
}
|
|
450
465
|
}
|
|
451
466
|
if (denied) {
|
|
452
|
-
|
|
453
|
-
|
|
467
|
+
// In a SHARED house this is the designed state, not a shortfall: the operator creates
|
|
468
|
+
// each member's rooms during `invite` precisely so the member cannot — a member who
|
|
469
|
+
// could CREATE TABLE here could add tables beside every housemate's. Saying "you do
|
|
470
|
+
// not hold these rights" and then "schema ensured" about the same statements reads as
|
|
471
|
+
// a contradiction on every routine ship, which is how often a member sees it.
|
|
472
|
+
// A member holds CREATE TABLE on their own name pattern, so this is no longer the
|
|
473
|
+
// ordinary path — it is a credential provisioned outside memhouse with less than the
|
|
474
|
+
// plan gives. Say what to ask for rather than whose job it is.
|
|
475
|
+
console.error(`[memhouse] ${denied} schema statement(s) needed rights this credential does not hold — continuing with what it can do.`);
|
|
476
|
+
console.error('[memhouse] the memhouse grant is one line — `memhouse install --print-sql` shows it; ask whoever administers the house to run it.');
|
|
454
477
|
}
|
|
455
478
|
// A house created before origin existed has no such column, and every read and write
|
|
456
479
|
// scopes by it. Add it in place; ReplacingMergeTree backfills the DEFAULT, so every
|
|
@@ -503,7 +526,7 @@ async function ensureSchema(client) {
|
|
|
503
526
|
// There used to be a second, hardcoded `ADD COLUMN … origin` here, from when the healer
|
|
504
527
|
// above could only add that one column. It is redundant now that the generic loop reads
|
|
505
528
|
// the template correctly — and it would have been actively wrong once this loop covered
|
|
506
|
-
//
|
|
529
|
+
// meta and events, which have no origin and want none.
|
|
507
530
|
|
|
508
531
|
await assertRoomsExist(client, rooms);
|
|
509
532
|
await assertRoomKeys(client, rooms);
|
|
@@ -534,7 +557,7 @@ async function ensureSchema(client) {
|
|
|
534
557
|
* Keep the house's record of itself current: which schema generation it is at, and which
|
|
535
558
|
* memhouse version is writing into it.
|
|
536
559
|
*
|
|
537
|
-
* Written only when something CHANGED. A row per pass would turn
|
|
560
|
+
* Written only when something CHANGED. A row per pass would turn events into a
|
|
538
561
|
* heartbeat log — under `--loop` at the default interval that is 288 rows a day per
|
|
539
562
|
* machine, and the one question the table exists to answer ("when did this house move,
|
|
540
563
|
* and who moved it") would be buried in noise.
|
|
@@ -548,7 +571,7 @@ async function recordHouseState(client, rooms, host) {
|
|
|
548
571
|
const writer = `${rooms.user}@${host}`;
|
|
549
572
|
try {
|
|
550
573
|
const rs = await client.query({
|
|
551
|
-
query: `SELECT key, value FROM ${rooms.
|
|
574
|
+
query: `SELECT key, value FROM ${rooms.meta} FINAL WHERE key IN ('schema_version', {ck:String}, {cs:String})`,
|
|
552
575
|
// Keyed per member AND host. Keyed by member alone, two machines of one member
|
|
553
576
|
// overwrote each other's entry on every pass, so the fleet view could only ever
|
|
554
577
|
// show the machine that shipped last — the exact machine that needs no attention.
|
|
@@ -588,15 +611,15 @@ async function recordHouseState(client, rooms, host) {
|
|
|
588
611
|
}
|
|
589
612
|
// The heartbeat, EVERY pass — the fleet view's health column. Latest-wins on the key,
|
|
590
613
|
// so it is one live row per writer however often it fires; version/schema above stay
|
|
591
|
-
// change-only so
|
|
614
|
+
// change-only so events remains a record of moves, not a pulse trace.
|
|
592
615
|
metas.push({ key: `last_ship:${writer}`, value: new Date().toISOString(), host });
|
|
593
616
|
await client.insert({
|
|
594
|
-
table: rooms.
|
|
617
|
+
table: rooms.meta_raw, values: metas, format: 'JSONEachRow',
|
|
595
618
|
clickhouse_settings: { async_insert: 0 },
|
|
596
619
|
});
|
|
597
620
|
if (events.length) {
|
|
598
621
|
await client.insert({
|
|
599
|
-
table: rooms.
|
|
622
|
+
table: rooms.events_raw, values: events, format: 'JSONEachRow',
|
|
600
623
|
clickhouse_settings: { async_insert: 0 },
|
|
601
624
|
});
|
|
602
625
|
}
|
|
@@ -609,8 +632,8 @@ async function recordHouseState(client, rooms, host) {
|
|
|
609
632
|
*
|
|
610
633
|
* Distinct from assertRoomKeys, which inspects key SHAPES — a future generation could be
|
|
611
634
|
* a data transform the keys do not show. This reads what the house says about itself:
|
|
612
|
-
*
|
|
613
|
-
*
|
|
635
|
+
* meta['schema_version'] (what generation the rooms are at) and
|
|
636
|
+
* meta['min_writer_schema'] (the floor `memhouse migrate` sets under writers).
|
|
614
637
|
* Too new -> the fix is on THIS machine: memhouse update. Below the floor -> same.
|
|
615
638
|
* (A house OLDER than this release is not an error here — the room checks catch it and
|
|
616
639
|
* name `memhouse migrate`; this guard must not fire on a pre-0.10 house that has no
|
|
@@ -619,17 +642,52 @@ async function recordHouseState(client, rooms, host) {
|
|
|
619
642
|
* Releases before 0.10.0 never read this — for them the floor is enforced by the
|
|
620
643
|
* pilot's GRANTs, not by code.
|
|
621
644
|
*/
|
|
645
|
+
/**
|
|
646
|
+
* A house from before the one-layout holds plain rooms — `messages`, not `<member>_messages`.
|
|
647
|
+
* Shipping into it would quietly create a second, empty set of rooms beside the full ones
|
|
648
|
+
* and write there: every past session invisible to the member's own reads, the dashboard
|
|
649
|
+
* and the skills, with nothing anywhere saying why. Refuse, and say exactly what moves the
|
|
650
|
+
* old rooms across — a RENAME, instant, nothing copied.
|
|
651
|
+
*
|
|
652
|
+
* Trips only when the member's rooms are ABSENT and the plain ones PRESENT; a house holding
|
|
653
|
+
* both is mid-conversion and is left alone.
|
|
654
|
+
*/
|
|
655
|
+
async function serverVersion(client) {
|
|
656
|
+
try {
|
|
657
|
+
const rs = await client.query({ query: 'SELECT version() AS v', format: 'JSONEachRow' });
|
|
658
|
+
return String(((await rs.json())[0] || {}).v || '');
|
|
659
|
+
} catch { return ''; }
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
async function assertNotLegacyLayout(client, rooms) {
|
|
663
|
+
const rs = await client.query({
|
|
664
|
+
query: `SELECT name FROM system.tables WHERE database = currentDatabase() AND name IN ('messages', '${rooms.physical.messages}')`,
|
|
665
|
+
format: 'JSONEachRow',
|
|
666
|
+
});
|
|
667
|
+
const names = new Set((await rs.json()).map((r) => r.name));
|
|
668
|
+
if (names.has(rooms.physical.messages) || !names.has('messages')) return;
|
|
669
|
+
// The plain rooms of a pre-one-layout house: the transcript rooms by type name, and the
|
|
670
|
+
// two bookkeeping rooms under the names they had then.
|
|
671
|
+
const legacyName = { meta: 'house_meta', events: 'house_events' };
|
|
672
|
+
const moves = [...ROOM_TYPES, ...META_TYPES].map((t) => `RENAME TABLE ${legacyName[t] || t} TO ${rooms.physical[t]};`).join('\n ');
|
|
673
|
+
throw new Error(
|
|
674
|
+
'this house holds plain rooms (messages, sessions, tool_calls) — the layout before rooms were named for their member.\n'
|
|
675
|
+
+ ` Shipping now would create empty ${rooms.pattern} rooms beside them and write there, hiding every past session.\n`
|
|
676
|
+
+ ` Move the old rooms across (a rename — instant, nothing copied), then ship again:\n ${moves}\n`
|
|
677
|
+
+ ` Then replace the database-wide grant with the one-layout grant: memhouse install --print-sql --member ${rooms.member}`);
|
|
678
|
+
}
|
|
679
|
+
|
|
622
680
|
async function assertWriterSupported(client, rooms) {
|
|
623
681
|
let have;
|
|
624
682
|
try {
|
|
625
683
|
const rs = await client.query({
|
|
626
|
-
query: `SELECT key, value FROM ${rooms.
|
|
684
|
+
query: `SELECT key, value FROM ${rooms.meta} FINAL WHERE key IN ('schema_version', 'min_writer_schema')`,
|
|
627
685
|
format: 'JSONEachRow',
|
|
628
686
|
});
|
|
629
687
|
have = new Map((await rs.json()).map((r) => [r.key, toInt(r.value)]));
|
|
630
688
|
} catch (e) {
|
|
631
689
|
// ONLY a missing table means a pre-0.10 house (the room checks own that case). Any
|
|
632
|
-
// other failure — an ACCESS_DENIED on
|
|
690
|
+
// other failure — an ACCESS_DENIED on meta, a timeout — used to fall through
|
|
633
691
|
// here too, and a writer that merely could not READ the record was treated as if the
|
|
634
692
|
// record did not exist: on a newer-schema house whose key shapes happen to match,
|
|
635
693
|
// that bypassed the whole compatibility guard. Not being able to check is a reason
|
|
@@ -1004,6 +1062,7 @@ async function runShip(client, opts = {}) {
|
|
|
1004
1062
|
// shape: writing at the wrong epoch either forks a session that did not change, or
|
|
1005
1063
|
// overwrites a stored parse that did.
|
|
1006
1064
|
const rooms = await resolveRooms(client);
|
|
1065
|
+
await assertNotLegacyLayout(client, rooms);
|
|
1007
1066
|
await assertWriterSupported(client, rooms);
|
|
1008
1067
|
await assertRoomsExist(client, rooms);
|
|
1009
1068
|
await assertRoomKeys(client, rooms);
|