memhouse 0.3.1 → 0.4.2

package/mem-house/per-member/provision.js

@@ -0,0 +1,142 @@
+#!/usr/bin/env node
+// mem — provision one member's rooms and grants. Idempotent; also the schema-rollout hook.
+//
+//   provision.js --member <name> [--merge] [--owner-grants]
+//
+// Run as the OWNER. Steps, all IF NOT EXISTS / re-runnable:
+//   1. create the member's three rooms from schema-member.sql.tpl
+//   2. grant the member their own three rooms, TWO statements each: `ALL` (not
+//      re-grantable), then `SELECT WITH GRANT OPTION`. The split is the point —
+//      `ALL` covers the mutation privileges the shipper's clear-then-insert needs
+//      without naming them, and attaching grant-option to `SELECT` alone makes a share
+//      read-only by construction. See the notes at the grants themselves.
+//
+//      THREE ROOMS, not four objects: the session rollup is a saved query over these
+//      same rooms, not a stored view, so it needs no object and no grant of its own.
+//   3. --merge: create/refresh the three Merge rooms, borrowing this member's columns
+//   4. grant the member SELECT on whichever Merge rooms exist — a Merge reduces to the
+//      rooms the caller can already read, so this is what makes the team room fail
+//      CLOSED rather than deny outright
+//
+// Grants are explicit, per room, and NEVER `ON mem.*` — a member who can read the whole
+// house can read every other member's rooms, which would make this a shared house with
+// longer table names. No wildcards are used anywhere; the only pattern in the design is
+// the Merge regex, anchored on a fixed room type.
+//
+// Env: MEM_URL / MEM_USER / MEM_PASSWORD / MEM_DB, falling back to MEMHOUSE_* and then
+// to the shipper's own defaults — http://localhost:8123, memhouse_root, mem.
+
+const fs = require('fs');
+const path = require('path');
+const { createClient } = require('@clickhouse/client');
+const { ROOM_TYPES, mergeRooms, assertUsableMember } = require('./rooms');
+
+const args = process.argv.slice(2);
+const flag = (n) => args.includes(`--${n}`);
+const opt = (n, d = null) => { const i = args.indexOf(`--${n}`); return i >= 0 ? args[i + 1] : d; };
+
+// Defaults MUST match what the shipper and CLI use, or an operator who overrides only
+// the thing that differs — a password, say — silently provisions rooms in a house the
+// product never reads, or fails to authenticate as a user that does not exist.
+const cfg = {
+  url: process.env.MEM_URL || process.env.MEMHOUSE_URL || 'http://localhost:8123',
+  username: process.env.MEM_USER || process.env.MEMHOUSE_USER || 'memhouse_root',
+  password: process.env.MEM_PASSWORD || process.env.MEMHOUSE_PASSWORD || '',
+  database: process.env.MEM_DB || process.env.MEMHOUSE_DB || 'mem',
+};
+
+// Statements are split on ';' at end of line — the templates contain no ';' inside a
+// statement body, and keeping the splitter dumb keeps the templates readable.
+function statements(sql) {
+  return sql
+    .split(/;\s*$/m)
+    .map((s) => s.replace(/^\s*--.*$/gm, '').trim())
+    .filter(Boolean);
+}
+
+/** Which of the three Merge rooms actually exist — a member may be provisioned first. */
+async function mergeRoomsPresent(client, database) {
+  const names = Object.values(mergeRooms());
+  const rs = await client.query({
+    query: `SELECT name FROM system.tables WHERE database = {db:String} AND name IN ({names:Array(String)}) ORDER BY name`,
+    query_params: { db: database, names },
+    format: 'JSONEachRow',
+  });
+  return (await rs.json()).map((r) => r.name);
+}
+
+async function main() {
+  const member = opt('member');
+  if (!member) { console.error('usage: provision.js --member <name> [--merge]'); process.exit(2); }
+  assertUsableMember(member);
+
+  const here = __dirname;
+
+  // Create the house with a client that has NOT selected it — on a fresh server the
+  // database does not exist yet, and selecting it fails before it can be created.
+  const bootstrap = createClient({ ...cfg, database: '' });
+  await bootstrap.command({ query: `CREATE DATABASE IF NOT EXISTS ${cfg.database}` });
+  await bootstrap.close();
+
+  console.log(`[mem] provisioning in ${cfg.url} database '${cfg.database}' as '${cfg.username}'`);
+  const client = createClient({ ...cfg, clickhouse_settings: { async_insert: 0 } });
+
+  // 1. rooms
+  const tpl = fs.readFileSync(path.join(here, 'schema-member.sql.tpl'), 'utf-8');
+  const roomSql = statements(tpl.replaceAll('{{MEMBER}}', member));
+  for (const q of roomSql) {
+    // allow_experimental_full_text_index: a no-op on 26.x, required on 25.x for the
+    // messages text indexes.
+    await client.command({ query: q, clickhouse_settings: { allow_experimental_full_text_index: 1 } });
+  }
+  console.log(`[mem] rooms ready for '${member}': ${ROOM_TYPES.map((t) => `${t}_${member}`).join(', ')}`);
+
+  // 2. grants — two statements per room, and the split is the point.
+  //
+  // ALL, NOT re-grantable: the member owns their room outright, DROP and TRUNCATE
+  // included, because it is their memory to destroy. It also covers the mutation
+  // privileges the shipper needs without naming them — the shipper clears a session's
+  // rows before re-inserting, and which of ALTER UPDATE / ALTER DELETE a server demands
+  // varies by version (`DELETE FROM` is a lightweight delete, implemented as
+  // `ALTER TABLE … UPDATE _row_exists = 0`, so 25.11 wants ALTER UPDATE where 26.7 wanted
+  // ALTER DELETE). Granting one and not the other produces a pass that fails elsewhere.
+  //
+  // SELECT, re-grantable: a share is READ-ONLY BY CONSTRUCTION rather than by convention.
+  // `GRANT ALL … WITH GRANT OPTION` expands to 45 privileges on 26.7 — DROP TABLE,
+  // TRUNCATE, CREATE ROW POLICY, SYSTEM DROP REPLICA among them — and every one of those
+  // would become something a member could hand to a colleague while meaning "let them
+  // read my sessions". Measured: with this split, `GRANT SELECT … TO bob` succeeds and
+  // `GRANT DROP TABLE … TO bob` is refused 497.
+  for (const t of ROOM_TYPES) {
+    const room = `${cfg.database}.${t}_${member}`;
+    await client.command({ query: `GRANT ALL ON ${room} TO ${member}` });
+    await client.command({ query: `GRANT SELECT ON ${room} TO ${member} WITH GRANT OPTION` });
+  }
+  console.log(`[mem] granted ALL on 3 rooms to '${member}'; SELECT is the only re-grantable one`);
+
+  // 3. Merge rooms
+  if (flag('merge')) {
+    const mtpl = fs.readFileSync(path.join(here, 'schema-merge.sql.tpl'), 'utf-8');
+    for (const q of statements(mtpl.replaceAll('{{TEMPLATE_MEMBER}}', member))) {
+      await client.command({ query: q });
+    }
+    console.log('[mem] merge rooms ready: all_sessions, all_messages, all_tool_calls');
+  }
+
+  // 4. Merge-room grants. Broad on purpose, and safe for the reason the whole layout
+  // rests on: a Merge table reduces to the underlying rooms the CALLER holds grants for.
+  // Measured on 25.11 — bob with SELECT on all_messages and nothing on messages_alice
+  // reads his own row and none of alice's, with `_table` showing only messages_bob, and
+  // no error. Withholding this grant does not make anything safer; it makes the team
+  // room deny outright instead of failing closed, which is a different and worse thing.
+  const merged = await mergeRoomsPresent(client, cfg.database);
+  for (const room of merged) {
+    await client.command({ query: `GRANT SELECT ON ${cfg.database}.${room} TO ${member}` });
+  }
+  if (merged.length) console.log(`[mem] granted SELECT on ${merged.join(', ')} to '${member}'`);
+  else console.log('[mem] no merge rooms yet — run once with --merge to create them');
+
+  await client.close();
+}
+
+main().catch((e) => { console.error(`[mem] provision failed: ${e.message}`); process.exit(1); });

package/mem-house/per-member/rooms.js

@@ -0,0 +1,132 @@
+// Room-name resolution.
+//
+// Every member owns their rooms, named TYPE-FIRST: `sessions_<member>`,
+// `messages_<member>`, `tool_calls_<member>`. Type-first is what lets the Merge rooms
+// anchor on a fixed room type (`^sessions_`) and never match themselves.
+//
+// THIS IS THE ONLY LAYOUT. There was a shared one — three rooms named `sessions`,
+// `messages`, `tool_calls`, with row policies narrowing each caller to their own rows —
+// and it is gone. A shared read is a Merge room plus a GRANT, which is strictly less
+// machinery than a policy that has to be right on every table and every read path; and
+// isolation by absent grant fails closed, where a row policy fails open the moment one is
+// missing.
+//
+// The member is never supplied by the caller — it is read back from the server with
+// currentUser(), the same value the rooms' user_id column is stamped with. A client that
+// could name its own member could write into someone else's rooms.
+//
+// `sessions_v` IS A SAVED QUERY, NOT A ROOM. It was briefly a stored view, which cost a
+// name inside the `^sessions_` namespace the Merge rooms select on (silently merged into
+// `all_sessions`, doubling every count), a fourth grant per member, and a fourth object to
+// provision, roll forward and drop. A saved query is substituted with the caller's own
+// room names and runs under the caller's credential, so it inherits their grants and
+// raises no view-ownership question at all.
+
+const ROOM_TYPES = ['sessions', 'messages', 'tool_calls'];
+
+// ClickHouse usernames are permissive; room names are not. Refuse anything that would
+// need quoting or could change how a Merge regex or a name-splitter reads.
+function assertUsableMember(member) {
+  if (typeof member !== 'string' || !/^[A-Za-z][A-Za-z0-9_]*$/.test(member)) {
+    throw new Error(`cannot build room names for user '${member}': expected [A-Za-z][A-Za-z0-9_]*`);
+  }
+}
+
+/**
+ * The session rollup, as SQL text rather than a name.
+ *
+ * Two properties are load bearing: the join is a LEFT JOIN, so a session with no messages
+ * still appears with zero aggregates instead of vanishing from every count; and
+ * `join_use_nulls = 1` makes the unmatched message columns NULL so `coalesce` yields true
+ * zeros rather than counting the placeholder row. The setting is applied by the CALLER (a
+ * client setting, not a trailing SETTINGS clause) because this text is used as a subquery,
+ * and a subquery cannot carry its own SETTINGS.
+ */
+function sessionsRollup({ sessions, messages }) {
+  return `(
+    SELECT
+        s.session_id AS session_id,
+        any(s.source) AS source,
+        any(s.host) AS host,
+        any(s.name) AS name,
+        any(s.mode) AS mode,
+        any(s.folder) AS folder,
+        any(s.project) AS project,
+        any(s.git_branch) AS git_branch,
+        s.user_id AS user_id,
+        any(s.created_at) AS created_at,
+        any(s.last_updated_at) AS last_updated_at,
+        min(m.ts) AS started,
+        max(m.ts) AS ended,
+        coalesce(dateDiff('second', min(m.ts), max(m.ts)), 0) AS duration_sec,
+        count(m.seq) AS total_msgs,
+        countIf(m.role = 'user') AS user_msgs,
+        countIf(m.role = 'assistant') AS assistant_msgs,
+        countIf(m.is_subagent) AS subagent_msgs,
+        groupUniqArrayIf(m.model, m.model NOT IN ('', '<synthetic>')) AS models,
+        coalesce(sum(m.input_tokens), 0) AS input_tokens,
+        coalesce(sum(m.output_tokens), 0) AS output_tokens,
+        coalesce(sum(m.cache_read_tokens), 0) AS cache_read_tokens,
+        coalesce(sum(m.cache_write_tokens), 0) AS cache_write_tokens,
+        coalesce(sumIf(length(m.text), m.role = 'user'), 0) AS user_chars,
+        coalesce(sumIf(length(m.text), m.role = 'assistant'), 0) AS assistant_chars,
+        coalesce(substring(argMinIf(m.text, m.seq, m.role = 'user' AND m.text != ''), 1, 200), '') AS first_prompt
+    FROM ${sessions} AS s
+    LEFT JOIN ${messages} AS m ON m.session_id = s.session_id AND m.user_id = s.user_id
+    GROUP BY s.session_id, s.user_id
+  )`;
+}
+
+/**
+ * The settings every read needs. `final` collapses ReplacingMergeTree versions;
+ * `join_use_nulls` is what the rollup's coalesce depends on.
+ */
+const READ_SETTINGS = { final: 1, join_use_nulls: 1 };
+
+/**
+ * Names, given the member the server reported. Shared by every caller so the two
+ * transports in the tree (@clickhouse/client and the CLI's raw fetch) cannot drift into
+ * two different naming rules.
+ *
+ * `sessions_v` is the rollup as SQL text, usable in the same `FROM ... AS c` position a
+ * view name would occupy — which is what lets the read layer treat it as just another
+ * resolved name.
+ */
+function roomNames(member, user = member) {
+  assertUsableMember(member);
+  const out = { member, user };
+  for (const t of ROOM_TYPES) out[t] = `${t}_${member}`;
+  out.sessions_v = sessionsRollup(out);
+  return out;
+}
+
+async function currentUser(client) {
+  const rs = await client.query({ query: 'SELECT currentUser() AS u', format: 'JSONEachRow' });
+  const rows = await rs.json();
+  const u = rows[0] && rows[0].u;
+  if (!u) throw new Error('could not determine currentUser()');
+  return u;
+}
+
+/**
+ * Resolve the rooms for whoever this client is connected as.
+ * Returns { sessions, messages, tool_calls, sessions_v, member, user }.
+ *
+ * `user` is the identity the server reports. Writers need the value itself, not just the
+ * name it produces — see ship.js's delete, which must BIND the user rather than call
+ * currentUser() inside a mutation, where it is not evaluated in the caller's context.
+ */
+async function resolveRooms(client) {
+  const user = await currentUser(client);
+  return roomNames(user, user);
+}
+
+/** The three Merge rooms — team-wide read paths, owner-managed. */
+function mergeRooms() {
+  return { sessions: 'all_sessions', messages: 'all_messages', tool_calls: 'all_tool_calls' };
+}
+
+module.exports = {
+  ROOM_TYPES, READ_SETTINGS, assertUsableMember,
+  sessionsRollup, roomNames, currentUser, resolveRooms, mergeRooms,
+};

package/mem-house/per-member/schema-member.sql.tpl

@@ -0,0 +1,85 @@
+-- mem — one member's rooms. Applied per member; {{MEMBER}} is the ClickHouse username.
+--
+-- Room naming is TYPE-FIRST (sessions_<m>, not <m>_sessions) so the Merge rooms
+-- can anchor on a fixed room type and never match themselves.
+--
+-- user_id MATERIALIZED currentUser() is retained even though the room names the member:
+-- it keeps provenance across a share and keeps the Merge rooms meaningful. Writers must
+-- use async_insert=0 or the stamp does not happen.
+--
+-- ClickHouse >= 26.2 natively; 25.11 works because the shipper passes
+-- allow_experimental_full_text_index=1 per-query when applying this file.
+
+CREATE TABLE IF NOT EXISTS sessions_{{MEMBER}}
+(
+    session_id String,
+    source LowCardinality(String),
+    host LowCardinality(String),
+    name String DEFAULT '',
+    mode LowCardinality(String) DEFAULT '',
+    folder String DEFAULT '',
+    project String DEFAULT '',
+    git_branch String DEFAULT '',
+    created_at Nullable(DateTime64(3, 'UTC')),
+    last_updated_at Nullable(DateTime64(3, 'UTC')),
+    message_count UInt32 DEFAULT 0,
+    path String DEFAULT '',
+    extra JSON,
+    user_id String MATERIALIZED currentUser(),
+    ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
+)
+ENGINE = ReplacingMergeTree(ingested_at)
+ORDER BY (session_id, user_id);
+
+CREATE TABLE IF NOT EXISTS messages_{{MEMBER}}
+(
+    session_id String,
+    seq UInt32,
+    source LowCardinality(String),
+    host LowCardinality(String),
+    ts DateTime64(3, 'UTC'),
+    role LowCardinality(String),
+    model LowCardinality(String) DEFAULT '',
+    input_tokens UInt64 DEFAULT 0,
+    output_tokens UInt64 DEFAULT 0,
+    cache_read_tokens UInt64 DEFAULT 0,
+    cache_write_tokens UInt64 DEFAULT 0,
+    text String,
+    project String DEFAULT '',
+    folder String DEFAULT '',
+    is_subagent Bool DEFAULT false,
+    extra JSON,
+    line_hash UInt64,
+    user_id String MATERIALIZED currentUser(),
+    ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
+    text_ngram String MATERIALIZED lower(text),
+    text_word  String MATERIALIZED lower(text),
+    INDEX idx_text_ngram text_ngram TYPE text(tokenizer = ngrams(3)) GRANULARITY 1,
+    INDEX idx_text_word  text_word  TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(ingested_at)
+ORDER BY (session_id, user_id, seq);
+
+CREATE TABLE IF NOT EXISTS tool_calls_{{MEMBER}}
+(
+    session_id String,
+    seq UInt32,
+    idx UInt32,
+    source LowCardinality(String),
+    host LowCardinality(String),
+    tool_name LowCardinality(String),
+    args String DEFAULT '{}',
+    ts DateTime64(3, 'UTC'),
+    project String DEFAULT '',
+    folder String DEFAULT '',
+    user_id String MATERIALIZED currentUser(),
+    ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
+)
+ENGINE = ReplacingMergeTree(ingested_at)
+ORDER BY (session_id, user_id, idx);
+
+-- NO sessions_v HERE, deliberately. The session rollup is a SAVED QUERY substituted
+-- with these room names and run under the caller's own credential (see rooms.js
+-- sessionsRollup). A stored view would need a name inside the `^sessions_` namespace
+-- the Merge rooms select on, a fourth grant, and a fourth object to provision and roll
+-- forward — for nothing the query does not already do.

package/mem-house/per-member/schema-merge.sql.tpl

@@ -0,0 +1,24 @@
+-- mem — the Merge rooms. Owner-managed, created once per house.
+--
+-- Team-wide reads. Each is a regex over room names anchored on the ROOM TYPE, so it
+-- matches member rooms only and can never match itself (`^messages_` does not match
+-- `all_messages`).
+--
+-- Three measured properties this relies on (ClickHouse 26.7.1):
+--   * fails closed on permissions — a caller sees only the underlying rooms they hold
+--     grants for, with no leak and no error, so these can be granted broadly;
+--   * auto-discovers rooms created after them, so member onboarding needs no DDL here;
+--   * tolerates schema drift, returning a column's default for rooms lacking it — which
+--     is what makes schema rollout progressive. Merge REJECTS mutations, so a new column
+--     is added with ADD COLUMN, on the Merge room FIRST.
+--
+-- {{TEMPLATE_MEMBER}} is any existing member; the Merge room borrows its column list.
+
+CREATE TABLE IF NOT EXISTS all_sessions AS sessions_{{TEMPLATE_MEMBER}}
+ENGINE = Merge(currentDatabase(), '^sessions_');
+
+CREATE TABLE IF NOT EXISTS all_messages AS messages_{{TEMPLATE_MEMBER}}
+ENGINE = Merge(currentDatabase(), '^messages_');
+
+CREATE TABLE IF NOT EXISTS all_tool_calls AS tool_calls_{{TEMPLATE_MEMBER}}
+ENGINE = Merge(currentDatabase(), '^tool_calls_');