memhouse 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -11
- package/bin/memhouse.js +129 -108
- package/memhouse/DESIGN.md +29 -30
- package/memhouse/delivery/PROMPT.md +7 -9
- package/memhouse/delivery/kernel-install.md +14 -29
- package/memhouse/delivery/plugin/skills/search/SKILL.md +25 -38
- package/memhouse/delivery/plugin/skills/sessions/SKILL.md +15 -28
- package/memhouse/delivery/plugin/skills/sql/SKILL.md +23 -36
- package/memhouse/host.js +1 -1
- package/memhouse/house/HOUSE.md +74 -0
- package/memhouse/house/house.js +165 -0
- package/memhouse/{per-member/schema-member.sql.tpl → house/schema.sql.tpl} +8 -11
- package/memhouse/resume.js +1 -1
- package/memhouse/server/queries.js +8 -2
- package/memhouse/shipper/ship.js +24 -30
- package/package.json +1 -1
- package/memhouse/per-member/INSTALL.md +0 -228
- package/memhouse/per-member/PLAN.md +0 -174
- package/memhouse/per-member/PROVISIONING.md +0 -177
- package/memhouse/per-member/README.md +0 -48
- package/memhouse/per-member/SCHEMA.md +0 -120
- package/memhouse/per-member/SHARING.md +0 -102
- package/memhouse/per-member/provision.js +0 -265
- package/memhouse/per-member/rooms.js +0 -235
- package/memhouse/per-member/schema-merge.sql.tpl +0 -24
|
@@ -15,7 +15,7 @@ absent grant rather than a row policy — it fails closed. There is no shared
|
|
|
15
15
|
`sessions`/`messages`/`tool_calls` layout and no row policy in the default path; 0.4.0
|
|
16
16
|
removed both. Team-wide reads are the `Merge` rooms plus a `GRANT`. The session rollup
|
|
17
17
|
is a **saved query**, not a stored view — `memhouse sessions-query` prints it.
|
|
18
|
-
Design and measurements: `../
|
|
18
|
+
Design and measurements: `../house/HOUSE.md`.
|
|
19
19
|
|
|
20
20
|
This is an operator runbook, so it speaks machine vocabulary throughout — ClickHouse,
|
|
21
21
|
`GRANT`, `currentUser()`. That is correct for this audience.
|
|
@@ -63,40 +63,25 @@ For each person joining, split across the two authorities:
|
|
|
63
63
|
python3 executor/executor.py submit register-member '{"handle":"alice"}'
|
|
64
64
|
python3 executor/executor.py approve <call_id> && python3 executor/executor.py drain
|
|
65
65
|
|
|
66
|
-
# OWNER
|
|
67
|
-
#
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
node memhouse/per-member/provision.js --member alice --merge
|
|
66
|
+
# OWNER admits the member to the house — two statements, any DBA knows them:
|
|
67
|
+
# CREATE USER alice IDENTIFIED BY '…';
|
|
68
|
+
# GRANT ALL ON mem.* TO alice;
|
|
69
|
+
# (memhouse can run them: memhouse install --admin-user … --member alice)
|
|
71
70
|
```
|
|
72
71
|
|
|
73
|
-
**Never `GRANT … ON mem.*` to a member.** A member who can read `mem.*` can read every
|
|
74
|
-
other member's rooms, and then this is a shared house with longer table names.
|
|
75
|
-
`provision.js` grants per room: `ALL` on each of alice's three, plus a re-grantable
|
|
76
|
-
`SELECT` on each (that split is what makes a share read-only by construction), plus
|
|
77
|
-
`SELECT` on the `Merge` rooms.
|
|
78
|
-
|
|
79
72
|
Each member then runs the shipper with **their own** credential
|
|
80
|
-
(`MEMHOUSE_USER=alice`) —
|
|
81
|
-
`
|
|
82
|
-
|
|
83
|
-
|
|
73
|
+
(`MEMHOUSE_USER=alice`) — everyone writes into the SAME three tables (`sessions`,
|
|
74
|
+
`messages`, `tool_calls`), and the house stamps `user_id='alice'` on her rows,
|
|
75
|
+
un-spoofably (`MATERIALIZED currentUser()`, with `async_insert = 0 CONST` pinned on the
|
|
76
|
+
user so the stamp cannot be skipped). Her machines are told apart by `host`.
|
|
84
77
|
|
|
85
78
|
## 4. Visibility
|
|
86
79
|
|
|
87
|
-
**
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
`all_tool_calls`) plus a `GRANT SELECT` on them, which `provision.js --merge` issues. A
|
|
93
|
-
`Merge` room narrows to whatever underlying rooms the caller already holds grants for —
|
|
94
|
-
measured on 26.7.1: no leak, no error — so it can be granted broadly. It also
|
|
95
|
-
auto-discovers rooms created after it, so onboarding a member needs no DDL there.
|
|
96
|
-
|
|
97
|
-
A member widens what a colleague sees by granting their own rooms directly
|
|
98
|
-
(`GRANT SELECT ON mem.messages_alice TO bob`) — self-serve, no operator, because the
|
|
99
|
-
member holds grant-option on their own `SELECT`. See `../per-member/SHARING.md`.
|
|
80
|
+
**A house is shared by its housemates.** The tables are common; `user_id` and `host` say
|
|
81
|
+
who wrote what, and `WHERE user_id = 'alice'` is one person. The boundary is the
|
|
82
|
+
DATABASE: a member of `mem` holds nothing on any other database, so two agencies on one
|
|
83
|
+
kernel ClickHouse cannot read each other. The model is collaborative — housemates trust
|
|
84
|
+
each other with the house; a separate house is the isolation mechanism.
|
|
100
85
|
|
|
101
86
|
Row policies are still the only way to share a *subset* of rows, which is owner-mediated
|
|
102
87
|
and documented in `SHARING.md` — they are not how isolation works.
|
|
@@ -9,7 +9,7 @@ allowed-tools: Bash
|
|
|
9
9
|
# /memhouse:search — search conversation memory
|
|
10
10
|
|
|
11
11
|
Search the full message history in the memhouse house. Every room is named for your
|
|
12
|
-
ClickHouse user — `
|
|
12
|
+
ClickHouse user — `messages`, `sessions` — see **Room names** below.
|
|
13
13
|
|
|
14
14
|
## Connection
|
|
15
15
|
|
|
@@ -59,35 +59,25 @@ SQL
|
|
|
59
59
|
`FORMAT PrettyCompact` for display; `FORMAT JSONEachRow` when you want to parse
|
|
60
60
|
the rows yourself.
|
|
61
61
|
|
|
62
|
-
## Room names —
|
|
62
|
+
## Room names — plain, shared tables
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
64
|
+
The house's rooms are three plain tables: `messages`, `sessions`, `tool_calls` — resolved
|
|
65
|
+
by the connection's database (`MEMHOUSE_DB`), not by who is asking. Everyone in the house
|
|
66
|
+
writes into the same tables; `user_id` (stamped by the server) says whose row it is and
|
|
67
|
+
`host` says which machine shipped it. Search the whole house by default; add
|
|
68
|
+
`AND user_id = '<name>'` only when the user asks for one person's sessions.
|
|
68
69
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
--data-binary "SELECT currentUser() FORMAT TabSeparated" \
|
|
75
|
-
"$MEMHOUSE_URL/?readonly=1"
|
|
76
|
-
# prints e.g. alice → your rooms are messages_alice, sessions_alice, tool_calls_alice
|
|
77
|
-
```
|
|
78
|
-
|
|
79
|
-
**Keep the SQL heredoc quoted (`<<'SQL'`) and put the real name in the text.** This used to
|
|
80
|
-
say "substitute it into every table name below" and show `messages_$MEM_ME`, which cannot
|
|
81
|
-
expand inside a quoted heredoc — so the agent unquotes it, and then the SEARCH TERMS expand
|
|
82
|
-
too. Both instances driving this skill did exactly that, unprompted, on the first attempt.
|
|
83
|
-
Measured consequences, from `system.query_log`:
|
|
70
|
+
**Keep the SQL heredoc quoted (`<<'SQL'`).** This skill once told the agent to substitute
|
|
71
|
+
a shell variable into the table names, which cannot expand inside a quoted heredoc — so
|
|
72
|
+
the agent unquotes it, and then the SEARCH TERMS expand too. Both instances driving this
|
|
73
|
+
skill did exactly that, unprompted, on the first attempt. Measured consequences, from
|
|
74
|
+
`system.query_log`:
|
|
84
75
|
|
|
85
76
|
- a term containing `$home` became the empty string, so `LIKE '%%'` matched **every row**
|
|
86
77
|
and reported hits with exit 0 — silently wrong, not an error;
|
|
87
78
|
- a term containing a backtick **executed the command inside it**.
|
|
88
79
|
|
|
89
|
-
A user's search term is arbitrary text. With the heredoc quoted
|
|
90
|
-
out, neither can happen.
|
|
80
|
+
A user's search term is arbitrary text. With the heredoc quoted, neither can happen.
|
|
91
81
|
|
|
92
82
|
**There is no `sessions_v` object.** The session rollup is a saved query over those same
|
|
93
83
|
rooms — `memhouse sessions-query` prints it for whoever you are connected as, ready to
|
|
@@ -98,13 +88,10 @@ both rooms and a trailing `SETTINGS join_use_nulls = 1`, so it is correct wherev
|
|
|
98
88
|
paste it. `final=1` on the connection still matters for reads of the rooms THEMSELVES,
|
|
99
89
|
which carry no FINAL of their own.
|
|
100
90
|
|
|
101
|
-
If you hold `SELECT` on them, `all_messages` / `all_sessions` / `all_tool_calls` read
|
|
102
|
-
across every member at once, narrowed to whatever grants you actually have — a Merge room
|
|
103
|
-
reduces to the rooms the caller can read, so it fails closed rather than denying outright.
|
|
104
91
|
|
|
105
92
|
## How to search (the FTS columns)
|
|
106
93
|
|
|
107
|
-
`
|
|
94
|
+
`messages` carries two MATERIALIZED lowercase copies of `text`, each with a
|
|
108
95
|
text index — search those, display `text`:
|
|
109
96
|
|
|
110
97
|
- `text_ngram` (ngram index) → substring match: `text_ngram LIKE '%term%'`
|
|
@@ -120,8 +107,8 @@ phrase → one `LIKE` clause for the whole phrase.
|
|
|
120
107
|
SELECT m.ts, m.source, m.project, m.role, m.session_id,
|
|
121
108
|
s.name AS session,
|
|
122
109
|
substring(m.text, 1, 300) AS snippet
|
|
123
|
-
FROM
|
|
124
|
-
LEFT JOIN
|
|
110
|
+
FROM messages AS m
|
|
111
|
+
LEFT JOIN sessions AS s USING (session_id, user_id)
|
|
125
112
|
WHERE m.text_ngram LIKE '%postgres%' AND m.text_ngram LIKE '%migration%'
|
|
126
113
|
ORDER BY m.ts DESC
|
|
127
114
|
LIMIT 30
|
|
@@ -132,7 +119,7 @@ FORMAT PrettyCompact
|
|
|
132
119
|
|
|
133
120
|
```sql
|
|
134
121
|
SELECT ts, project, session_id, substring(text, 1, 200) AS snippet
|
|
135
|
-
FROM
|
|
122
|
+
FROM messages
|
|
136
123
|
WHERE hasToken(text_word, 'rls')
|
|
137
124
|
AND source = 'claude-code'
|
|
138
125
|
AND ts > now() - INTERVAL 14 DAY
|
|
@@ -144,12 +131,12 @@ FORMAT PrettyCompact
|
|
|
144
131
|
### Example 3 — which sessions mention it most (then drill in)
|
|
145
132
|
|
|
146
133
|
```sql
|
|
147
|
-
SELECT m.session_id, any(s.name) AS session, any(m.project) AS project,
|
|
134
|
+
SELECT m.session_id, m.user_id, any(s.name) AS session, any(m.project) AS project,
|
|
148
135
|
count() AS hits, max(m.ts) AS last_hit
|
|
149
|
-
FROM
|
|
150
|
-
LEFT JOIN
|
|
136
|
+
FROM messages AS m
|
|
137
|
+
LEFT JOIN sessions AS s USING (session_id, user_id)
|
|
151
138
|
WHERE m.text_ngram LIKE '%clickhouse cache%'
|
|
152
|
-
GROUP BY m.session_id
|
|
139
|
+
GROUP BY m.session_id, m.user_id
|
|
153
140
|
ORDER BY hits DESC
|
|
154
141
|
LIMIT 10
|
|
155
142
|
FORMAT PrettyCompact
|
|
@@ -163,14 +150,14 @@ FORMAT PrettyCompact
|
|
|
163
150
|
`cursor`, `cursor-agent`, `vscode`, `zed`, `opencode`, `gemini-cli`,
|
|
164
151
|
`windsurf`, `antigravity`, `copilot-cli`, `goose`, `kiro`, …)
|
|
165
152
|
- Skip subagent noise → `AND NOT m.is_subagent`
|
|
166
|
-
- Only user/assistant text lives in `
|
|
167
|
-
`
|
|
153
|
+
- Only user/assistant text lives in `messages`; tool invocations are in
|
|
154
|
+
`tool_calls` (`tool_name`, `args`) — search `args ILIKE '%term%'` there
|
|
168
155
|
only if the user explicitly wants tool calls searched.
|
|
169
156
|
|
|
170
157
|
## Output
|
|
171
158
|
|
|
172
159
|
Summarize hits grouped by session (name, source, project, date, 1–2 best
|
|
173
160
|
snippets each) and give the total hit count. Offer to pull the full transcript
|
|
174
|
-
of the best session (`SELECT role, text FROM
|
|
161
|
+
of the best session (`SELECT role, text FROM messages WHERE session_id = '…'
|
|
175
162
|
ORDER BY seq`). If zero hits, retry with broader/fewer terms and report the
|
|
176
|
-
corpus size (`SELECT count() FROM
|
|
163
|
+
corpus size (`SELECT count() FROM messages`).
|
|
@@ -63,35 +63,25 @@ FORMAT PrettyCompact
|
|
|
63
63
|
SQL
|
|
64
64
|
```
|
|
65
65
|
|
|
66
|
-
## Room names —
|
|
66
|
+
## Room names — plain, shared tables
|
|
67
67
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
68
|
+
The house's rooms are three plain tables: `messages`, `sessions`, `tool_calls` — resolved
|
|
69
|
+
by the connection's database (`MEMHOUSE_DB`), not by who is asking. Everyone in the house
|
|
70
|
+
writes into the same tables; `user_id` (stamped by the server) says whose row it is and
|
|
71
|
+
`host` says which machine shipped it. Search the whole house by default; add
|
|
72
|
+
`AND user_id = '<name>'` only when the user asks for one person's sessions.
|
|
72
73
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
--data-binary "SELECT currentUser() FORMAT TabSeparated" \
|
|
79
|
-
"$MEMHOUSE_URL/?readonly=1"
|
|
80
|
-
# prints e.g. alice → your rooms are messages_alice, sessions_alice, tool_calls_alice
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
**Keep the SQL heredoc quoted (`<<'SQL'`) and put the real name in the text.** This used to
|
|
84
|
-
say "substitute it into every table name below" and show `messages_$MEM_ME`, which cannot
|
|
85
|
-
expand inside a quoted heredoc — so the agent unquotes it, and then the SEARCH TERMS expand
|
|
86
|
-
too. Both instances driving this skill did exactly that, unprompted, on the first attempt.
|
|
87
|
-
Measured consequences, from `system.query_log`:
|
|
74
|
+
**Keep the SQL heredoc quoted (`<<'SQL'`).** This skill once told the agent to substitute
|
|
75
|
+
a shell variable into the table names, which cannot expand inside a quoted heredoc — so
|
|
76
|
+
the agent unquotes it, and then the SEARCH TERMS expand too. Both instances driving this
|
|
77
|
+
skill did exactly that, unprompted, on the first attempt. Measured consequences, from
|
|
78
|
+
`system.query_log`:
|
|
88
79
|
|
|
89
80
|
- a term containing `$home` became the empty string, so `LIKE '%%'` matched **every row**
|
|
90
81
|
and reported hits with exit 0 — silently wrong, not an error;
|
|
91
82
|
- a term containing a backtick **executed the command inside it**.
|
|
92
83
|
|
|
93
|
-
A user's search term is arbitrary text. With the heredoc quoted
|
|
94
|
-
out, neither can happen.
|
|
84
|
+
A user's search term is arbitrary text. With the heredoc quoted, neither can happen.
|
|
95
85
|
|
|
96
86
|
**There is no `sessions_v` object.** The session rollup is a saved query over those same
|
|
97
87
|
rooms — `memhouse sessions-query` prints it for whoever you are connected as, ready to
|
|
@@ -105,14 +95,11 @@ grows); without `join_use_nulls` a session with no messages reports `total_msgs
|
|
|
105
95
|
rather than 0.
|
|
106
96
|
|
|
107
97
|
**If the `memhouse` binary is not on PATH**, you cannot print the rollup — the skills are
|
|
108
|
-
installable on their own. Query the rooms directly instead: `
|
|
109
|
-
and `
|
|
98
|
+
installable on their own. Query the rooms directly instead: `sessions` for metadata
|
|
99
|
+
and `messages` for counts, each read `FINAL`, joined on `session_id` and `user_id`.
|
|
110
100
|
Prefer the binary when it is there; a rollup you assemble by hand and one printed by a
|
|
111
101
|
DIFFERENT memhouse version are the two ways this goes quietly wrong.
|
|
112
102
|
|
|
113
|
-
If you hold `SELECT` on them, `all_messages` / `all_sessions` / `all_tool_calls` read
|
|
114
|
-
across every member at once, narrowed to whatever grants you actually have — a Merge room
|
|
115
|
-
reduces to the rooms the caller can read, so it fails closed rather than denying outright.
|
|
116
103
|
|
|
117
104
|
## Default listing
|
|
118
105
|
|
|
@@ -173,5 +160,5 @@ FORMAT PrettyCompact
|
|
|
173
160
|
|
|
174
161
|
A compact table: when, name (or first prompt), source, host, project, msgs,
|
|
175
162
|
tokens. Offer to pull a full transcript next
|
|
176
|
-
(`SELECT role, text FROM
|
|
163
|
+
(`SELECT role, text FROM messages WHERE session_id = '…' ORDER BY seq`) or to
|
|
177
164
|
run /memhouse:search for a specific quote.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: sql
|
|
3
|
-
description: Run free-form read-only SQL against memhouse conversation memory (typed sessions/messages/tool_calls
|
|
3
|
+
description: Run free-form read-only SQL against memhouse conversation memory (typed sessions/messages/tool_calls tables on ClickHouse, shared by the house). Use for ad-hoc analytics the other memhouse skills don't cover — token spend, model/editor usage, tool rankings, activity heatmaps, busiest days/projects, cache-hit ratios, or any custom question over conversation data.
|
|
4
4
|
user-invocable: true
|
|
5
5
|
argument-hint: "<question or SQL>"
|
|
6
6
|
allowed-tools: Bash
|
|
@@ -73,35 +73,25 @@ SQL
|
|
|
73
73
|
|
|
74
74
|
`FORMAT PrettyCompact` for display, `FORMAT JSONEachRow` to parse.
|
|
75
75
|
|
|
76
|
-
## Room names —
|
|
76
|
+
## Room names — plain, shared tables
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
78
|
+
The house's rooms are three plain tables: `messages`, `sessions`, `tool_calls` — resolved
|
|
79
|
+
by the connection's database (`MEMHOUSE_DB`), not by who is asking. Everyone in the house
|
|
80
|
+
writes into the same tables; `user_id` (stamped by the server) says whose row it is and
|
|
81
|
+
`host` says which machine shipped it. Filter with `WHERE user_id = '<name>'` when you want
|
|
82
|
+
one person, or leave it off for the whole house.
|
|
82
83
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
--data-binary "SELECT currentUser() FORMAT TabSeparated" \
|
|
89
|
-
"$MEMHOUSE_URL/?readonly=1"
|
|
90
|
-
# prints e.g. alice → your rooms are messages_alice, sessions_alice, tool_calls_alice
|
|
91
|
-
```
|
|
92
|
-
|
|
93
|
-
**Keep the SQL heredoc quoted (`<<'SQL'`) and put the real name in the text.** This used to
|
|
94
|
-
say "substitute it into every table name below" and show `messages_$MEM_ME`, which cannot
|
|
95
|
-
expand inside a quoted heredoc — so the agent unquotes it, and then the SEARCH TERMS expand
|
|
96
|
-
too. Both instances driving this skill did exactly that, unprompted, on the first attempt.
|
|
97
|
-
Measured consequences, from `system.query_log`:
|
|
84
|
+
**Keep the SQL heredoc quoted (`<<'SQL'`).** This skill once told the agent to substitute
|
|
85
|
+
a shell variable into the table names, which cannot expand inside a quoted heredoc — so
|
|
86
|
+
the agent unquotes it, and then the SEARCH TERMS expand too. Both instances driving this
|
|
87
|
+
skill did exactly that, unprompted, on the first attempt. Measured consequences, from
|
|
88
|
+
`system.query_log`:
|
|
98
89
|
|
|
99
90
|
- a term containing `$home` became the empty string, so `LIKE '%%'` matched **every row**
|
|
100
91
|
and reported hits with exit 0 — silently wrong, not an error;
|
|
101
92
|
- a term containing a backtick **executed the command inside it**.
|
|
102
93
|
|
|
103
|
-
A user's search term is arbitrary text. With the heredoc quoted
|
|
104
|
-
out, neither can happen.
|
|
94
|
+
A user's search term is arbitrary text. With the heredoc quoted, neither can happen.
|
|
105
95
|
|
|
106
96
|
**There is no `sessions_v` object.** The session rollup is a saved query over those same
|
|
107
97
|
rooms — `memhouse sessions-query` prints it for whoever you are connected as, ready to
|
|
@@ -115,22 +105,19 @@ grows); without `join_use_nulls` a session with no messages reports `total_msgs
|
|
|
115
105
|
rather than 0.
|
|
116
106
|
|
|
117
107
|
**If the `memhouse` binary is not on PATH**, you cannot print the rollup — the skills are
|
|
118
|
-
installable on their own. Query the rooms directly instead: `
|
|
119
|
-
|
|
108
|
+
installable on their own. Query the rooms directly instead: `sessions` for metadata and
|
|
109
|
+
`messages` for counts, each read `FINAL`, joined on `session_id` **and `user_id`** — the
|
|
110
|
+
tables are shared, and joining on `session_id` alone can merge two housemates' rows.
|
|
120
111
|
Prefer the binary when it is there; a rollup you assemble by hand and one printed by a
|
|
121
112
|
DIFFERENT memhouse version are the two ways this goes quietly wrong.
|
|
122
113
|
|
|
123
|
-
If you hold `SELECT` on them, `all_messages` / `all_sessions` / `all_tool_calls` read
|
|
124
|
-
across every member at once, narrowed to whatever grants you actually have — a Merge room
|
|
125
|
-
reduces to the rooms the caller can read, so it fails closed rather than denying outright.
|
|
126
|
-
|
|
127
114
|
## Schema (the house)
|
|
128
115
|
|
|
129
116
|
| Object | Kind | Columns |
|
|
130
117
|
|---|---|---|
|
|
131
|
-
| `
|
|
132
|
-
| `
|
|
133
|
-
| `
|
|
118
|
+
| `sessions` | table, 1 row/session | `session_id, source, host, name, mode, folder, project, git_branch, created_at, last_updated_at, message_count, path, extra JSON, user_id, ingested_at` |
|
|
119
|
+
| `messages` | table, 1 row/message | `session_id, seq, source, host, ts, role, model, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, text, project, folder, is_subagent, extra JSON, line_hash, user_id, ingested_at` + FTS columns `text_ngram`/`text_word` (lowercased; see /memhouse:search) |
|
|
120
|
+
| `tool_calls` | table, 1 row/tool call | `session_id, seq, idx, source, host, tool_name, args, ts, project, folder, user_id, ingested_at` |
|
|
134
121
|
| the rollup | **saved query**, not an object — `memhouse sessions-query` prints it | `session_id, source, host, name, mode, folder, project, git_branch, user_id, created_at, last_updated_at, started, ended, duration_sec, total_msgs, user_msgs, assistant_msgs, subagent_msgs, models, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, user_chars, assistant_chars, first_prompt` |
|
|
135
122
|
|
|
136
123
|
Notes: `source` = editor id (`claude-code`, `codex`, `cursor`, `cursor-agent`,
|
|
@@ -147,7 +134,7 @@ Token spend by model:
|
|
|
147
134
|
```sql
|
|
148
135
|
SELECT model, sum(input_tokens) AS in_tok, sum(output_tokens) AS out_tok,
|
|
149
136
|
sum(cache_read_tokens) AS cache_read
|
|
150
|
-
FROM
|
|
137
|
+
FROM messages
|
|
151
138
|
WHERE model NOT IN ('', '<synthetic>')
|
|
152
139
|
GROUP BY model
|
|
153
140
|
ORDER BY out_tok DESC
|
|
@@ -158,7 +145,7 @@ Which tools do I use most (memhouse exclusive — memory-house has no tool table
|
|
|
158
145
|
|
|
159
146
|
```sql
|
|
160
147
|
SELECT tool_name, count() AS calls, uniqExact(session_id) AS sessions
|
|
161
|
-
FROM
|
|
148
|
+
FROM tool_calls
|
|
162
149
|
GROUP BY tool_name
|
|
163
150
|
ORDER BY calls DESC
|
|
164
151
|
LIMIT 20
|
|
@@ -169,7 +156,7 @@ Busiest days, last two weeks:
|
|
|
169
156
|
|
|
170
157
|
```sql
|
|
171
158
|
SELECT toDate(ts) AS day, uniqExact(session_id) AS sessions, count() AS msgs
|
|
172
|
-
FROM
|
|
159
|
+
FROM messages
|
|
173
160
|
WHERE ts > now() - INTERVAL 14 DAY
|
|
174
161
|
GROUP BY day
|
|
175
162
|
ORDER BY day DESC
|
|
@@ -180,7 +167,7 @@ Activity by editor this week / subagent share:
|
|
|
180
167
|
|
|
181
168
|
```sql
|
|
182
169
|
SELECT source, count() AS msgs, countIf(is_subagent) AS subagent_msgs
|
|
183
|
-
FROM
|
|
170
|
+
FROM messages
|
|
184
171
|
WHERE ts > now() - INTERVAL 7 DAY
|
|
185
172
|
GROUP BY source
|
|
186
173
|
ORDER BY msgs DESC
|
package/memhouse/host.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Which machine shipped this row.
|
|
2
2
|
//
|
|
3
3
|
// All of one member's machines ship into that member's rooms — that is the point of the
|
|
4
|
-
//
|
|
4
|
+
// shared-house layout — and the `host` column is the only thing telling them apart. So the
|
|
5
5
|
// value has to be BOTH unique per machine and stable across that machine's lifetime.
|
|
6
6
|
//
|
|
7
7
|
// It used to be derived, `sha256(hostname|platform|arch)`, and that fails in both
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# The house — a database, three shared rooms
|
|
2
|
+
|
|
3
|
+
**A house is a ClickHouse database.** Any database, named whatever its people name it —
|
|
4
|
+
`polat`, `team_a`, even `default`. Its rooms are three plain tables: `sessions`,
|
|
5
|
+
`messages`, `tool_calls`. There is nothing else to provision: no per-member tables, no
|
|
6
|
+
Merge rooms, no views, no settings profile.
|
|
7
|
+
|
|
8
|
+
**Everyone in the house writes into the same tables**, each with their own credential,
|
|
9
|
+
and two columns say where every row came from:
|
|
10
|
+
|
|
11
|
+
| Column | Meaning | Comes from |
|
|
12
|
+
|---|---|---|
|
|
13
|
+
| `user_id` | who wrote it | `String MATERIALIZED currentUser()` — stamped by the server, unforgeable by clients. `async_insert = 0 CONST` is pinned on each user (`ADD SETTING`, which merges — a bare `SETTINGS` clause replaces the user's whole list) because an async flush stores the stamp as the empty string (measured on 25.11.9.34) |
|
|
14
|
+
| `host` | which machine | the install's fingerprint (`../host.js`) — random, minted once, so two machines with the same hostname cannot collide and renaming one cannot split its history |
|
|
15
|
+
|
|
16
|
+
`WHERE user_id = 'alice'` is one person. `WHERE host = 'macbook-4127a95b'` is one
|
|
17
|
+
machine. No filter is the whole house — which is what a team dashboard wants.
|
|
18
|
+
|
|
19
|
+
## Joining a house
|
|
20
|
+
|
|
21
|
+
The whole tenancy model is the database boundary:
|
|
22
|
+
|
|
23
|
+
```sql
|
|
24
|
+
CREATE USER alice IDENTIFIED BY '…';
|
|
25
|
+
GRANT ALL ON team_a.* TO alice;
|
|
26
|
+
ALTER USER alice ADD SETTING async_insert = 0 CONST;
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`ALL` on your own house reaches nothing outside it, and it is what lets alice's own
|
|
30
|
+
shipper create and evolve the rooms (`ship.js --ensure-schema`) — no admin has to run
|
|
31
|
+
memhouse code. Joining a ClickHouse that happens to run a kernel is the same three
|
|
32
|
+
statements: a database and a credential, nothing else.
|
|
33
|
+
|
|
34
|
+
The model is **collaborative**: housemates trust each other with the house. Narrower
|
|
35
|
+
grants can be layered on later if a team wants them; the isolation mechanism between
|
|
36
|
+
groups that do NOT trust each other is a separate house.
|
|
37
|
+
|
|
38
|
+
## One user on two machines, same session id
|
|
39
|
+
|
|
40
|
+
Session ids are minted per machine, so this needs a synced home directory to happen at
|
|
41
|
+
all — but when it does (the same `~/.claude` on two synced machines), both shippers ship
|
|
42
|
+
the same `(session_id, user_id)`. The keys deliberately do NOT include `host`, so the
|
|
43
|
+
rows collapse to the LAST shipper's copy: one logical session stays one session, with
|
|
44
|
+
`host` recording who shipped it most recently. Keying on host instead would duplicate
|
|
45
|
+
every such session and double what the rollup counts. An interleaved clear+insert
|
|
46
|
+
between the two shippers can transiently drop rows; the next pass's skip predicate sees
|
|
47
|
+
the count mismatch and re-ships — self-healing, at the price of churn. Truly parallel
|
|
48
|
+
shippers per user are the deferred "ephemeral fingerprint" expert feature, not today's.
|
|
49
|
+
|
|
50
|
+
## The rollup is a saved query, not an object
|
|
51
|
+
|
|
52
|
+
`sessions_v` is SQL text (house.js `sessionsRollup`), substituted into the same
|
|
53
|
+
`FROM … AS c` position a view name would occupy and run under the caller's own
|
|
54
|
+
credential. It groups by `(session_id, user_id)` — two housemates' rows never merge,
|
|
55
|
+
even on a colliding session_id — and carries its own `FINAL` and
|
|
56
|
+
`SETTINGS join_use_nulls = 1`, so it is correct wherever it is pasted.
|
|
57
|
+
|
|
58
|
+
## What this replaced, and why
|
|
59
|
+
|
|
60
|
+
The per-member layout: rooms named `sessions_<member>` with a narrowed per-member grant
|
|
61
|
+
set, Merge rooms (`all_sessions`…) for team reads, and isolation between members of one
|
|
62
|
+
database. Retired because every piece of it was cost paid for an adversarial model the
|
|
63
|
+
product does not have:
|
|
64
|
+
|
|
65
|
+
- suffixed names made every client resolve its room names before writing SQL, and
|
|
66
|
+
`FROM messages` a documented trap the acceptance suite had to test for;
|
|
67
|
+
- the Merge rooms existed only to undo the splitting the suffixes created;
|
|
68
|
+
- the narrowed grant set existed to stop housemates attacking each other — but
|
|
69
|
+
housemates are collaborators, and groups that are not do not share a database.
|
|
70
|
+
|
|
71
|
+
The origin story that survives: rows carry `origin` (in the messages/tool_calls sorting
|
|
72
|
+
keys, deliberately NOT in sessions'), the shipper's clear binds
|
|
73
|
+
`(session_id, user_id, origin='ship')`, and imported history survives every re-ship.
|
|
74
|
+
See the comments in `schema.sql.tpl` — they are the authority on the keys.
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
// The house.
|
|
2
|
+
//
|
|
3
|
+
// A house is a ClickHouse DATABASE — any database, named whatever its people name it
|
|
4
|
+
// (`polat`, `team_a`, even `default`). Its rooms are three plain tables: `sessions`,
|
|
5
|
+
// `messages`, `tool_calls`. Everyone in the house writes into the SAME tables, each with
|
|
6
|
+
// their own credential, and two columns say who and where every row came from:
|
|
7
|
+
//
|
|
8
|
+
// user_id String MATERIALIZED currentUser() — stamped by the server, unforgeable by
|
|
9
|
+
// clients (async_insert is pinned to 0
|
|
10
|
+
// on the user so the stamp cannot be
|
|
11
|
+
// skipped)
|
|
12
|
+
// host LowCardinality(String) — the machine's fingerprint, minted once
|
|
13
|
+
// per install (../host.js)
|
|
14
|
+
//
|
|
15
|
+
// That pair IS the provenance model. Alice and Bob point their shippers at `team_a` and
|
|
16
|
+
// both write; `WHERE user_id = 'alice'` is one person, `WHERE host = '…'` is one machine.
|
|
17
|
+
// The model is collaborative — housemates trust each other with the house — and the
|
|
18
|
+
// boundary between houses is the database: joining someone's ClickHouse takes a database
|
|
19
|
+
// and a credential (`GRANT ALL ON team_a.* TO alice`), nothing else, because ALL on your
|
|
20
|
+
// own database reaches nothing outside it.
|
|
21
|
+
//
|
|
22
|
+
// THIS REPLACES THE PER-MEMBER LAYOUT. Rooms were `sessions_<member>` with per-member
|
|
23
|
+
// grants, Merge rooms for team reads, and isolation between members of one database.
|
|
24
|
+
// All of it is gone: suffixed names made every client resolve its room names first (and
|
|
25
|
+
// `FROM messages` a documented trap), the Merge rooms existed only to undo the
|
|
26
|
+
// splitting, and per-member isolation inside a shared house solved an adversarial
|
|
27
|
+
// problem the product does not have. Separate houses isolate; one house shares.
|
|
28
|
+
//
|
|
29
|
+
// `sessions_v` IS A SAVED QUERY, NOT AN OBJECT — substituted into the same `FROM … AS c`
|
|
30
|
+
// position a view name would occupy, running under the caller's own credential. It was
|
|
31
|
+
// briefly a stored view; that cost an object to provision and grant, and a name for the
|
|
32
|
+
// old Merge selectors to swallow.
|
|
33
|
+
|
|
34
|
+
const ROOM_TYPES = ['sessions', 'messages', 'tool_calls'];
|
|
35
|
+
|
|
36
|
+
// The one server-side setting a member carries, applied directly on the user with
|
|
37
|
+
// ADD SETTING (bare `ALTER USER … SETTINGS` REPLACES the user's whole list — measured, it
|
|
38
|
+
// wiped an operator-set ceiling; ADD SETTING merges and upserts). There is no settings
|
|
39
|
+
// profile: the shared object detached members on replace and its server-global name made
|
|
40
|
+
// two houses on one server rewrite each other's.
|
|
41
|
+
//
|
|
42
|
+
// The pin is correctness, not policy. `user_id MATERIALIZED currentUser()` is computed
|
|
43
|
+
// during the INSERT, and an ASYNC insert flushes outside that context — measured on
|
|
44
|
+
// 25.11.9.34, `async_insert=1` stores user_id as the EMPTY STRING while a sync insert
|
|
45
|
+
// stamps correctly. The shipper always passes async_insert=0; the pin guards every OTHER
|
|
46
|
+
// holder of the credential. CONST refuses the override; a client that never mentions the
|
|
47
|
+
// setting is held at 0 and its insert lands stamped.
|
|
48
|
+
const MEMBER_PIN = 'async_insert = 0 CONST';
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The global-install command that will actually work HERE.
|
|
52
|
+
*
|
|
53
|
+
* Every site that printed `npm install -g memhouse --allow-scripts=better-sqlite3` printed
|
|
54
|
+
* a line that fails with EACCES on any distro-packaged Node, because the global prefix is
|
|
55
|
+
* root-owned — measured on a clean Ubuntu machine, exit 243. `sudo` is right for exactly
|
|
56
|
+
* one of the two causes and makes the other worse, and what tells them apart is who owns
|
|
57
|
+
* the prefix — so look, rather than guess.
|
|
58
|
+
*/
|
|
59
|
+
function installCommand() {
|
|
60
|
+
const base = 'npm install -g memhouse --allow-scripts=better-sqlite3';
|
|
61
|
+
try {
|
|
62
|
+
const { execFileSync } = require('child_process');
|
|
63
|
+
const prefix = execFileSync('npm', ['prefix', '-g'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
64
|
+
const st = require('fs').statSync(prefix);
|
|
65
|
+
if (st.uid !== process.getuid()) return `sudo ${base}`;
|
|
66
|
+
} catch { /* npm not on PATH, or an unreadable prefix: fall through to the plain form */ }
|
|
67
|
+
return base;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Identifier check for anything we splice into DDL unquoted — house (database) names and
|
|
71
|
+
// user names. Refuse what would need quoting rather than quote it: a name a person cannot
|
|
72
|
+
// type bare into their own SQL is a name they will get wrong. `default` is deliberately
|
|
73
|
+
// ALLOWED as a house name — a stock container's default database is a real place to keep
|
|
74
|
+
// a house, and the pilot may pick user 'alice' with house 'default'. Only the names that
|
|
75
|
+
// are not a database at all are refused.
|
|
76
|
+
const RESERVED_DBS = new Set(['system', 'information_schema']);
|
|
77
|
+
function assertUsableName(name, what = 'name') {
|
|
78
|
+
if (typeof name !== 'string' || !/^[A-Za-z][A-Za-z0-9_]*$/.test(name)) {
|
|
79
|
+
throw new Error(`'${name}' cannot be a ${what} — expected [A-Za-z][A-Za-z0-9_]*`);
|
|
80
|
+
}
|
|
81
|
+
if (RESERVED_DBS.has(name.toLowerCase())) {
|
|
82
|
+
throw new Error(`'${name}' cannot be a ${what} — it is ClickHouse's own`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The session rollup as SQL text, usable in the same `FROM ... AS c` position a view name
|
|
88
|
+
* would occupy. Groups by (session_id, user_id): two housemates' rows never merge, even
|
|
89
|
+
* on a colliding session_id.
|
|
90
|
+
*/
|
|
91
|
+
function sessionsRollup({ sessions, messages }) {
|
|
92
|
+
return `(
|
|
93
|
+
SELECT
|
|
94
|
+
s.session_id AS session_id,
|
|
95
|
+
any(s.source) AS source,
|
|
96
|
+
any(s.host) AS host,
|
|
97
|
+
any(s.name) AS name,
|
|
98
|
+
any(s.mode) AS mode,
|
|
99
|
+
any(s.folder) AS folder,
|
|
100
|
+
any(s.project) AS project,
|
|
101
|
+
any(s.git_branch) AS git_branch,
|
|
102
|
+
s.user_id AS user_id,
|
|
103
|
+
any(s.created_at) AS created_at,
|
|
104
|
+
any(s.last_updated_at) AS last_updated_at,
|
|
105
|
+
min(m.ts) AS started,
|
|
106
|
+
max(m.ts) AS ended,
|
|
107
|
+
coalesce(dateDiff('second', min(m.ts), max(m.ts)), 0) AS duration_sec,
|
|
108
|
+
count(m.seq) AS total_msgs,
|
|
109
|
+
countIf(m.role = 'user') AS user_msgs,
|
|
110
|
+
countIf(m.role = 'assistant') AS assistant_msgs,
|
|
111
|
+
countIf(m.is_subagent) AS subagent_msgs,
|
|
112
|
+
groupUniqArrayIf(m.model, m.model NOT IN ('', '<synthetic>')) AS models,
|
|
113
|
+
coalesce(sum(m.input_tokens), 0) AS input_tokens,
|
|
114
|
+
coalesce(sum(m.output_tokens), 0) AS output_tokens,
|
|
115
|
+
coalesce(sum(m.cache_read_tokens), 0) AS cache_read_tokens,
|
|
116
|
+
coalesce(sum(m.cache_write_tokens), 0) AS cache_write_tokens,
|
|
117
|
+
coalesce(sumIf(length(m.text), m.role = 'user'), 0) AS user_chars,
|
|
118
|
+
coalesce(sumIf(length(m.text), m.role = 'assistant'), 0) AS assistant_chars,
|
|
119
|
+
coalesce(substring(argMinIf(m.text, m.seq, m.role = 'user' AND m.text != ''), 1, 200), '') AS first_prompt
|
|
120
|
+
FROM ${sessions} AS s FINAL
|
|
121
|
+
LEFT JOIN ${messages} AS m FINAL ON m.session_id = s.session_id AND m.user_id = s.user_id
|
|
122
|
+
GROUP BY s.session_id, s.user_id
|
|
123
|
+
SETTINGS join_use_nulls = 1
|
|
124
|
+
)`;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* The settings every read needs. `final` collapses ReplacingMergeTree versions;
|
|
129
|
+
* `join_use_nulls` is what the rollup's coalesce depends on.
|
|
130
|
+
*/
|
|
131
|
+
const READ_SETTINGS = { final: 1, join_use_nulls: 1 };
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Room names. They are constants now — `sessions`, `messages`, `tool_calls`, resolved by
|
|
135
|
+
* the connection's database rather than by who is asking — but the shape survives from
|
|
136
|
+
* the per-member layout so every consumer keeps addressing rooms the same way, and
|
|
137
|
+
* `user` still carries the server-reported identity: writers need the VALUE, not the
|
|
138
|
+
* name it produces. See ship.js's delete, which must BIND the user rather than call
|
|
139
|
+
* currentUser() inside a mutation, where it is not evaluated in the caller's context.
|
|
140
|
+
*/
|
|
141
|
+
function roomNames(user) {
|
|
142
|
+
const out = { member: user, user };
|
|
143
|
+
for (const t of ROOM_TYPES) out[t] = t;
|
|
144
|
+
out.sessions_v = sessionsRollup(out);
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function currentUser(client) {
|
|
149
|
+
const rs = await client.query({ query: 'SELECT currentUser() AS u', format: 'JSONEachRow' });
|
|
150
|
+
const rows = await rs.json();
|
|
151
|
+
const u = rows[0] && rows[0].u;
|
|
152
|
+
if (!u) throw new Error('could not determine currentUser()');
|
|
153
|
+
return u;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Resolve the rooms for whoever this client is connected as. */
|
|
157
|
+
async function resolveRooms(client) {
|
|
158
|
+
return roomNames(await currentUser(client));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
module.exports = {
|
|
162
|
+
ROOM_TYPES, READ_SETTINGS, MEMBER_PIN,
|
|
163
|
+
installCommand, assertUsableName,
|
|
164
|
+
sessionsRollup, roomNames, currentUser, resolveRooms,
|
|
165
|
+
};
|