memhouse 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -5
- package/bin/memhouse.js +75 -3
- package/editors/adapter-errors.js +52 -0
- package/editors/antigravity.js +32 -3
- package/editors/cursor.js +67 -7
- package/editors/goose.js +35 -7
- package/editors/index.js +97 -2
- package/editors/opencode.js +71 -13
- package/editors/zed.js +75 -10
- package/mem-house/delivery/AGENT-INSTALL.md +53 -23
- package/mem-house/delivery/kernel-install.md +15 -8
- package/mem-house/shipper/ship.js +63 -4
- package/package.json +5 -2
- package/public/assets/{index-BsgjOSTv.js → index-CCPh7a2Z.js} +2 -2
- package/public/index.html +1 -1
- package/ui/src/pages/SqlViewer.jsx +35 -9
package/editors/opencode.js
CHANGED
|
@@ -2,6 +2,7 @@ const path = require('path');
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const os = require('os');
|
|
4
4
|
const Database = require('better-sqlite3');
|
|
5
|
+
const adapterErrors = require('./adapter-errors');
|
|
5
6
|
|
|
6
7
|
// OpenCode stores data in XDG-style paths across all platforms
|
|
7
8
|
function getOpenCodeStoragePath() {
|
|
@@ -31,7 +32,10 @@ function queryDb(sql) {
|
|
|
31
32
|
const rows = db.prepare(sql).all();
|
|
32
33
|
db.close();
|
|
33
34
|
return rows;
|
|
34
|
-
} catch {
|
|
35
|
+
} catch (e) {
|
|
36
|
+
// The store exists but could not be read. Returning [] keeps one bad database
|
|
37
|
+
// from killing the scan, but staying quiet would hide missing sessions.
|
|
38
|
+
adapterErrors.record('opencode', e, DB_PATH);
|
|
35
39
|
return [];
|
|
36
40
|
}
|
|
37
41
|
}
|
|
@@ -90,7 +94,12 @@ function getSqliteMessages(sessionId) {
|
|
|
90
94
|
const result = [];
|
|
91
95
|
for (const msg of messages) {
|
|
92
96
|
let msgData;
|
|
93
|
-
|
|
97
|
+
// A record we cannot decode is dropped from the transcript. Silently, that
|
|
98
|
+
// yields a TRUNCATED history that looks complete — and a re-ship would replace
|
|
99
|
+
// the full stored transcript with the shortened one. Report it so the shipper
|
|
100
|
+
// refuses to overwrite good data with a partial read.
|
|
101
|
+
try { msgData = JSON.parse(msg.msg_data); }
|
|
102
|
+
catch (e) { adapterErrors.record('opencode', e, `message ${msg.msg_id}`); continue; }
|
|
94
103
|
|
|
95
104
|
const role = msgData.role;
|
|
96
105
|
if (!role) continue;
|
|
@@ -102,7 +111,8 @@ function getSqliteMessages(sessionId) {
|
|
|
102
111
|
const contentParts = [];
|
|
103
112
|
for (const part of parts) {
|
|
104
113
|
let partData;
|
|
105
|
-
try { partData = JSON.parse(part.data); }
|
|
114
|
+
try { partData = JSON.parse(part.data); }
|
|
115
|
+
catch (e) { adapterErrors.record('opencode', e, `part of message ${msg.msg_id}`); continue; }
|
|
106
116
|
const type = partData.type;
|
|
107
117
|
|
|
108
118
|
if (type === 'text' && partData.text) {
|
|
@@ -143,7 +153,10 @@ function getSqliteMessages(sessionId) {
|
|
|
143
153
|
|
|
144
154
|
db.close();
|
|
145
155
|
return result;
|
|
146
|
-
} catch {
|
|
156
|
+
} catch (e) {
|
|
157
|
+
// Message reads must report too, not just the session scan: an empty transcript
|
|
158
|
+
// is written over the stored one on re-ship.
|
|
159
|
+
adapterErrors.record('opencode', e, DB_PATH);
|
|
147
160
|
return [];
|
|
148
161
|
}
|
|
149
162
|
}
|
|
@@ -167,26 +180,41 @@ function getAllSessions() {
|
|
|
167
180
|
if (!fs.statSync(projectDir).isDirectory()) continue;
|
|
168
181
|
|
|
169
182
|
let files;
|
|
170
|
-
|
|
183
|
+
// Discovery-time drop: the project directory is there but will not enumerate, so
|
|
184
|
+
// every session under it vanishes before anything downstream can miss it.
|
|
185
|
+
try { files = fs.readdirSync(projectDir).filter(f => f.startsWith('ses_') && f.endsWith('.json')); }
|
|
186
|
+
catch (e) { adapterErrors.record('opencode', e, projectDir); continue; }
|
|
171
187
|
|
|
172
188
|
for (const file of files) {
|
|
173
189
|
const filePath = path.join(projectDir, file);
|
|
174
190
|
const data = readJson(filePath);
|
|
175
191
|
if (data && data.id) {
|
|
176
192
|
sessions.push({ ...data, _filePath: filePath });
|
|
193
|
+
} else {
|
|
194
|
+
// The file was listed, so it exists. Failing to decode it drops the session
|
|
195
|
+
// from discovery entirely — it reaches neither doctor nor the shipper, and
|
|
196
|
+
// nothing downstream can miss what it never saw.
|
|
197
|
+
adapterErrors.record('opencode', new Error('unreadable session metadata file'), filePath);
|
|
177
198
|
}
|
|
178
199
|
}
|
|
179
200
|
}
|
|
180
201
|
return sessions;
|
|
181
202
|
}
|
|
182
203
|
|
|
204
|
+
// Returns the message count, or null when it could not be determined. Null and 0
|
|
205
|
+
// are NOT interchangeable: bubbleCount feeds the shipper's skip predicate, and a
|
|
206
|
+
// zero standing in for "unknown" makes `prev.bc >= 0` trivially true, so the session
|
|
207
|
+
// is skipped before anything can read it — stale forever, on every incremental pass.
|
|
183
208
|
function getMessageCount(sessionId) {
|
|
184
209
|
const sessionMsgDir = path.join(MESSAGE_DIR, sessionId);
|
|
185
210
|
if (!fs.existsSync(sessionMsgDir)) return 0;
|
|
186
211
|
|
|
187
212
|
try {
|
|
188
213
|
return fs.readdirSync(sessionMsgDir).filter(f => f.startsWith('msg_') && f.endsWith('.json')).length;
|
|
189
|
-
} catch {
|
|
214
|
+
} catch (e) {
|
|
215
|
+
adapterErrors.record('opencode', e, sessionMsgDir);
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
190
218
|
}
|
|
191
219
|
|
|
192
220
|
function getMessagesForSession(sessionId) {
|
|
@@ -194,13 +222,22 @@ function getMessagesForSession(sessionId) {
|
|
|
194
222
|
if (!fs.existsSync(sessionMsgDir)) return [];
|
|
195
223
|
|
|
196
224
|
let files;
|
|
197
|
-
try { files = fs.readdirSync(sessionMsgDir).filter(f => f.startsWith('msg_') && f.endsWith('.json')); }
|
|
225
|
+
try { files = fs.readdirSync(sessionMsgDir).filter(f => f.startsWith('msg_') && f.endsWith('.json')); }
|
|
226
|
+
catch (e) {
|
|
227
|
+
// The directory exists (checked above) but is unreadable — report it, or the
|
|
228
|
+
// empty result overwrites the stored transcript on re-ship.
|
|
229
|
+
adapterErrors.record('opencode', e, sessionMsgDir);
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
198
232
|
|
|
199
233
|
const rawMsgs = [];
|
|
200
234
|
for (const file of files) {
|
|
201
235
|
const msgPath = path.join(sessionMsgDir, file);
|
|
202
236
|
const msg = readJson(msgPath);
|
|
203
|
-
|
|
237
|
+
// The file was listed, so it exists; failing to decode it drops a message from
|
|
238
|
+
// the transcript and would let a re-ship overwrite the stored copy with less.
|
|
239
|
+
if (!msg) { adapterErrors.record('opencode', new Error('unreadable message file'), msgPath); continue; }
|
|
240
|
+
if (!msg.id) continue;
|
|
204
241
|
rawMsgs.push(msg);
|
|
205
242
|
}
|
|
206
243
|
|
|
@@ -216,10 +253,17 @@ function getMessagesForSession(sessionId) {
|
|
|
216
253
|
try {
|
|
217
254
|
const partFiles = fs.readdirSync(msgPartDir).filter(f => f.startsWith('prt_') && f.endsWith('.json'));
|
|
218
255
|
for (const partFile of partFiles) {
|
|
219
|
-
const
|
|
256
|
+
const partPath = path.join(msgPartDir, partFile);
|
|
257
|
+
const part = readJson(partPath);
|
|
220
258
|
if (part) parts.push(part);
|
|
259
|
+
else adapterErrors.record('opencode', new Error('unreadable part file'), partPath);
|
|
221
260
|
}
|
|
222
|
-
} catch {
|
|
261
|
+
} catch (e) {
|
|
262
|
+
// The directory exists (checked above) but cannot be enumerated, so none of
|
|
263
|
+
// the per-file records above ran and the message collapses to a bare [role]
|
|
264
|
+
// placeholder. Unreported, that truncation would overwrite stored content.
|
|
265
|
+
adapterErrors.record('opencode', e, msgPartDir);
|
|
266
|
+
}
|
|
223
267
|
}
|
|
224
268
|
|
|
225
269
|
// Build content from parts
|
|
@@ -285,6 +329,7 @@ function getChats() {
|
|
|
285
329
|
const fileSessions = getAllSessions();
|
|
286
330
|
for (const s of fileSessions) {
|
|
287
331
|
seen.add(s.id);
|
|
332
|
+
const msgCount = getMessageCount(s.id);
|
|
288
333
|
chats.push({
|
|
289
334
|
source: 'opencode',
|
|
290
335
|
composerId: s.id,
|
|
@@ -294,7 +339,8 @@ function getChats() {
|
|
|
294
339
|
mode: s.mode || 'opencode',
|
|
295
340
|
folder: s.directory || null,
|
|
296
341
|
encrypted: false,
|
|
297
|
-
bubbleCount:
|
|
342
|
+
bubbleCount: msgCount ?? 0,
|
|
343
|
+
_countUnknown: msgCount === null,
|
|
298
344
|
_agent: s.agent,
|
|
299
345
|
_model: s.modelID,
|
|
300
346
|
_provider: s.providerID,
|
|
@@ -332,10 +378,22 @@ function cleanTitle(title) {
|
|
|
332
378
|
}
|
|
333
379
|
|
|
334
380
|
function getMessages(chat) {
|
|
335
|
-
// Prefer file-based messages; fall back to SQLite
|
|
381
|
+
// Prefer file-based messages; fall back to SQLite.
|
|
382
|
+
//
|
|
383
|
+
// Failures on the file path are only failures of the READ if the fallback also
|
|
384
|
+
// comes up empty. An unreadable message directory for a session that opencode.db
|
|
385
|
+
// still holds in full is not a reason to withhold the session — and leaving those
|
|
386
|
+
// entries in the sink would make the shipper do exactly that, forever. So they are
|
|
387
|
+
// retracted once SQLite produces a transcript. Scoped to the file attempt, so any
|
|
388
|
+
// truncation the SQLite reader reports about itself survives.
|
|
389
|
+
const beforeFile = adapterErrors.mark();
|
|
336
390
|
const fileMessages = getMessagesForSession(chat.composerId);
|
|
391
|
+
const afterFile = adapterErrors.mark();
|
|
337
392
|
if (fileMessages.length > 0) return fileMessages;
|
|
338
|
-
|
|
393
|
+
|
|
394
|
+
const sqliteMessages = getSqliteMessages(chat.composerId);
|
|
395
|
+
if (sqliteMessages.length > 0) adapterErrors.dropRange(beforeFile, afterFile);
|
|
396
|
+
return sqliteMessages;
|
|
339
397
|
}
|
|
340
398
|
|
|
341
399
|
const labels = { 'opencode': 'OpenCode' };
|
package/editors/zed.js
CHANGED
|
@@ -3,6 +3,7 @@ const fs = require('fs');
|
|
|
3
3
|
const os = require('os');
|
|
4
4
|
|
|
5
5
|
const Database = require('better-sqlite3');
|
|
6
|
+
const adapterErrors = require('./adapter-errors');
|
|
6
7
|
|
|
7
8
|
// Zed stores data in different locations depending on the platform
|
|
8
9
|
// - Windows: %LOCALAPPDATA%\Zed (not Roaming)
|
|
@@ -73,11 +74,51 @@ function queryDb(sql) {
|
|
|
73
74
|
db.close();
|
|
74
75
|
return rows;
|
|
75
76
|
} catch (e) {
|
|
76
|
-
//
|
|
77
|
+
// Locked, inaccessible, or corrupt. Keep the scan alive but say so — silence
|
|
78
|
+
// here reads as "the user does not use Zed".
|
|
79
|
+
adapterErrors.record('zed', e, THREADS_DB);
|
|
77
80
|
return [];
|
|
78
81
|
}
|
|
79
82
|
}
|
|
80
83
|
|
|
84
|
+
// Zed's `threads` schema varies by version: `worktree_branch` exists in some builds
|
|
85
|
+
// and not others, and older builds lack `created_at`. Naming a column that is not
|
|
86
|
+
// there fails the WHOLE query with "no such column", which cost every Zed thread on
|
|
87
|
+
// this machine — silently, until the adapter started reporting. So ask the table what
|
|
88
|
+
// it has rather than assuming. Cached: the schema cannot change while we scan.
|
|
89
|
+
let threadColumns = null;
|
|
90
|
+
function getThreadColumns() {
|
|
91
|
+
if (threadColumns) return threadColumns;
|
|
92
|
+
if (!fs.existsSync(THREADS_DB)) return new Set();
|
|
93
|
+
try {
|
|
94
|
+
const db = new Database(THREADS_DB, { readonly: true });
|
|
95
|
+
const cols = db.prepare('PRAGMA table_info(threads)').all().map((r) => r.name);
|
|
96
|
+
db.close();
|
|
97
|
+
// PRAGMA on a missing or renamed table SUCCEEDS with zero rows rather than
|
|
98
|
+
// throwing. Left unreported, that would cache an empty set, return no sessions,
|
|
99
|
+
// and look exactly like an unused Zed install — the same silent whole-adapter
|
|
100
|
+
// drop this function exists to end. Do not cache it either: the next scan should
|
|
101
|
+
// look again rather than inherit the verdict.
|
|
102
|
+
if (cols.length === 0) {
|
|
103
|
+
adapterErrors.record('zed', new Error('threads table missing or unreadable — incompatible Zed schema'), THREADS_DB);
|
|
104
|
+
return new Set();
|
|
105
|
+
}
|
|
106
|
+
threadColumns = new Set(cols);
|
|
107
|
+
return threadColumns;
|
|
108
|
+
} catch (e) {
|
|
109
|
+
adapterErrors.record('zed', e, THREADS_DB);
|
|
110
|
+
return new Set();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// index.js calls this between passes of a long-running `ship --loop`. Without it the
|
|
115
|
+
// column set would be fixed for the life of the process, so a Zed upgrade mid-run
|
|
116
|
+
// would keep querying a column that no longer exists — or keep ignoring one newly
|
|
117
|
+
// added — until someone restarted the shipper.
|
|
118
|
+
function resetCache() {
|
|
119
|
+
threadColumns = null;
|
|
120
|
+
}
|
|
121
|
+
|
|
81
122
|
function queryBlob(id) {
|
|
82
123
|
if (!fs.existsSync(THREADS_DB)) return null;
|
|
83
124
|
try {
|
|
@@ -85,7 +126,10 @@ function queryBlob(id) {
|
|
|
85
126
|
const row = db.prepare('SELECT data FROM threads WHERE id = ?').get(id);
|
|
86
127
|
db.close();
|
|
87
128
|
return row ? row.data : null;
|
|
88
|
-
} catch {
|
|
129
|
+
} catch (e) {
|
|
130
|
+
// This is the message-blob read. A silent null here becomes a zero-message
|
|
131
|
+
// session, which overwrites the stored transcript on re-ship.
|
|
132
|
+
adapterErrors.record('zed', e, THREADS_DB);
|
|
89
133
|
return null;
|
|
90
134
|
}
|
|
91
135
|
}
|
|
@@ -97,22 +141,33 @@ function queryBlob(id) {
|
|
|
97
141
|
const name = 'zed';
|
|
98
142
|
|
|
99
143
|
function getChats() {
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
144
|
+
const cols = getThreadColumns();
|
|
145
|
+
if (cols.size === 0) return [];
|
|
146
|
+
|
|
147
|
+
// Only these are relied on. Anything else is taken when the build provides it.
|
|
148
|
+
const select = ['id', 'summary', 'updated_at', 'data_type', 'length(data) as data_size', 'parent_id'];
|
|
149
|
+
const hasBranch = cols.has('worktree_branch');
|
|
150
|
+
const hasCreated = cols.has('created_at');
|
|
151
|
+
if (hasBranch) select.push('worktree_branch');
|
|
152
|
+
if (hasCreated) select.push('created_at');
|
|
153
|
+
|
|
154
|
+
const rows = queryDb(`SELECT ${select.join(', ')} FROM threads ORDER BY updated_at DESC`);
|
|
103
155
|
|
|
104
156
|
return rows.map(row => ({
|
|
105
157
|
source: 'zed',
|
|
106
158
|
composerId: row.id,
|
|
107
159
|
name: row.summary || null,
|
|
108
|
-
|
|
160
|
+
// created_at when the build has it. Falling back to updated_at is not a
|
|
161
|
+
// creation time — it just keeps the field non-null on builds that lack it.
|
|
162
|
+
createdAt: (hasCreated && row.created_at) ? new Date(row.created_at).getTime()
|
|
163
|
+
: (row.updated_at ? new Date(row.updated_at).getTime() : null),
|
|
109
164
|
lastUpdatedAt: row.updated_at ? new Date(row.updated_at).getTime() : null,
|
|
110
165
|
mode: 'thread',
|
|
111
166
|
folder: null,
|
|
112
167
|
encrypted: false,
|
|
113
168
|
bubbleCount: 0,
|
|
114
169
|
_dataType: row.data_type,
|
|
115
|
-
_gitBranch: row.worktree_branch,
|
|
170
|
+
_gitBranch: hasBranch ? row.worktree_branch : null,
|
|
116
171
|
}));
|
|
117
172
|
}
|
|
118
173
|
|
|
@@ -129,12 +184,22 @@ function getMessages(chat) {
|
|
|
129
184
|
json = blob.toString('utf-8');
|
|
130
185
|
}
|
|
131
186
|
} catch (e) {
|
|
132
|
-
// Decompression failed
|
|
187
|
+
// Decompression failed — usually no zstd CLI and no native zstd in this Node.
|
|
188
|
+
// The thread exists and has content we cannot read, so this must be reported:
|
|
189
|
+
// silently returning [] would overwrite its stored transcript on re-ship.
|
|
190
|
+
adapterErrors.record('zed', e, THREADS_DB);
|
|
133
191
|
return [];
|
|
134
192
|
}
|
|
135
193
|
|
|
136
194
|
let data;
|
|
137
|
-
try { data = JSON.parse(json); }
|
|
195
|
+
try { data = JSON.parse(json); }
|
|
196
|
+
catch (e) {
|
|
197
|
+
// Decompressed but undecodable — malformed, or a thread schema this parser does
|
|
198
|
+
// not know. The thread has content, so returning [] unreported would let a
|
|
199
|
+
// re-ship replace the stored transcript with nothing.
|
|
200
|
+
adapterErrors.record('zed', e, THREADS_DB);
|
|
201
|
+
return [];
|
|
202
|
+
}
|
|
138
203
|
|
|
139
204
|
const model = data.model?.model || null;
|
|
140
205
|
const messages = [];
|
|
@@ -210,4 +275,4 @@ function getMCPServers() {
|
|
|
210
275
|
return results;
|
|
211
276
|
}
|
|
212
277
|
|
|
213
|
-
module.exports = { name, labels, getChats, getMessages, getMCPServers };
|
|
278
|
+
module.exports = { name, labels, getChats, getMessages, getMCPServers, resetCache };
|
|
@@ -1,21 +1,48 @@
|
|
|
1
1
|
# mem-house — agent install
|
|
2
2
|
|
|
3
3
|
Imperative steps for an AI coding agent installing mem-house on a machine. Execute
|
|
4
|
-
in order; every command is copy-pasteable. Prereqs: Node >= 20.19
|
|
4
|
+
in order; every command is copy-pasteable. Prereqs: Node >= 20.19 and a reachable
|
|
5
5
|
ClickHouse >= 26.2 (local server, Cloud, or a kernel house — see
|
|
6
|
-
`kernel-install.md` for the kernel path)
|
|
6
|
+
`kernel-install.md` for the kernel path). No checkout required — memhouse is on npm.
|
|
7
7
|
|
|
8
8
|
## 1. Install
|
|
9
9
|
|
|
10
10
|
```bash
|
|
11
|
-
|
|
12
|
-
npm install --no-audit --no-fund
|
|
11
|
+
npm install -g memhouse --allow-scripts=better-sqlite3
|
|
13
12
|
```
|
|
14
13
|
|
|
14
|
+
`--allow-scripts=better-sqlite3` is required, not cosmetic. Five adapters (cursor,
|
|
15
|
+
goose, opencode, zed, antigravity) read SQLite session stores, and npm 12 blocks
|
|
16
|
+
the install script that builds the native binding. Without it those five silently
|
|
17
|
+
return zero sessions and you ship a partial history. Verify with step 4 —
|
|
18
|
+
`memhouse discover` names any adapter it had to skip.
|
|
19
|
+
|
|
20
|
+
On `EACCES`, decide by **who owns the prefix root**. Not the path — Homebrew's
|
|
21
|
+
prefix is `/usr/local` on Intel and `/opt/homebrew` on Apple Silicon and is
|
|
22
|
+
user-owned in both. Not the failing file either — an earlier `sudo npm` leaves
|
|
23
|
+
root-owned files inside a prefix that is still the user's, and judging by those
|
|
24
|
+
misreads it as system-managed.
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
ls -ld "$(npm prefix -g)" # the prefix ROOT decides
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Prefix root owned by `root` → a system-managed Node; re-run as
|
|
31
|
+
`sudo npm install -g memhouse --allow-scripts=better-sqlite3`.
|
|
32
|
+
|
|
33
|
+
Prefix root owned by the current user → the prefix is theirs (Homebrew, fnm, nvm,
|
|
34
|
+
volta), so the root-owned file npm tripped on is a stray from an earlier `sudo npm`
|
|
35
|
+
and another `sudo` adds more. Repair that path instead:
|
|
36
|
+
`sudo chown -R "$(id -u):$(id -g)" <path npm named>` (same for `~/.npm`).
|
|
37
|
+
|
|
38
|
+
From a checkout instead (contributors): `npm install --no-audit --no-fund` in the
|
|
39
|
+
repo root — its `allowScripts` field already covers the binding — then substitute
|
|
40
|
+
`node bin/memhouse.js` for `memhouse` below.
|
|
41
|
+
|
|
15
42
|
## 2. Configure the connection
|
|
16
43
|
|
|
17
|
-
Write `~/.memhouse/env` (
|
|
18
|
-
|
|
44
|
+
Write `~/.memhouse/env` (or run `memhouse install --yes --url … --user … --password
|
|
45
|
+
… --db …`, which writes it for you):
|
|
19
46
|
|
|
20
47
|
```bash
|
|
21
48
|
mkdir -p ~/.memhouse
|
|
@@ -31,9 +58,7 @@ chmod 600 ~/.memhouse/env
|
|
|
31
58
|
## 3. Create the schema and run the first ship
|
|
32
59
|
|
|
33
60
|
```bash
|
|
34
|
-
|
|
35
|
-
node mem-house/shipper/ship.js --ensure-schema
|
|
36
|
-
node mem-house/shipper/ship.js # first full parse-on-client ship
|
|
61
|
+
memhouse install --yes # applies the schema, then runs the first full ship
|
|
37
62
|
```
|
|
38
63
|
|
|
39
64
|
Expected output shape: `[mem-house] shipped N sessions (0 skipped) → M msg rows, T
|
|
@@ -42,34 +67,39 @@ tool rows in Xs` with N in the hundreds on a machine with real agent usage.
|
|
|
42
67
|
## 4. Verify
|
|
43
68
|
|
|
44
69
|
```bash
|
|
45
|
-
|
|
70
|
+
memhouse discover # which adapters were read, and which were SKIPPED
|
|
71
|
+
memhouse stats # per-source counts now in the house
|
|
46
72
|
```
|
|
47
73
|
|
|
48
|
-
|
|
49
|
-
|
|
74
|
+
Read `discover` first. It prints one line per editor it found sessions for, and
|
|
75
|
+
warns by name about any adapter it had to skip — that warning is the only signal
|
|
76
|
+
that step 1's `--allow-scripts` flag was missed and five editors are dark.
|
|
77
|
+
|
|
78
|
+
`stats` should show one row per source (claude-code, codex, gemini-cli, …) with
|
|
79
|
+
non-zero sessions/messages. Or verify by SQL:
|
|
50
80
|
|
|
51
81
|
```bash
|
|
52
82
|
curl -s -u "$MEMHOUSE_USER:$MEMHOUSE_PASSWORD" "$MEMHOUSE_URL/?database=$MEMHOUSE_DB" \
|
|
53
83
|
--data-binary "SELECT source, count() FROM sessions_v GROUP BY source SETTINGS final=1 FORMAT PrettyCompact"
|
|
54
84
|
```
|
|
55
85
|
|
|
56
|
-
## 5. Start the dashboard
|
|
86
|
+
## 5. Start the dashboard and the shipper loop
|
|
57
87
|
|
|
58
88
|
```bash
|
|
59
|
-
|
|
60
|
-
|
|
89
|
+
memhouse start # both as background daemons; UI is prebuilt in the package
|
|
90
|
+
memhouse status # daemons, connection, counts, freshness (--json for agents)
|
|
61
91
|
```
|
|
62
92
|
|
|
63
|
-
|
|
93
|
+
The dashboard comes up on http://localhost:4640. `start` also runs the incremental
|
|
94
|
+
re-ship loop, so this replaces any separate cron. `memhouse stop` ends both.
|
|
64
95
|
|
|
65
|
-
|
|
96
|
+
## 6. Install the skills (optional, for Claude Code)
|
|
66
97
|
|
|
67
98
|
```bash
|
|
68
|
-
|
|
99
|
+
memhouse plugins install claude
|
|
69
100
|
```
|
|
70
101
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
Claude Code plugin. The skills read the connection from `~/.memhouse/env`.
|
|
102
|
+
Installs the `memhouse-search` / `memhouse-sessions` / `memhouse-sql` skills into
|
|
103
|
+
the agent's skills directory; they read the connection from `~/.memhouse/env`.
|
|
104
|
+
`memhouse plugins list` shows what is installed, `memhouse plugins remove claude`
|
|
105
|
+
undoes it.
|
|
@@ -23,17 +23,24 @@ the executor (running as the ego, the `kernel` user) creates database `memhouse`
|
|
|
23
23
|
(`realm_user` profile), and owner grants. It returns a one-time **credential** —
|
|
24
24
|
rotate on first connect (`issue-credential{user:'memhouse_root'}`).
|
|
25
25
|
|
|
26
|
-
## 2. Owner sets up the house
|
|
26
|
+
## 2. Owner sets up the house
|
|
27
27
|
|
|
28
28
|
```bash
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
29
|
+
npm install -g memhouse --allow-scripts=better-sqlite3
|
|
30
|
+
memhouse install --yes \
|
|
31
|
+
--url https://<kernel-host>:8443 \
|
|
32
|
+
--user memhouse_root \
|
|
33
|
+
--password <credential-from-provision> \
|
|
34
|
+
--db memhouse
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
+
That writes `~/.memhouse/env`, applies the schema, and runs the first ship.
|
|
38
|
+
`memhouse start` then keeps it fresh on a 300s loop.
|
|
39
|
+
|
|
40
|
+
Keep `--allow-scripts=better-sqlite3`: without it npm 12 leaves `better-sqlite3`
|
|
41
|
+
with no native binding and the five SQLite-backed adapters ship nothing, silently.
|
|
42
|
+
`memhouse discover` names any adapter it had to skip.
|
|
43
|
+
|
|
37
44
|
The kernel never runs or reads any of this — it is content-blind; the shipper is
|
|
38
45
|
the agency's own resident, a deterministic `worker` on a loop (no LLM, not an
|
|
39
46
|
injection surface). It is the only thing here that writes without being asked.
|
|
@@ -69,7 +76,7 @@ Pick one; do not mix on the same role.
|
|
|
69
76
|
## 5. Verify
|
|
70
77
|
|
|
71
78
|
```bash
|
|
72
|
-
|
|
79
|
+
memhouse stats # per-source counts as the owner
|
|
73
80
|
# as a member (own-only): counts reflect only that member's rows
|
|
74
81
|
curl -s -u "alice:<pw>" "$MEMHOUSE_URL/?database=memhouse" \
|
|
75
82
|
--data-binary "SELECT count() FROM sessions_v SETTINGS final=1"
|
|
@@ -28,7 +28,8 @@ const fs = require('fs');
|
|
|
28
28
|
const path = require('path');
|
|
29
29
|
const crypto = require('crypto');
|
|
30
30
|
const { createClient } = require('@clickhouse/client');
|
|
31
|
-
const { getAllChats, getMessages, resetCaches } = require('../../editors');
|
|
31
|
+
const { getAllChats, getAdapterErrors, getMessages, resetCaches } = require('../../editors');
|
|
32
|
+
const adapterErrorSink = require('../../editors/adapter-errors');
|
|
32
33
|
|
|
33
34
|
const BATCH_ROWS = 2000; // insert batch ceiling (binding)
|
|
34
35
|
const TEXT_MAX = 50000; // messages.text truncation
|
|
@@ -158,7 +159,15 @@ const TOOL_CALL_RE = /\[tool-call: ([^(\]]+)/g;
|
|
|
158
159
|
// absorbs the incremental skip for stable-empty chats).
|
|
159
160
|
function rowsForChat(chat, host) {
|
|
160
161
|
let messages;
|
|
161
|
-
try { messages = getMessages(chat) || []; }
|
|
162
|
+
try { messages = getMessages(chat) || []; }
|
|
163
|
+
catch (e) {
|
|
164
|
+
// Some readers throw instead of reporting through adapter-errors — Cursor's
|
|
165
|
+
// agent-store path on a bad blobs table, for one. Converting that to a bare
|
|
166
|
+
// `unreadable` count would leave the end-of-pass report with nothing to print,
|
|
167
|
+
// so operators would see a number rising every pass and no cause anywhere.
|
|
168
|
+
adapterErrorSink.record(chat.source, e, chat.composerId);
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
162
171
|
|
|
163
172
|
const source = chat.source;
|
|
164
173
|
// Canonical globally-unique session id: '<source>:<adapter-local id>'.
|
|
@@ -263,6 +272,15 @@ async function runShip(client, opts = {}) {
|
|
|
263
272
|
const existing = await loadExisting(client);
|
|
264
273
|
const chats = getAllChats();
|
|
265
274
|
|
|
275
|
+
// An adapter that cannot load contributes zero sessions, which is indistinguishable
|
|
276
|
+
// from an editor the user does not have — so the shipper would silently ship a
|
|
277
|
+
// partial history forever. Say so on every pass.
|
|
278
|
+
// Reported once here for what the scan already knows (a dead binding is visible
|
|
279
|
+
// before any work happens, and this pass may take minutes), and again after the
|
|
280
|
+
// loop for failures that only surface while reading messages.
|
|
281
|
+
const warned = new Set();
|
|
282
|
+
reportAdapterErrors(warned);
|
|
283
|
+
|
|
266
284
|
const batches = { sessions: [], messages: [], tool_calls: [] };
|
|
267
285
|
const flush = async (table) => {
|
|
268
286
|
if (!batches[table].length) return;
|
|
@@ -300,11 +318,33 @@ async function runShip(client, opts = {}) {
|
|
|
300
318
|
// row count == recorded message_count) so an interrupted re-ship — even one
|
|
301
319
|
// that died mid-flush leaving a partial transcript — repairs itself on the
|
|
302
320
|
// next pass instead of being skipped forever.
|
|
303
|
-
|
|
321
|
+
// _countUnknown means the adapter could not determine this chat's message
|
|
322
|
+
// count, so bubbleCount is a placeholder rather than a measurement. Skipping on
|
|
323
|
+
// it would compare against a number that means nothing — `prev.bc >= 0` is
|
|
324
|
+
// trivially true — and the session would stay stale forever. Re-read instead.
|
|
325
|
+
if (notNewer && !chat._countUnknown && prev.bc >= (chat.bubbleCount || 0) && prev.intact) { skipped++; continue; }
|
|
304
326
|
}
|
|
305
327
|
|
|
328
|
+
// rowsForChat returns null only when getMessages() *throws*. An adapter that
|
|
329
|
+
// swallows its own failure returns [] instead, which would look like a session
|
|
330
|
+
// that genuinely has no messages — and for a known session the re-ship below
|
|
331
|
+
// deletes the old transcript before inserting that emptiness. So treat a
|
|
332
|
+
// failure recorded while reading THIS chat as unreadable too: write nothing,
|
|
333
|
+
// delete nothing, retry next pass.
|
|
334
|
+
const errsBefore = adapterErrorSink.recorded().length;
|
|
306
335
|
const rows = rowsForChat(chat, host);
|
|
307
|
-
|
|
336
|
+
const failedHere = adapterErrorSink.recorded().slice(errsBefore).some((e) => e.source === chat.source);
|
|
337
|
+
// Write nothing on a failed read, whether or not the session is already stored.
|
|
338
|
+
//
|
|
339
|
+
// Shipping a partial first read looks tempting — nothing is there to destroy —
|
|
340
|
+
// but it is a trap: message_count would be written from the partial rows, so
|
|
341
|
+
// prev.intact becomes true, prev.bc already matches the source, the timestamp is
|
|
342
|
+
// unchanged, and the skip predicate above then withholds the session forever.
|
|
343
|
+
// The truncation would become permanent AND the warning would stop, because the
|
|
344
|
+
// adapter is never asked to read it again. Withholding keeps the session out of
|
|
345
|
+
// the skip predicate entirely, so every pass retries it and re-reports it until
|
|
346
|
+
// the underlying store is readable.
|
|
347
|
+
if (!rows || failedHere) { unreadable++; continue; }
|
|
308
348
|
if (prev) {
|
|
309
349
|
// Known session being re-shipped: clear its old rows BEFORE inserting so a
|
|
310
350
|
// shorter re-parse can't leave stale seq/idx tails. A crash between the
|
|
@@ -324,9 +364,28 @@ async function runShip(client, opts = {}) {
|
|
|
324
364
|
for (const r of rows.toolRows) { await push('tool_calls', r); toolRows++; }
|
|
325
365
|
}
|
|
326
366
|
for (const table of Object.keys(batches)) await flush(table);
|
|
367
|
+
// Anything that only failed while reading messages — the sink is reset by the
|
|
368
|
+
// next getAllChats(), so unreported here means never reported at all.
|
|
369
|
+
reportAdapterErrors(warned);
|
|
327
370
|
return { sessions, skipped, msgRows, toolRows, unreadable };
|
|
328
371
|
}
|
|
329
372
|
|
|
373
|
+
// Warn once per adapter per pass. `warned` carries across the two call sites so a
|
|
374
|
+
// dead binding is not reported twice in the same run.
|
|
375
|
+
function reportAdapterErrors(warned) {
|
|
376
|
+
const errors = getAdapterErrors().filter((e) => !warned.has(e.source));
|
|
377
|
+
if (!errors.length) return;
|
|
378
|
+
for (const e of errors) warned.add(e.source);
|
|
379
|
+
const noBinding = errors.filter((e) => e.missingBinding).map((e) => e.source);
|
|
380
|
+
if (noBinding.length) {
|
|
381
|
+
console.log(`[mem-house] WARNING: ${noBinding.length} adapter(s) skipped, sessions NOT shipped — better-sqlite3 has no native binding: ${noBinding.join(', ')}`);
|
|
382
|
+
console.log('[mem-house] fix: npm install -g memhouse --allow-scripts=better-sqlite3');
|
|
383
|
+
}
|
|
384
|
+
for (const e of errors.filter((x) => !x.missingBinding)) {
|
|
385
|
+
console.log(`[mem-house] WARNING: ${e.source} skipped — ${e.message}`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
330
389
|
// Per-source rollup straight from sessions_v (final=1 so ReplacingMergeTree collapses).
|
|
331
390
|
async function printStats(client) {
|
|
332
391
|
const rs = await client.query({
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "memhouse",
|
|
3
|
-
"version": "0.3.
|
|
4
|
-
"description": "memhouse
|
|
3
|
+
"version": "0.3.2",
|
|
4
|
+
"description": "memhouse — agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store, kernel-agency install, and the agentlytics dashboard",
|
|
5
5
|
"main": "bin/memhouse.js",
|
|
6
6
|
"bin": {
|
|
7
7
|
"memhouse": "./bin/memhouse.js",
|
|
@@ -56,6 +56,9 @@
|
|
|
56
56
|
"engines": {
|
|
57
57
|
"node": ">=20.19.0"
|
|
58
58
|
},
|
|
59
|
+
"allowScripts": {
|
|
60
|
+
"better-sqlite3": true
|
|
61
|
+
},
|
|
59
62
|
"repository": {
|
|
60
63
|
"type": "git",
|
|
61
64
|
"url": "https://github.com/agent-realm/memhouse"
|