memhouse 0.18.2 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/editors/claude.js +90 -10
- package/memhouse/delivery/plugin/.claude-plugin/plugin.json +1 -1
- package/memhouse/delivery/plugin/reference/HOUSE.md +17 -0
- package/memhouse/delivery/plugin/skills/recall/SKILL.md +5 -0
- package/memhouse/house/house.js +173 -1
- package/memhouse/server/queries.js +271 -137
- package/memhouse/shipper/ship.js +62 -4
- package/package.json +3 -2
package/editors/claude.js
CHANGED
|
@@ -195,11 +195,19 @@ function cleanPrompt(prompt) {
|
|
|
195
195
|
// Parse one Claude session .jsonl into the adapter's message shape. When
|
|
196
196
|
// `isSubagent` is set, each message is tagged `[subagent]` so folded subagent turns
|
|
197
197
|
// are clearly attributed in the transcript.
|
|
198
|
-
function parseSessionFile(filePath, isSubagent) {
|
|
198
|
+
function parseSessionFile(filePath, isSubagent, agent = null) {
|
|
199
199
|
const messages = [];
|
|
200
200
|
let lines;
|
|
201
201
|
try { lines = fs.readFileSync(filePath, 'utf-8').split('\n').filter(Boolean); } catch { return messages; }
|
|
202
202
|
const tag = isSubagent ? '[subagent] ' : '';
|
|
203
|
+
// A folded subagent turn used to carry only the `[subagent]` tag: the house held the
|
|
204
|
+
// turns but not WHICH subagent said them, so one fork's transcript could not be isolated,
|
|
205
|
+
// cited, or told apart from a sibling's. An agent looking for the fork as its own session
|
|
206
|
+
// concluded it was never shipped. Every turn now carries the agent's id, the description
|
|
207
|
+
// the parent gave it, and its position within that subagent — in `extra`, which is not
|
|
208
|
+
// part of the line hash, so nothing already shipped is re-shipped for this.
|
|
209
|
+
let turn = 0;
|
|
210
|
+
const stamp = (m) => { if (agent) { m._agent = { ...agent, turn: turn++ }; } return m; };
|
|
203
211
|
|
|
204
212
|
for (const line of lines) {
|
|
205
213
|
let obj;
|
|
@@ -223,19 +231,19 @@ function parseSessionFile(filePath, isSubagent) {
|
|
|
223
231
|
|
|
224
232
|
if (obj.type === 'user' && obj.message) {
|
|
225
233
|
const content = extractContent(obj.message.content);
|
|
226
|
-
if (content) messages.push({ role: 'user', content: tag + content, _ts: at });
|
|
234
|
+
if (content) messages.push(stamp({ role: 'user', content: tag + content, _ts: at }));
|
|
227
235
|
} else if (obj.type === 'assistant' && obj.message) {
|
|
228
236
|
const { text, toolCalls } = extractAssistantContent(obj.message.content);
|
|
229
237
|
const usage = obj.message.usage;
|
|
230
|
-
if (text) messages.push({
|
|
238
|
+
if (text) messages.push(stamp({
|
|
231
239
|
role: 'assistant', content: tag + text, _model: obj.message.model,
|
|
232
240
|
_inputTokens: usage?.input_tokens, _outputTokens: usage?.output_tokens,
|
|
233
241
|
_cacheRead: usage?.cache_read_input_tokens, _cacheWrite: usage?.cache_creation_input_tokens,
|
|
234
242
|
_toolCalls: toolCalls, _ts: at,
|
|
235
|
-
});
|
|
243
|
+
}));
|
|
236
244
|
} else if (obj.type === 'system') {
|
|
237
245
|
const text = typeof obj.message?.content === 'string' ? obj.message.content : '';
|
|
238
|
-
if (text) messages.push({ role: 'system', content: tag + text, _ts: at });
|
|
246
|
+
if (text) messages.push(stamp({ role: 'system', content: tag + text, _ts: at }));
|
|
239
247
|
}
|
|
240
248
|
}
|
|
241
249
|
return messages;
|
|
@@ -252,14 +260,86 @@ function getMessages(chat) {
|
|
|
252
260
|
// after the parent's turns, so nothing is lost and subagents are never
|
|
253
261
|
// double-counted as standalone sessions.
|
|
254
262
|
const subagentsDir = path.join(filePath.replace(/\.jsonl$/, ''), 'subagents');
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
263
|
+
const subFiles = subagentFiles(subagentsDir);
|
|
264
|
+
const named = subFiles.length ? subagentNames(filePath) : new Map();
|
|
265
|
+
for (const { file, workflow } of subFiles) {
|
|
266
|
+
const id = path.basename(file).replace(/^agent-/, '').replace(/\.jsonl$/, '');
|
|
267
|
+
const meta = named.get(id) || {};
|
|
268
|
+
const agent = { id, description: meta.description || '', type: meta.type || (workflow ? 'workflow-subagent' : ''), file: path.basename(file) };
|
|
269
|
+
if (workflow) agent.workflow = workflow;
|
|
270
|
+
const turns = parseSessionFile(file, true, agent);
|
|
271
|
+
// A fork the parent never named (a Workflow-run agent, or a parent from before the
|
|
272
|
+
// ids were reported) is described by its own first prompt — what it was told to do.
|
|
273
|
+
if (!agent.description) {
|
|
274
|
+
const first = turns.find((m) => m.role === 'user');
|
|
275
|
+
if (first) { const d = String(first.content).replace(/^\[subagent\] /, '').replace(/\s+/g, ' ').trim().slice(0, 160); for (const m of turns) m._agent.description = d; }
|
|
276
|
+
}
|
|
277
|
+
messages.push(...turns);
|
|
278
|
+
}
|
|
259
279
|
|
|
260
280
|
return messages;
|
|
261
281
|
}
|
|
262
282
|
|
|
283
|
+
/**
|
|
284
|
+
* Every subagent transcript under a session, wherever Claude Code put it:
|
|
285
|
+
* <session>/subagents/agent-<id>.jsonl — Agent tool forks
|
|
286
|
+
* <session>/subagents/workflows/wf_<id>/agent-<id>.jsonl — Workflow-run agents
|
|
287
|
+
* journal.jsonl and *.meta.json are bookkeeping, not transcripts. Sorted for a stable
|
|
288
|
+
* fold order. On real data the workflow layer held 100 of 165 forks and shipped none.
|
|
289
|
+
*/
|
|
290
|
+
function subagentFiles(subagentsDir) {
|
|
291
|
+
const out = [];
|
|
292
|
+
let names = [];
|
|
293
|
+
try { names = fs.readdirSync(subagentsDir); } catch { return out; }
|
|
294
|
+
for (const f of names.sort()) if (f.startsWith('agent-') && f.endsWith('.jsonl')) out.push({ file: path.join(subagentsDir, f), workflow: null });
|
|
295
|
+
const wfDir = path.join(subagentsDir, 'workflows');
|
|
296
|
+
let wfs = [];
|
|
297
|
+
try { wfs = fs.readdirSync(wfDir).filter((d) => d.startsWith('wf_')).sort(); } catch { return out; }
|
|
298
|
+
for (const wf of wfs) {
|
|
299
|
+
let files = [];
|
|
300
|
+
try { files = fs.readdirSync(path.join(wfDir, wf)); } catch { continue; }
|
|
301
|
+
for (const f of files.sort()) if (f.startsWith('agent-') && f.endsWith('.jsonl')) out.push({ file: path.join(wfDir, wf, f), workflow: wf });
|
|
302
|
+
}
|
|
303
|
+
return out;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* What the parent called each subagent. The parent's `Agent` tool_use carries the
|
|
308
|
+
* description and subagent_type; the tool_result that answers it names the agentId
|
|
309
|
+
* ("agentId: <id>"). Join the two through tool_use_id → a map agentId → {description, type}.
|
|
310
|
+
* Best effort: a parent that predates the ids, or a truncated line, just yields no name.
|
|
311
|
+
*/
|
|
312
|
+
function subagentNames(parentFile) {
|
|
313
|
+
const byToolUse = new Map(); const out = new Map();
|
|
314
|
+
let lines = [];
|
|
315
|
+
try { lines = fs.readFileSync(parentFile, 'utf-8').split('\n').filter(Boolean); } catch { return out; }
|
|
316
|
+
for (const line of lines) {
|
|
317
|
+
let obj; try { obj = JSON.parse(line); } catch { continue; }
|
|
318
|
+
const content = obj.message && Array.isArray(obj.message.content) ? obj.message.content : [];
|
|
319
|
+
for (const c of content) {
|
|
320
|
+
if (c && c.type === 'tool_use' && c.name === 'Agent' && c.id) {
|
|
321
|
+
byToolUse.set(c.id, { description: String((c.input && c.input.description) || ''), type: String((c.input && c.input.subagent_type) || '') });
|
|
322
|
+
} else if (c && c.type === 'tool_result' && c.tool_use_id && byToolUse.has(c.tool_use_id)) {
|
|
323
|
+
const text = typeof c.content === 'string' ? c.content : (Array.isArray(c.content) ? c.content.map((x) => (x && x.text) || '').join('\n') : '');
|
|
324
|
+
const m = /agentId:\s*([0-9a-f]{6,})/i.exec(text);
|
|
325
|
+
if (m) out.set(m[1], byToolUse.get(c.tool_use_id));
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// A background subagent reports through a task notification instead — a user-role
|
|
329
|
+
// line whose text carries <task-id>agentId</task-id> and <tool-use-id>toolu_…</tool-use-id>.
|
|
330
|
+
// On real data 30 of 65 forks were named only this way.
|
|
331
|
+
const flat = typeof (obj.message && obj.message.content) === 'string' ? obj.message.content
|
|
332
|
+
: content.map((x) => (x && typeof x.text === 'string') ? x.text : '').join('\n');
|
|
333
|
+
if (flat.includes('<task-notification>')) {
|
|
334
|
+
for (const block of flat.split('<task-notification>').slice(1)) {
|
|
335
|
+
const t = /<task-id>([0-9a-f]{6,})<\/task-id>/i.exec(block); const u = /<tool-use-id>(toolu_[A-Za-z0-9]+)<\/tool-use-id>/.exec(block);
|
|
336
|
+
if (t && u && byToolUse.has(u[1]) && !out.has(t[1])) out.set(t[1], byToolUse.get(u[1]));
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return out;
|
|
341
|
+
}
|
|
342
|
+
|
|
263
343
|
function extractContent(content) {
|
|
264
344
|
if (typeof content === 'string') return content;
|
|
265
345
|
if (!Array.isArray(content)) return '';
|
|
@@ -420,4 +500,4 @@ function getMCPServers() {
|
|
|
420
500
|
// answer to a different question: where this pilot's Claude Code instances live. Two
|
|
421
501
|
// implementations of "find every CLAUDE_CONFIG_DIR" would drift the first time a playbook
|
|
422
502
|
// layout changes, and this is the copy exercised on every single ship.
|
|
423
|
-
module.exports = { name, labels, getChats, getMessages, getUsage, getArtifacts, getMCPServers, discoverClaudeRoots };
|
|
503
|
+
module.exports = { name, labels, getChats, getMessages, getUsage, getArtifacts, getMCPServers, discoverClaudeRoots, subagentNames, subagentFiles };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mem",
|
|
3
3
|
"description": "Conversation memory for AI agents over the memhouse typed ClickHouse store: recall past sessions and answer from them, run analytics, check the house, bring people in, and administer the server. Skills install as /mem:recall, /mem:sql, /mem:house, /mem:access, and /mem:admin.",
|
|
4
|
-
"version": "0.18.
|
|
4
|
+
"version": "0.18.3",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Ramazan Polat"
|
|
7
7
|
},
|
|
@@ -92,6 +92,21 @@ wrong rooms. Say whose memory an answer came from.
|
|
|
92
92
|
|
|
93
93
|
## Schema
|
|
94
94
|
|
|
95
|
+
**Subagents (forks) live inside their parent session.** Claude Code writes them to
|
|
96
|
+
`<session>/subagents/agent-<id>.jsonl` (Agent tool) and
|
|
97
|
+
`<session>/subagents/workflows/wf_<id>/agent-<id>.jsonl` (Workflow runs); the shipper folds them into the parent with
|
|
98
|
+
`is_subagent = 1` and stamps *which* one in `extra.agent`. Looking for a fork as its own
|
|
99
|
+
session finds nothing — look inside the parent:
|
|
100
|
+
|
|
101
|
+
```sql
|
|
102
|
+
-- the subagents a session spawned
|
|
103
|
+
SELECT toString(extra.agent.id) AS agent, any(toString(extra.agent.description)) AS asked_to, count() AS turns
|
|
104
|
+
FROM messages WHERE session_id = '<sid>' AND is_subagent GROUP BY agent -- toString: a JSON path cannot be a GROUP BY key
|
|
105
|
+
-- one subagent's transcript, in order
|
|
106
|
+
SELECT seq, role, substring(text, 1, 2000) FROM messages
|
|
107
|
+
WHERE session_id = '<sid>' AND extra.agent.id = '<agent>' ORDER BY seq
|
|
108
|
+
```
|
|
109
|
+
|
|
95
110
|
**`messages`** — one row per turn.
|
|
96
111
|
|
|
97
112
|
| column | meaning |
|
|
@@ -102,6 +117,8 @@ wrong rooms. Say whose memory an answer came from.
|
|
|
102
117
|
| `host` | the machine, `<hostname>-<8 hex>` — random per install, not derived |
|
|
103
118
|
| `ts` | when the turn happened (`DateTime64(3,'UTC')`) |
|
|
104
119
|
| `role` | `user`, `assistant`, `system` |
|
|
120
|
+
| `is_subagent` | 1 for a turn that a Claude Code **subagent** (a fork) produced; folded into the parent session after the parent's own turns. Subagents are never separate sessions. |
|
|
121
|
+
| `extra` | JSON. For a subagent turn: `extra.agent.id`, `extra.agent.description` (what the parent asked it to do, else the fork's own first prompt), `extra.agent.type` (`general-purpose`, `workflow-subagent`, …), `extra.agent.turn` (position within that subagent), and `extra.agent.workflow` (`wf_…`) when a Workflow run spawned it. Empty `{}` otherwise, and on rows shipped before 0.18.3. |
|
|
105
122
|
| `model`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens` | usage; **empty for several adapters** — see below |
|
|
106
123
|
| `text` | the turn, **truncated at 50,000 chars** |
|
|
107
124
|
| `text_ngram`, `text_word` | `MATERIALIZED lower(text)`, carrying the text indexes |
|
|
@@ -112,6 +112,11 @@ Beyond the three in the reference:
|
|
|
112
112
|
|
|
113
113
|
- **Ranking is yours.** There is no relevance score in the store. `hits` plus recency is
|
|
114
114
|
the whole of it — say so rather than implying the top row is "the best match".
|
|
115
|
+
- **A subagent (fork) is not a session.** Its turns are inside the parent session with
|
|
116
|
+
`is_subagent = 1` and `extra.agent.id` / `extra.agent.description` saying which one; the
|
|
117
|
+
reference has the two queries. "Find the fork that did X" → search `messages` as usual,
|
|
118
|
+
then group the hits by `toString(extra.agent.id)` — never conclude it was not shipped because no
|
|
119
|
+
session carries its name.
|
|
115
120
|
- **Tool *results* are not stored.** "What did that command print" is unanswerable from
|
|
116
121
|
the house; only the invocation and its arguments survive.
|
|
117
122
|
- **`memhouse search`** exists as a CLI, but it joins your terms into ONE literal phrase
|
package/memhouse/house/house.js
CHANGED
|
@@ -306,6 +306,117 @@ function sessionsRollup({ sessions, messages }) {
|
|
|
306
306
|
)`;
|
|
307
307
|
}
|
|
308
308
|
|
|
309
|
+
// The precomputed rollups' table names. One place, because the shipper that fills them,
|
|
310
|
+
// the runtime probe and the read layer must all agree.
|
|
311
|
+
const SESSION_STATS = 'session_stats';
|
|
312
|
+
const MODEL_STATS = 'session_model_stats';
|
|
313
|
+
const TOOL_STATS = 'session_tool_stats';
|
|
314
|
+
// One-layout: like every room, a stat table is named for its member — `<member>_session_stats`
|
|
315
|
+
// — so it sits under the same `<db>.<member>_*` grant the member already holds. No
|
|
316
|
+
// database-level privilege exists in this layout, and none is needed.
|
|
317
|
+
function statTableNames(member) {
|
|
318
|
+
return {
|
|
319
|
+
session_stats: physicalRoom(SESSION_STATS, member),
|
|
320
|
+
model_stats: physicalRoom(MODEL_STATS, member),
|
|
321
|
+
tool_stats: physicalRoom(TOOL_STATS, member),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* The precomputed rollups — the tables the dashboard reads instead of rebuilding the
|
|
327
|
+
* session rollup on every query. Measured on a 714k-message house: 1,413,624 rows read
|
|
328
|
+
* per query became ~1,250, and a page load went from 4–10 seconds to ~250 ms.
|
|
329
|
+
*
|
|
330
|
+
* REFRESHED BY THE SHIPPER, NOT BY A MATERIALIZED VIEW — and that is forced, not chosen.
|
|
331
|
+
* The obvious tool is a refreshable materialized view (`REFRESH EVERY 5 MINUTE TO t`),
|
|
332
|
+
* which is exactly what the 0.17 line uses. It cannot exist in this layout: a refreshable
|
|
333
|
+
* view swaps its target atomically through a temporary table with a generated name, so
|
|
334
|
+
* ClickHouse demands `SELECT, INSERT, CREATE TABLE, DROP TABLE ON <db>.*` — the whole
|
|
335
|
+
* database — and a one-layout member holds `<db>.<member>_*` and nothing else, on
|
|
336
|
+
* purpose. Measured on 25.8: the CREATE is refused with exactly that message, and the
|
|
337
|
+
* grant that would satisfy it is the one this layout exists to withhold.
|
|
338
|
+
*
|
|
339
|
+
* (An ordinary insert-trigger MV is wrong for a different reason: it counts every
|
|
340
|
+
* ReplacingMergeTree version the merge has not collapsed yet, cannot see the epoch
|
|
341
|
+
* filter, and cannot join.)
|
|
342
|
+
*
|
|
343
|
+
* So the shipper runs the rollup itself, once at the end of every pass — the same
|
|
344
|
+
* cadence a view would have had — and INSERTs the result stamped with one
|
|
345
|
+
* `refreshed_at`. Every insert is a complete snapshot, a GENERATION; the read layer
|
|
346
|
+
* takes only the newest generation. Nothing is deleted, nothing is swapped, nothing
|
|
347
|
+
* needs a privilege beyond INSERT and SELECT on the member's own rooms. A TTL of one day
|
|
348
|
+
* keeps old generations from accumulating (288 a day at the default cadence), and the
|
|
349
|
+
* sorting key leads with `refreshed_at` so reading the newest one is one index range,
|
|
350
|
+
* not a scan.
|
|
351
|
+
*
|
|
352
|
+
* Built from the SAME rollup text the read layer would otherwise run, so the two
|
|
353
|
+
* cannot drift; column types are inferred with EMPTY AS for the same reason.
|
|
354
|
+
*/
|
|
355
|
+
const STAT_TTL = 'TTL toDateTime(refreshed_at) + INTERVAL 1 DAY';
|
|
356
|
+
|
|
357
|
+
function statBodies(rooms) {
|
|
358
|
+
// sessionsRollup wraps itself in parentheses for the FROM position; strip them.
|
|
359
|
+
const rollup = sessionsRollup(rooms).replace(/^\s*\(/, '').replace(/\)\s*$/, '');
|
|
360
|
+
return {
|
|
361
|
+
session_stats: {
|
|
362
|
+
key: '(refreshed_at, session_id, user_id)',
|
|
363
|
+
body: rollup,
|
|
364
|
+
},
|
|
365
|
+
// One row per (session, user, model) with the message count and the four token sums.
|
|
366
|
+
// Every model/cost aggregate the dashboard runs is a sum or argMax over these rows —
|
|
367
|
+
// a few thousand — instead of a scan of the full messages room. The read layer's
|
|
368
|
+
// any-token predicates survive unchanged: a zero-token row contributes zero to every
|
|
369
|
+
// sum, so filtering before summing and summing over everything agree.
|
|
370
|
+
model_stats: {
|
|
371
|
+
key: '(refreshed_at, session_id, user_id, model)',
|
|
372
|
+
body: `SELECT session_id, user_id, any(source) AS source, any(folder) AS folder, model,
|
|
373
|
+
count() AS msgs,
|
|
374
|
+
sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens,
|
|
375
|
+
sum(cache_read_tokens) AS cache_read_tokens, sum(cache_write_tokens) AS cache_write_tokens
|
|
376
|
+
FROM ${rooms.messages} AS m
|
|
377
|
+
GROUP BY session_id, user_id, model`,
|
|
378
|
+
},
|
|
379
|
+
// One row per (session, user, tool_name) with the call count. NOTE the epoch source:
|
|
380
|
+
// rooms.tool_calls is currentParse(tool_calls, 'messages') — the tool room takes its
|
|
381
|
+
// epoch from MESSAGES, because a parse that keeps its messages but emits no tool calls
|
|
382
|
+
// writes nothing at the new epoch, and this room's own max(epoch) would resurrect the
|
|
383
|
+
// superseded parse's calls.
|
|
384
|
+
tool_stats: {
|
|
385
|
+
key: '(refreshed_at, session_id, user_id, tool_name)',
|
|
386
|
+
body: `SELECT session_id, user_id, any(source) AS source, any(folder) AS folder, tool_name,
|
|
387
|
+
count() AS calls
|
|
388
|
+
FROM ${rooms.tool_calls} AS tc
|
|
389
|
+
GROUP BY session_id, user_id, tool_name`,
|
|
390
|
+
},
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** The CREATE statements — additive, idempotent, run by ensureSchema. */
|
|
395
|
+
function statCreateStatements(rooms) {
|
|
396
|
+
const names = statTableNames(rooms.member);
|
|
397
|
+
const bodies = statBodies(rooms);
|
|
398
|
+
return Object.keys(bodies).map((k) =>
|
|
399
|
+
`CREATE TABLE IF NOT EXISTS ${names[k]} ENGINE = MergeTree ORDER BY ${bodies[k].key} ${STAT_TTL} `
|
|
400
|
+
+ `EMPTY AS SELECT now64(3) AS refreshed_at, * FROM (${bodies[k].body})`);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* The refresh — one INSERT per table, one generation per call. All three carry the same
|
|
405
|
+
* `refreshed_at` so a reader never mixes a session_stats generation with a model_stats
|
|
406
|
+
* one. Run by the shipper after every pass and by --ensure-schema.
|
|
407
|
+
*/
|
|
408
|
+
function statRefreshStatements(rooms, at = 'now64(3)') {
|
|
409
|
+
const names = statTableNames(rooms.member);
|
|
410
|
+
const bodies = statBodies(rooms);
|
|
411
|
+
return Object.keys(bodies).map((k) =>
|
|
412
|
+
`INSERT INTO ${names[k]} SELECT ${at} AS refreshed_at, * FROM (${bodies[k].body})`);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** The read-side source for one stat table: its newest generation, and nothing else. */
|
|
416
|
+
function latestGeneration(table) {
|
|
417
|
+
return `(SELECT * FROM ${table} WHERE refreshed_at = (SELECT max(refreshed_at) FROM ${table}))`;
|
|
418
|
+
}
|
|
419
|
+
|
|
309
420
|
/**
|
|
310
421
|
* The settings every read needs. `final` collapses ReplacingMergeTree versions;
|
|
311
422
|
* `join_use_nulls` is what the rollup's coalesce depends on.
|
|
@@ -381,8 +492,67 @@ async function currentUser(client) {
|
|
|
381
492
|
}
|
|
382
493
|
|
|
383
494
|
/** Resolve the rooms for whoever this client is connected as. */
|
|
495
|
+
/**
|
|
496
|
+
* The precomputed rollup, when the house has one.
|
|
497
|
+
*
|
|
498
|
+
* `sessions_v` is a SAVED QUERY that rebuilds the whole rollup — sessions FINAL joined
|
|
499
|
+
* to the current parse of messages, grouped over the entire house — on every call. A
|
|
500
|
+
* house that has run the session_stats migration carries the same rows as a flat table,
|
|
501
|
+
* refreshed on a timer, and the read layer can name that instead. Measured on a 714k
|
|
502
|
+
* message house: 1,413,624 rows read per query becomes 1,230, and p50 query time 68ms
|
|
503
|
+
* becomes 4ms.
|
|
504
|
+
*
|
|
505
|
+
* Resolved at RUNTIME rather than by version, and that is the point: one binary has to
|
|
506
|
+
* serve a migrated house, a house whose migration has not run, and a friend's house
|
|
507
|
+
* shared read-only that was never migrated at all. Asking the server what exists is the
|
|
508
|
+
* only answer that is correct in all three.
|
|
509
|
+
*/
|
|
510
|
+
async function statTables(client, member) {
|
|
511
|
+
const names = statTableNames(member);
|
|
512
|
+
const wanted = Object.values(names);
|
|
513
|
+
try {
|
|
514
|
+
const rs = await client.query({
|
|
515
|
+
query: `SELECT name FROM system.tables
|
|
516
|
+
WHERE database = currentDatabase()
|
|
517
|
+
AND name IN (${wanted.map((n) => `'${n}'`).join(', ')})`,
|
|
518
|
+
format: 'JSONEachRow',
|
|
519
|
+
});
|
|
520
|
+
const present = (await rs.json()).map((r) => r.name);
|
|
521
|
+
// Present is not enough: a table exists from the moment ensureSchema creates it and
|
|
522
|
+
// holds nothing until the first refresh lands. A dashboard reading it then would show
|
|
523
|
+
// a house with no sessions at all (caught by house-test the first time). An empty
|
|
524
|
+
// stat table reads as absent, so the inline rollup answers until the table can.
|
|
525
|
+
const usable = new Set();
|
|
526
|
+
for (const name of present) {
|
|
527
|
+
try {
|
|
528
|
+
const c = await client.query({ query: `SELECT count() AS n FROM ${name}`, format: 'JSONEachRow' });
|
|
529
|
+
const rows = await c.json();
|
|
530
|
+
if (Number(rows[0] && rows[0].n) > 0) usable.add(name);
|
|
531
|
+
} catch { /* unreadable: treat as absent */ }
|
|
532
|
+
}
|
|
533
|
+
return usable;
|
|
534
|
+
} catch {
|
|
535
|
+
// No grant on system.tables is not an error — it means "assume none", and the
|
|
536
|
+
// subquery path is always correct.
|
|
537
|
+
return new Set();
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
async function hasSessionStats(client, member) {
|
|
542
|
+
return (await statTables(client, member)).has(statTableNames(member).session_stats);
|
|
543
|
+
}
|
|
544
|
+
|
|
384
545
|
async function resolveRooms(client) {
|
|
385
|
-
|
|
546
|
+
const rooms = roomNames(await currentUser(client));
|
|
547
|
+
const names = statTableNames(rooms.member);
|
|
548
|
+
const have = await statTables(client, rooms.member);
|
|
549
|
+
if (have.has(names.session_stats)) rooms.sessions_v = latestGeneration(names.session_stats);
|
|
550
|
+
// Null when absent: the read layer branches to the legacy scan-the-room SQL. Never a
|
|
551
|
+
// token substitution, because the legacy SQL has a different shape, not just a
|
|
552
|
+
// different table name.
|
|
553
|
+
rooms.model_stats = have.has(names.model_stats) ? latestGeneration(names.model_stats) : null;
|
|
554
|
+
rooms.tool_stats = have.has(names.tool_stats) ? latestGeneration(names.tool_stats) : null;
|
|
555
|
+
return rooms;
|
|
386
556
|
}
|
|
387
557
|
|
|
388
558
|
/**
|
|
@@ -426,4 +596,6 @@ module.exports = {
|
|
|
426
596
|
READ_SETTINGS, MEMBER_PIN,
|
|
427
597
|
installCommand, assertUsableName,
|
|
428
598
|
sessionsRollup, currentParse, createStatement, roomNames, currentUser, resolveRooms,
|
|
599
|
+
SESSION_STATS, MODEL_STATS, TOOL_STATS, statTableNames, hasSessionStats, statTables,
|
|
600
|
+
statCreateStatements, statRefreshStatements, latestGeneration,
|
|
429
601
|
};
|
|
@@ -140,6 +140,28 @@ function inSessions(f, col) {
|
|
|
140
140
|
return ` AND (${alias}.session_id, ${alias}.user_id) IN (SELECT session_id, user_id FROM {{sessions_v}} AS c WHERE 1=1${f.and})`;
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Narrow a query to an explicit set of (session_id, user_id) pairs.
|
|
145
|
+
*
|
|
146
|
+
* Distinct from inSessions(), which restricts to whatever the USER filtered to. This
|
|
147
|
+
* restricts to the rows a caller is actually going to use — the fifty sessions on the
|
|
148
|
+
* page, not the twelve hundred in the house.
|
|
149
|
+
*
|
|
150
|
+
* arrayZip, not two INs: the rooms are keyed on the PAIR, and matching the columns
|
|
151
|
+
* independently would pull in another member's rows for a colliding adapter-local
|
|
152
|
+
* session_id. Same idiom getDeepAnalytics already uses.
|
|
153
|
+
*/
|
|
154
|
+
function scopeAnd(scope, alias) {
|
|
155
|
+
if (!scope || !scope.ids || !scope.ids.length) return '';
|
|
156
|
+
const a = alias ? `${alias}.` : '';
|
|
157
|
+
return ` AND (${a}session_id, ${a}user_id) IN arrayZip({scopeIds:Array(String)}, {scopeUsers:Array(String)})`;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function scopeParams(f, scope) {
|
|
161
|
+
if (!scope || !scope.ids || !scope.ids.length) return f.params;
|
|
162
|
+
return { ...f.params, scopeIds: scope.ids, scopeUsers: scope.users };
|
|
163
|
+
}
|
|
164
|
+
|
|
143
165
|
// Fold a [{name|model, cnt}] list through normalizeModelName into a freq map.
|
|
144
166
|
function normalizedModelFreq(rows) {
|
|
145
167
|
const freq = {};
|
|
@@ -151,37 +173,52 @@ function normalizedModelFreq(rows) {
|
|
|
151
173
|
}
|
|
152
174
|
// The same exclusions as EXCLUDED_MODELS, usable in JS.
|
|
153
175
|
const EXCLUDED_MODEL_SET = new Set(['', '<synthetic>']);
|
|
154
|
-
|
|
176
|
+
// Tiebreak on the name. A top-N cut over counts alone is not a total order: two tools
|
|
177
|
+
// tied at the cut line resolve by arrival order, so the same house can show a different
|
|
178
|
+
// tenth tool on consecutive loads. Same rule as getChats' ORDER BY tiebreaker.
|
|
179
|
+
const topN = (freq, n) => Object.entries(freq).sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)).slice(0, n)
|
|
155
180
|
.map(([name, count]) => ({ name, count }));
|
|
156
181
|
|
|
157
182
|
// ── overview ────────────────────────────────────────────────────────────────────
|
|
158
183
|
async function getOverview(opts = {}) {
|
|
159
184
|
const f = filters(opts);
|
|
160
|
-
//
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
185
|
+
// Six independent reads, issued together. They share nothing but the filter, and every
|
|
186
|
+
// one of them is a full round trip: awaited in sequence they cost 6 x RTT before any
|
|
187
|
+
// result exists. Measured against a ClickHouse 6,000 km away, that sequencing — not the
|
|
188
|
+
// database — was roughly two thirds of what the browser waited for.
|
|
189
|
+
//
|
|
190
|
+
// The comments on each query below are the reasons its SHAPE is what it is; none of
|
|
191
|
+
// them is a reason for it to wait on the one above.
|
|
192
|
+
const [totalRow, editors, modes, rows, projects, range] = await Promise.all([
|
|
193
|
+
// Same definition as getChats(). The nav bar read this and the Sessions page read
|
|
194
|
+
// getChats, so a house with one placeholder session showed "9 sessions" beside
|
|
195
|
+
// "8 sessions" in the same header, and the depth histogram summed to 7. A session row
|
|
196
|
+
// with no name and no messages is a placeholder the shipper keeps to absorb the
|
|
197
|
+
// incremental skip; it is not a conversation, and only one of the two counters knew.
|
|
198
|
+
q1(`SELECT count() AS cnt FROM {{sessions_v}} AS c WHERE 1=1${f.and} AND (c.name != '' OR c.total_msgs > 0)`,
|
|
199
|
+
f.params),
|
|
200
|
+
// Root parity WAS: without a folder filter the editor breakdown is global — which put
|
|
201
|
+
// "34 + 3" chips under a header reading "3 sessions" when an editor filter was applied,
|
|
202
|
+
// and 12 under a header of 11 with no filter at all. Parity with a wrong number is not a
|
|
203
|
+
// feature.
|
|
204
|
+
q(`SELECT source, count() AS count FROM {{sessions_v}} AS c
|
|
205
|
+
WHERE (c.name != '' OR c.total_msgs > 0)${f.and} GROUP BY source ORDER BY count DESC`, f.params),
|
|
206
|
+
q(`SELECT mode, count() AS count FROM {{sessions_v}} AS c WHERE mode != ''${f.and} GROUP BY mode`, f.params),
|
|
207
|
+
q(`
|
|
208
|
+
SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
|
|
209
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and}
|
|
210
|
+
GROUP BY month, source ORDER BY month`, f.params),
|
|
211
|
+
q(`
|
|
212
|
+
SELECT folder, count() AS count FROM {{sessions_v}} AS c
|
|
213
|
+
WHERE folder != ''${f.and} GROUP BY folder ORDER BY count DESC LIMIT 20`, f.params),
|
|
214
|
+
q1(`SELECT min(${MS}) AS oldest, max(${MS}) AS newest FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params),
|
|
215
|
+
]);
|
|
216
|
+
|
|
217
|
+
const totalChats = Number(totalRow.cnt);
|
|
218
|
+
|
|
178
219
|
const byMode = {};
|
|
179
220
|
for (const m of modes) byMode[m.mode] = Number(m.count);
|
|
180
221
|
|
|
181
|
-
const rows = await q(`
|
|
182
|
-
SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
|
|
183
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and}
|
|
184
|
-
GROUP BY month, source ORDER BY month`, f.params);
|
|
185
222
|
const monthMap = {};
|
|
186
223
|
for (const r of rows) {
|
|
187
224
|
if (!monthMap[r.month]) monthMap[r.month] = { count: 0, editors: {} };
|
|
@@ -190,17 +227,12 @@ async function getOverview(opts = {}) {
|
|
|
190
227
|
}
|
|
191
228
|
const byMonth = Object.keys(monthMap).sort().map(m => ({ month: m, ...monthMap[m] }));
|
|
192
229
|
|
|
193
|
-
const projects = await q(`
|
|
194
|
-
SELECT folder, count() AS count FROM {{sessions_v}} AS c
|
|
195
|
-
WHERE folder != ''${f.and} GROUP BY folder ORDER BY count DESC LIMIT 20`, f.params);
|
|
196
230
|
const topProjects = projects.map(p => ({
|
|
197
231
|
name: p.folder.split(/[/\\]/).slice(-2).join('/'),
|
|
198
232
|
fullPath: p.folder,
|
|
199
233
|
count: Number(p.count),
|
|
200
234
|
}));
|
|
201
235
|
|
|
202
|
-
const range = await q1(`SELECT min(${MS}) AS oldest, max(${MS}) AS newest FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params);
|
|
203
|
-
|
|
204
236
|
return {
|
|
205
237
|
totalChats,
|
|
206
238
|
editors: editors.map(e => ({ id: e.source, count: Number(e.count) })),
|
|
@@ -238,21 +270,62 @@ async function getDailyActivity(opts = {}) {
|
|
|
238
270
|
async function getDashboardStats(opts = {}) {
|
|
239
271
|
const f = filters(opts);
|
|
240
272
|
const p = { ...f.params, tz: TZ };
|
|
273
|
+
const r = await rooms();
|
|
241
274
|
|
|
242
|
-
|
|
275
|
+
// Nine independent reads, one round trip. Sequential, this endpoint paid 9 x RTT
|
|
276
|
+
// before any result existed — the largest fan-out on the dashboard. Each query's
|
|
277
|
+
// shape is unchanged; only the waiting is gone.
|
|
278
|
+
const [hourlyRows, weekdayRows, depthRows, tokenRow, streakRows,
|
|
279
|
+
monthEditorRows, velocityRows, modelRows, toolRows] = await Promise.all([
|
|
280
|
+
q(`
|
|
243
281
|
SELECT toHour(${TS}, {tz:String}) AS hour, count() AS count
|
|
244
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY hour ORDER BY hour`, p)
|
|
282
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY hour ORDER BY hour`, p),
|
|
283
|
+
q(`
|
|
284
|
+
SELECT toDayOfWeek(${TS}, 0, {tz:String}) % 7 AS dow, count() AS count
|
|
285
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY dow ORDER BY dow`, p),
|
|
286
|
+
q(`
|
|
287
|
+
SELECT total_msgs AS msgs FROM {{sessions_v}} AS c WHERE total_msgs > 0${f.and}`, f.params),
|
|
288
|
+
q1(`
|
|
289
|
+
SELECT COALESCE(sum(input_tokens), 0) AS input, COALESCE(sum(output_tokens), 0) AS output,
|
|
290
|
+
COALESCE(sum(cache_read_tokens), 0) AS cacheRead, COALESCE(sum(cache_write_tokens), 0) AS cacheWrite,
|
|
291
|
+
COALESCE(sum(user_chars), 0) AS userChars, COALESCE(sum(assistant_chars), 0) AS assistantChars,
|
|
292
|
+
COALESCE(sum(total_msgs), 0) AS messages,
|
|
293
|
+
-- The SAME definition the KPI beside it uses. A plain count() here put a
|
|
294
|
+
-- fourth session number on a screen that already had three: nav 11, chips 12,
|
|
295
|
+
-- avg depth 40/12, histogram 10.
|
|
296
|
+
countIf(c.name != '' OR c.total_msgs > 0) AS sessions
|
|
297
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params),
|
|
298
|
+
q(`
|
|
299
|
+
SELECT DISTINCT formatDateTime(${TS}, '%Y-%m-%d', {tz:String}) AS day
|
|
300
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY day`, p),
|
|
301
|
+
q(`
|
|
302
|
+
SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
|
|
303
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY month, source ORDER BY month`, f.params),
|
|
304
|
+
q(`
|
|
305
|
+
SELECT formatDateTime(c.last_updated_at, '%Y-%m', 'UTC') AS month,
|
|
306
|
+
avg(total_msgs) AS avgMsgs, avg(input_tokens + output_tokens) AS avgTokens
|
|
307
|
+
FROM {{sessions_v}} AS c WHERE c.last_updated_at IS NOT NULL${f.and}
|
|
308
|
+
GROUP BY month ORDER BY month`, f.params),
|
|
309
|
+
q(r.model_stats ? `
|
|
310
|
+
SELECT m.model AS model, sum(m.msgs) AS cnt FROM {{model_stats}} AS m
|
|
311
|
+
WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
312
|
+
GROUP BY model` : `
|
|
313
|
+
SELECT m.model AS model, count() AS cnt FROM {{messages}} AS m
|
|
314
|
+
WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
315
|
+
GROUP BY model`, f.params),
|
|
316
|
+
q(r.tool_stats ? `
|
|
317
|
+
SELECT tool_name, sum(tc.calls) AS cnt FROM {{tool_stats}} AS tc
|
|
318
|
+
WHERE 1=1${inSessions(f, 'tc.session_id')} GROUP BY tool_name` : `
|
|
319
|
+
SELECT tool_name, count() AS cnt FROM {{tool_calls}} AS tc
|
|
320
|
+
WHERE 1=1${inSessions(f, 'tc.session_id')} GROUP BY tool_name`, f.params),
|
|
321
|
+
]);
|
|
322
|
+
|
|
245
323
|
const hourly = new Array(24).fill(0);
|
|
246
324
|
for (const r of hourlyRows) hourly[Number(r.hour)] = Number(r.count);
|
|
247
325
|
|
|
248
|
-
const weekdayRows = await q(`
|
|
249
|
-
SELECT toDayOfWeek(${TS}, 0, {tz:String}) % 7 AS dow, count() AS count
|
|
250
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY dow ORDER BY dow`, p);
|
|
251
326
|
const weekdays = new Array(7).fill(0);
|
|
252
327
|
for (const r of weekdayRows) weekdays[Number(r.dow)] = Number(r.count);
|
|
253
328
|
|
|
254
|
-
const depthRows = await q(`
|
|
255
|
-
SELECT total_msgs AS msgs FROM {{sessions_v}} AS c WHERE total_msgs > 0${f.and}`, f.params);
|
|
256
329
|
const depthBuckets = { '1': 0, '2-5': 0, '6-10': 0, '11-20': 0, '21-50': 0, '51-100': 0, '100+': 0 };
|
|
257
330
|
for (const r of depthRows) {
|
|
258
331
|
const m = Number(r.msgs);
|
|
@@ -265,20 +338,6 @@ async function getDashboardStats(opts = {}) {
|
|
|
265
338
|
else depthBuckets['100+']++;
|
|
266
339
|
}
|
|
267
340
|
|
|
268
|
-
const tokenRow = await q1(`
|
|
269
|
-
SELECT COALESCE(sum(input_tokens), 0) AS input, COALESCE(sum(output_tokens), 0) AS output,
|
|
270
|
-
COALESCE(sum(cache_read_tokens), 0) AS cacheRead, COALESCE(sum(cache_write_tokens), 0) AS cacheWrite,
|
|
271
|
-
COALESCE(sum(user_chars), 0) AS userChars, COALESCE(sum(assistant_chars), 0) AS assistantChars,
|
|
272
|
-
COALESCE(sum(total_msgs), 0) AS messages,
|
|
273
|
-
-- The SAME definition the KPI beside it uses. A plain count() here put a
|
|
274
|
-
-- fourth session number on a screen that already had three: nav 11, chips 12,
|
|
275
|
-
-- avg depth 40/12, histogram 10.
|
|
276
|
-
countIf(c.name != '' OR c.total_msgs > 0) AS sessions
|
|
277
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params);
|
|
278
|
-
|
|
279
|
-
const streakRows = await q(`
|
|
280
|
-
SELECT DISTINCT formatDateTime(${TS}, '%Y-%m-%d', {tz:String}) AS day
|
|
281
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY day`, p);
|
|
282
341
|
let currentStreak = 0, longestStreak = 0, tempStreak = 1;
|
|
283
342
|
const today = new Date().toISOString().split('T')[0];
|
|
284
343
|
for (let i = 1; i < streakRows.length; i++) {
|
|
@@ -299,9 +358,6 @@ async function getDashboardStats(opts = {}) {
|
|
|
299
358
|
}
|
|
300
359
|
if (streakRows.length === 0) longestStreak = 0;
|
|
301
360
|
|
|
302
|
-
const monthEditorRows = await q(`
|
|
303
|
-
SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
|
|
304
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY month, source ORDER BY month`, f.params);
|
|
305
361
|
const monthEditors = {};
|
|
306
362
|
const allSources = new Set();
|
|
307
363
|
for (const r of monthEditorRows) {
|
|
@@ -310,21 +366,8 @@ async function getDashboardStats(opts = {}) {
|
|
|
310
366
|
allSources.add(r.source);
|
|
311
367
|
}
|
|
312
368
|
|
|
313
|
-
const velocityRows = await q(`
|
|
314
|
-
SELECT formatDateTime(c.last_updated_at, '%Y-%m', 'UTC') AS month,
|
|
315
|
-
avg(total_msgs) AS avgMsgs, avg(input_tokens + output_tokens) AS avgTokens
|
|
316
|
-
FROM {{sessions_v}} AS c WHERE c.last_updated_at IS NOT NULL${f.and}
|
|
317
|
-
GROUP BY month ORDER BY month`, f.params);
|
|
318
|
-
|
|
319
|
-
const modelRows = await q(`
|
|
320
|
-
SELECT m.model AS model, count() AS cnt FROM {{messages}} AS m
|
|
321
|
-
WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
322
|
-
GROUP BY model`, f.params);
|
|
323
369
|
const topModels = topN(normalizedModelFreq(modelRows), 10);
|
|
324
370
|
|
|
325
|
-
const toolRows = await q(`
|
|
326
|
-
SELECT tool_name, count() AS cnt FROM {{tool_calls}} AS tc
|
|
327
|
-
WHERE 1=1${inSessions(f, 'tc.session_id')} GROUP BY tool_name`, f.params);
|
|
328
371
|
let totalToolCalls = 0;
|
|
329
372
|
const toolFreq = {};
|
|
330
373
|
for (const r of toolRows) { toolFreq[r.tool_name] = Number(r.cnt); totalToolCalls += Number(r.cnt); }
|
|
@@ -373,7 +416,13 @@ async function getChats(opts = {}) {
|
|
|
373
416
|
FROM {{sessions_v}} AS c WHERE 1=1${f.and}`;
|
|
374
417
|
const params = { ...f.params };
|
|
375
418
|
if (opts.named !== false) sql += " AND (c.name != '' OR c.total_msgs > 0)";
|
|
376
|
-
|
|
419
|
+
// Tiebreak on the session key. `ORDER BY <ts> DESC` alone is not a total order —
|
|
420
|
+
// sessions that stopped in the same millisecond tie, and ClickHouse is free to return
|
|
421
|
+
// ties in any order, INCLUDING A DIFFERENT ONE PER QUERY. With LIMIT/OFFSET on top,
|
|
422
|
+
// that means page 2 can repeat or skip a row that page 1 already showed. Found by
|
|
423
|
+
// diffing this endpoint's response between two equivalent sources: identical rows,
|
|
424
|
+
// different order, 129 distinct timestamps across 300 sessions.
|
|
425
|
+
sql += ` ORDER BY ${MS} DESC, c.session_id, c.user_id`;
|
|
377
426
|
if (opts.limit) { sql += ' LIMIT {limit:UInt64}'; params.limit = opts.limit; }
|
|
378
427
|
if (opts.offset) { sql += ' OFFSET {offset:UInt64}'; params.offset = opts.offset; }
|
|
379
428
|
const rows = await q(sql, params);
|
|
@@ -388,7 +437,13 @@ async function getChats(opts = {}) {
|
|
|
388
437
|
// and left the next one. computePerChatCosts already handles all of them — the orphan
|
|
389
438
|
// bucket, the source-dominant and global-dominant fallbacks, the char estimate — so call
|
|
390
439
|
// it instead of growing a second copy of the same logic.
|
|
391
|
-
|
|
440
|
+
// Price the fifty rows on the page, not the twelve hundred in the house. This query
|
|
441
|
+
// set was the most expensive thing the dashboard ran — 1,413,624 rows read and 272 MiB
|
|
442
|
+
// per call, three endpoints calling it — and getChats used ~4% of what it asked for.
|
|
443
|
+
const perChat = await computePerChatCosts(f, {
|
|
444
|
+
ids: rows.map((r) => String(r.id)),
|
|
445
|
+
users: rows.map((r) => String(r.user_id)),
|
|
446
|
+
});
|
|
392
447
|
const costBySession = {};
|
|
393
448
|
const topModelBySession = {};
|
|
394
449
|
for (const r of perChat) {
|
|
@@ -455,14 +510,15 @@ async function getChat(id) {
|
|
|
455
510
|
// Sessions are keyed (session_id, user_id): constrain the row reloads to the
|
|
456
511
|
// selected rollup's writer, or an owner/no-RLS reader viewing a house where two
|
|
457
512
|
// members share an adapter-local session_id would see their rows merged.
|
|
458
|
-
const messages = await
|
|
513
|
+
const [messages, toolCalls] = await Promise.all([
|
|
514
|
+
q(`
|
|
459
515
|
SELECT role, text AS content, model, input_tokens, output_tokens
|
|
460
516
|
FROM {{messages}} WHERE session_id = {cid:String} AND user_id = {uid:String} ORDER BY seq`,
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
const toolCalls = await q(`
|
|
517
|
+
{ cid: chat.id, uid: chat.user_id }),
|
|
518
|
+
q(`
|
|
464
519
|
SELECT tool_name, args FROM {{tool_calls}} WHERE session_id = {cid:String} AND user_id = {uid:String} ORDER BY idx`,
|
|
465
|
-
|
|
520
|
+
{ cid: chat.id, uid: chat.user_id }),
|
|
521
|
+
]);
|
|
466
522
|
const toolCallDetails = toolCalls.map(tc => ({ name: tc.tool_name, args: safeParseJson(tc.args) }));
|
|
467
523
|
|
|
468
524
|
let toolMessages = 0, systemMessages = 0;
|
|
@@ -498,15 +554,37 @@ async function getChat(id) {
|
|
|
498
554
|
|
|
499
555
|
// ── projects ────────────────────────────────────────────────────────────────────
|
|
500
556
|
async function getProjects(opts = {}) {
|
|
557
|
+
const r = await rooms();
|
|
501
558
|
const df = filters({
|
|
502
559
|
hiddenFolders: opts.includeHidden ? [] : opts.hiddenFolders,
|
|
503
560
|
dateFrom: opts.dateFrom, dateTo: opts.dateTo,
|
|
504
561
|
});
|
|
505
562
|
|
|
506
|
-
|
|
563
|
+
// Four independent reads, one round trip.
|
|
564
|
+
const [perSource, sums, modelRows, toolRows] = await Promise.all([
|
|
565
|
+
q(`
|
|
507
566
|
SELECT folder, source, count() AS count, min(${MS}) AS first_seen, max(${MS}) AS last_seen
|
|
508
567
|
FROM {{sessions_v}} AS c WHERE folder != ''${df.and}
|
|
509
|
-
GROUP BY folder, source ORDER BY folder, count DESC`, df.params)
|
|
568
|
+
GROUP BY folder, source ORDER BY folder, count DESC`, df.params),
|
|
569
|
+
q(`
|
|
570
|
+
SELECT folder, sum(total_msgs) AS totalMessages,
|
|
571
|
+
sum(input_tokens) AS totalInputTokens, sum(output_tokens) AS totalOutputTokens,
|
|
572
|
+
sum(user_chars) AS totalUserChars, sum(assistant_chars) AS totalAssistantChars,
|
|
573
|
+
sum(cache_read_tokens) AS totalCacheRead, sum(cache_write_tokens) AS totalCacheWrite
|
|
574
|
+
FROM {{sessions_v}} AS c WHERE folder != ''${df.and} GROUP BY folder`, df.params),
|
|
575
|
+
q(r.model_stats ? `
|
|
576
|
+
SELECT m.folder AS folder, m.model AS model, sum(m.msgs) AS cnt FROM {{model_stats}} AS m
|
|
577
|
+
WHERE m.folder != '' AND m.model NOT IN ${EXCLUDED_MODELS}${inSessions(df, 'm.session_id')}
|
|
578
|
+
GROUP BY folder, model` : `
|
|
579
|
+
SELECT m.folder AS folder, m.model AS model, count() AS cnt FROM {{messages}} AS m
|
|
580
|
+
WHERE m.folder != '' AND m.model NOT IN ${EXCLUDED_MODELS}${inSessions(df, 'm.session_id')}
|
|
581
|
+
GROUP BY folder, model`, df.params),
|
|
582
|
+
q(r.tool_stats ? `
|
|
583
|
+
SELECT tc.folder AS folder, tc.tool_name AS tool_name, sum(tc.calls) AS cnt FROM {{tool_stats}} AS tc
|
|
584
|
+
WHERE tc.folder != ''${inSessions(df, 'tc.session_id')} GROUP BY folder, tool_name` : `
|
|
585
|
+
SELECT tc.folder AS folder, tc.tool_name AS tool_name, count() AS cnt FROM {{tool_calls}} AS tc
|
|
586
|
+
WHERE tc.folder != ''${inSessions(df, 'tc.session_id')} GROUP BY folder, tool_name`, df.params),
|
|
587
|
+
]);
|
|
510
588
|
|
|
511
589
|
const map = {};
|
|
512
590
|
for (const r of perSource) {
|
|
@@ -517,19 +595,9 @@ async function getProjects(opts = {}) {
|
|
|
517
595
|
if (r.last_seen != null && r.last_seen > map[r.folder].lastSeen) map[r.folder].lastSeen = r.last_seen;
|
|
518
596
|
}
|
|
519
597
|
|
|
520
|
-
const sums = await q(`
|
|
521
|
-
SELECT folder, sum(total_msgs) AS totalMessages,
|
|
522
|
-
sum(input_tokens) AS totalInputTokens, sum(output_tokens) AS totalOutputTokens,
|
|
523
|
-
sum(user_chars) AS totalUserChars, sum(assistant_chars) AS totalAssistantChars,
|
|
524
|
-
sum(cache_read_tokens) AS totalCacheRead, sum(cache_write_tokens) AS totalCacheWrite
|
|
525
|
-
FROM {{sessions_v}} AS c WHERE folder != ''${df.and} GROUP BY folder`, df.params);
|
|
526
598
|
const sumByFolder = {};
|
|
527
599
|
for (const r of sums) sumByFolder[r.folder] = r;
|
|
528
600
|
|
|
529
|
-
const modelRows = await q(`
|
|
530
|
-
SELECT m.folder AS folder, m.model AS model, count() AS cnt FROM {{messages}} AS m
|
|
531
|
-
WHERE m.folder != '' AND m.model NOT IN ${EXCLUDED_MODELS}${inSessions(df, 'm.session_id')}
|
|
532
|
-
GROUP BY folder, model`, df.params);
|
|
533
601
|
const modelsByFolder = {};
|
|
534
602
|
for (const r of modelRows) {
|
|
535
603
|
const k = normalizeModelName(r.model) || r.model;
|
|
@@ -537,9 +605,6 @@ async function getProjects(opts = {}) {
|
|
|
537
605
|
modelsByFolder[r.folder][k] = (modelsByFolder[r.folder][k] || 0) + Number(r.cnt);
|
|
538
606
|
}
|
|
539
607
|
|
|
540
|
-
const toolRows = await q(`
|
|
541
|
-
SELECT tc.folder AS folder, tc.tool_name AS tool_name, count() AS cnt FROM {{tool_calls}} AS tc
|
|
542
|
-
WHERE tc.folder != ''${inSessions(df, 'tc.session_id')} GROUP BY folder, tool_name`, df.params);
|
|
543
608
|
const toolsByFolder = {};
|
|
544
609
|
for (const r of toolRows) {
|
|
545
610
|
if (!toolsByFolder[r.folder]) toolsByFolder[r.folder] = {};
|
|
@@ -582,7 +647,7 @@ async function getDeepAnalytics(opts = {}) {
|
|
|
582
647
|
let sql = `
|
|
583
648
|
SELECT c.session_id AS id, c.user_id AS user_id, c.total_msgs AS msgs, c.user_chars AS uc, c.assistant_chars AS ac,
|
|
584
649
|
c.input_tokens AS ti, c.output_tokens AS to_, c.cache_read_tokens AS cr, c.cache_write_tokens AS cw
|
|
585
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY ${MS} DESC`;
|
|
650
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY ${MS} DESC, c.session_id, c.user_id`;
|
|
586
651
|
const params = { ...f.params };
|
|
587
652
|
if (opts.limit) { sql += ' LIMIT {limit:UInt64}'; params.limit = opts.limit; }
|
|
588
653
|
const rows = await q(sql, params);
|
|
@@ -606,18 +671,26 @@ async function getDeepAnalytics(opts = {}) {
|
|
|
606
671
|
let topTools = [], topModels = [], totalToolCalls = 0;
|
|
607
672
|
if (ids.length > 0) {
|
|
608
673
|
const pairs = { ids, users };
|
|
609
|
-
const
|
|
674
|
+
const r = await rooms();
|
|
675
|
+
const [toolRows, modelRows] = await Promise.all([
|
|
676
|
+
q(r.tool_stats ? `
|
|
677
|
+
SELECT tool_name, sum(calls) AS cnt FROM {{tool_stats}}
|
|
678
|
+
WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
|
|
679
|
+
GROUP BY tool_name` : `
|
|
610
680
|
SELECT tool_name, count() AS cnt FROM {{tool_calls}}
|
|
611
681
|
WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
|
|
612
|
-
GROUP BY tool_name`, pairs)
|
|
682
|
+
GROUP BY tool_name`, pairs),
|
|
683
|
+
q(r.model_stats ? `
|
|
684
|
+
SELECT model, sum(msgs) AS cnt FROM {{model_stats}}
|
|
685
|
+
WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
|
|
686
|
+
AND model NOT IN ${EXCLUDED_MODELS} GROUP BY model` : `
|
|
687
|
+
SELECT model, count() AS cnt FROM {{messages}}
|
|
688
|
+
WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
|
|
689
|
+
AND model NOT IN ${EXCLUDED_MODELS} GROUP BY model`, pairs),
|
|
690
|
+
]);
|
|
613
691
|
const toolFreq = {};
|
|
614
692
|
for (const r of toolRows) { toolFreq[r.tool_name] = Number(r.cnt); totalToolCalls += Number(r.cnt); }
|
|
615
693
|
topTools = topN(toolFreq, 30);
|
|
616
|
-
|
|
617
|
-
const modelRows = await q(`
|
|
618
|
-
SELECT model, count() AS cnt FROM {{messages}}
|
|
619
|
-
WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
|
|
620
|
-
AND model NOT IN ${EXCLUDED_MODELS} GROUP BY model`, pairs);
|
|
621
694
|
topModels = topN(normalizedModelFreq(modelRows), 20);
|
|
622
695
|
}
|
|
623
696
|
|
|
@@ -674,13 +747,19 @@ async function getToolCalls(toolName, opts = {}) {
|
|
|
674
747
|
const ORPHAN_TOKENS = '(m.input_tokens > 0 OR m.output_tokens > 0 OR m.cache_read_tokens > 0 OR m.cache_write_tokens > 0)';
|
|
675
748
|
|
|
676
749
|
// session_id → dominant model (most frequent across the session's messages).
|
|
677
|
-
async function sessionDominantMap(f) {
|
|
678
|
-
|
|
750
|
+
async function sessionDominantMap(f, scope = null) {
|
|
751
|
+
// model_stats already holds one row per (session, model) with the message count —
|
|
752
|
+
// argMax over a few thousand pre-aggregated rows instead of a scan of the room.
|
|
753
|
+
const r = await rooms();
|
|
754
|
+
const rows = await q(r.model_stats ? `
|
|
755
|
+
SELECT session_id, user_id, argMax(model, msgs) AS dominant
|
|
756
|
+
FROM {{model_stats}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
|
|
757
|
+
GROUP BY session_id, user_id` : `
|
|
679
758
|
SELECT session_id, user_id, argMax(model, cnt) AS dominant
|
|
680
759
|
FROM (SELECT m.session_id AS session_id, m.user_id AS user_id, m.model AS model, count() AS cnt
|
|
681
|
-
FROM {{messages}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
760
|
+
FROM {{messages}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
|
|
682
761
|
GROUP BY session_id, user_id, model)
|
|
683
|
-
GROUP BY session_id, user_id`, f
|
|
762
|
+
GROUP BY session_id, user_id`, scopeParams(f, scope));
|
|
684
763
|
const map = {};
|
|
685
764
|
for (const r of rows) map[`${r.session_id}::${r.user_id}`] = r.dominant;
|
|
686
765
|
return map;
|
|
@@ -688,7 +767,11 @@ async function sessionDominantMap(f) {
|
|
|
688
767
|
|
|
689
768
|
// source → dominant model + global dominant (for sessions with tokens but no model).
|
|
690
769
|
async function sourceDominantMap(f) {
|
|
691
|
-
const
|
|
770
|
+
const r = await rooms();
|
|
771
|
+
const rows = await q(r.model_stats ? `
|
|
772
|
+
SELECT m.source AS source, m.model AS model, sum(m.msgs) AS cnt
|
|
773
|
+
FROM {{model_stats}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
774
|
+
GROUP BY source, model` : `
|
|
692
775
|
SELECT m.source AS source, m.model AS model, count() AS cnt
|
|
693
776
|
FROM {{messages}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
694
777
|
GROUP BY source, model`, f.params);
|
|
@@ -709,21 +792,50 @@ const CHARS_PER_TOKEN = 4;
|
|
|
709
792
|
async function estimateCosts(opts = {}) {
|
|
710
793
|
const f = filters(opts, { editorLike: true });
|
|
711
794
|
|
|
712
|
-
|
|
795
|
+
// Five independent reads, one round trip. sourceDominantMap stays lazy below — it is
|
|
796
|
+
// only needed when unmodeled sessions exist, which most passes have none of.
|
|
797
|
+
//
|
|
798
|
+
// The ORPHAN_TOKENS predicate is dropped on the model_stats path on purpose: it
|
|
799
|
+
// filtered individual message rows to any-token ones before summing, and a row with
|
|
800
|
+
// zero tokens contributes zero to every sum — the predicate saved work, never changed
|
|
801
|
+
// a number. The stat rows are already summed, so there is no work left to save.
|
|
802
|
+
const r = await rooms();
|
|
803
|
+
const [modelTokens, orphanRows, dominantMap, charRows, unmodeledRows] = await Promise.all([
|
|
804
|
+
q(r.model_stats ? `
|
|
805
|
+
SELECT m.model AS model, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
|
|
806
|
+
sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
|
|
807
|
+
FROM {{model_stats}} AS m
|
|
808
|
+
WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
809
|
+
GROUP BY model` : `
|
|
713
810
|
SELECT m.model AS model, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
|
|
714
811
|
sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
|
|
715
812
|
FROM {{messages}} AS m
|
|
716
813
|
WHERE m.model NOT IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
|
|
717
|
-
GROUP BY model`, f.params)
|
|
718
|
-
|
|
719
|
-
|
|
814
|
+
GROUP BY model`, f.params),
|
|
815
|
+
q(r.model_stats ? `
|
|
816
|
+
SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
|
|
817
|
+
sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
|
|
818
|
+
FROM {{model_stats}} AS m
|
|
819
|
+
WHERE m.model IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
|
|
820
|
+
GROUP BY session_id, user_id
|
|
821
|
+
HAVING sum(m.input_tokens) > 0 OR sum(m.output_tokens) > 0 OR sum(m.cache_read_tokens) > 0 OR sum(m.cache_write_tokens) > 0` : `
|
|
720
822
|
SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
|
|
721
823
|
sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
|
|
722
824
|
FROM {{messages}} AS m
|
|
723
825
|
WHERE m.model IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
|
|
724
|
-
GROUP BY session_id, user_id`, f.params)
|
|
725
|
-
|
|
726
|
-
|
|
826
|
+
GROUP BY session_id, user_id`, f.params),
|
|
827
|
+
sessionDominantMap(f),
|
|
828
|
+
q(`
|
|
829
|
+
SELECT c.session_id AS session_id, c.user_id AS user_id, c.user_chars AS userChars, c.assistant_chars AS asstChars
|
|
830
|
+
FROM {{sessions_v}} AS c
|
|
831
|
+
WHERE notEmpty(c.models) AND c.input_tokens = 0 AND c.output_tokens = 0
|
|
832
|
+
AND (c.user_chars > 0 OR c.assistant_chars > 0)${f.and}`, f.params),
|
|
833
|
+
q(`
|
|
834
|
+
SELECT c.source AS source, c.input_tokens AS input, c.output_tokens AS output,
|
|
835
|
+
c.cache_read_tokens AS cacheRead, c.cache_write_tokens AS cacheWrite
|
|
836
|
+
FROM {{sessions_v}} AS c
|
|
837
|
+
WHERE empty(c.models) AND (c.input_tokens > 0 OR c.output_tokens > 0)${f.and}`, f.params),
|
|
838
|
+
]);
|
|
727
839
|
|
|
728
840
|
const orphanByModel = {};
|
|
729
841
|
const addOrphan = (dominant, input, output, cacheRead, cacheWrite) => {
|
|
@@ -739,22 +851,12 @@ async function estimateCosts(opts = {}) {
|
|
|
739
851
|
}
|
|
740
852
|
|
|
741
853
|
// Sessions that name models but report zero tokens → estimate from chars.
|
|
742
|
-
const charRows = await q(`
|
|
743
|
-
SELECT c.session_id AS session_id, c.user_id AS user_id, c.user_chars AS userChars, c.assistant_chars AS asstChars
|
|
744
|
-
FROM {{sessions_v}} AS c
|
|
745
|
-
WHERE notEmpty(c.models) AND c.input_tokens = 0 AND c.output_tokens = 0
|
|
746
|
-
AND (c.user_chars > 0 OR c.assistant_chars > 0)${f.and}`, f.params);
|
|
747
854
|
for (const r of charRows) {
|
|
748
855
|
const dominant = dominantMap[`${r.session_id}::${r.user_id}`];
|
|
749
856
|
if (dominant) addOrphan(dominant, Math.round((Number(r.userChars) || 0) / CHARS_PER_TOKEN), Math.round((Number(r.asstChars) || 0) / CHARS_PER_TOKEN), 0, 0);
|
|
750
857
|
}
|
|
751
858
|
|
|
752
859
|
// Sessions with tokens but NO model at all → the source's dominant model.
|
|
753
|
-
const unmodeledRows = await q(`
|
|
754
|
-
SELECT c.source AS source, c.input_tokens AS input, c.output_tokens AS output,
|
|
755
|
-
c.cache_read_tokens AS cacheRead, c.cache_write_tokens AS cacheWrite
|
|
756
|
-
FROM {{sessions_v}} AS c
|
|
757
|
-
WHERE empty(c.models) AND (c.input_tokens > 0 OR c.output_tokens > 0)${f.and}`, f.params);
|
|
758
860
|
if (unmodeledRows.length > 0) {
|
|
759
861
|
const { sourceDominant, globalDominant } = await sourceDominantMap(f);
|
|
760
862
|
for (const r of unmodeledRows) {
|
|
@@ -792,26 +894,43 @@ async function estimateCosts(opts = {}) {
|
|
|
792
894
|
|
|
793
895
|
// Per-session cost attribution in a FIXED number of bulk queries (no per-session
|
|
794
896
|
// round-trips). Mirrors root computePerChatCosts.
|
|
795
|
-
|
|
796
|
-
|
|
897
|
+
/**
|
|
898
|
+
* @param {object} f the user's filter
|
|
899
|
+
* @param {object|null} scope {ids, users} — the sessions the caller will actually use.
|
|
900
|
+
* getChats passes the page it just fetched; the cost pages pass nothing, because they
|
|
901
|
+
* are pricing the whole house on purpose.
|
|
902
|
+
*/
|
|
903
|
+
async function computePerChatCosts(f, scope = null) {
|
|
904
|
+
const p = scopeParams(f, scope);
|
|
905
|
+
const r = await rooms();
|
|
906
|
+
// Five independent reads, one round trip. This function is called by three endpoints;
|
|
907
|
+
// sequential, it alone cost 5 x RTT per call. On the model_stats path the ORPHAN
|
|
908
|
+
// predicate is dropped/HAVINGed for the reason written in estimateCosts.
|
|
909
|
+
const [aRows, bRows, cRows, dominantMap, sourceDominants] = await Promise.all([
|
|
910
|
+
q(r.model_stats ? `
|
|
911
|
+
SELECT m.session_id AS session_id, m.user_id AS user_id, m.model AS model, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
|
|
912
|
+
sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
|
|
913
|
+
FROM {{model_stats}} AS m
|
|
914
|
+
WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
|
|
915
|
+
GROUP BY session_id, user_id, model` : `
|
|
797
916
|
SELECT m.session_id AS session_id, m.user_id AS user_id, m.model AS model, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
|
|
798
917
|
sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
|
|
799
918
|
FROM {{messages}} AS m
|
|
800
|
-
WHERE m.model NOT IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
|
|
801
|
-
GROUP BY session_id, user_id, model`,
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
919
|
+
WHERE m.model NOT IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
|
|
920
|
+
GROUP BY session_id, user_id, model`, p),
|
|
921
|
+
q(r.model_stats ? `
|
|
922
|
+
SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
|
|
923
|
+
sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
|
|
924
|
+
FROM {{model_stats}} AS m
|
|
925
|
+
WHERE m.model IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
|
|
926
|
+
GROUP BY session_id, user_id
|
|
927
|
+
HAVING sum(m.input_tokens) > 0 OR sum(m.output_tokens) > 0 OR sum(m.cache_read_tokens) > 0 OR sum(m.cache_write_tokens) > 0` : `
|
|
806
928
|
SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
|
|
807
929
|
sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
|
|
808
930
|
FROM {{messages}} AS m
|
|
809
|
-
WHERE m.model IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
|
|
810
|
-
GROUP BY session_id, user_id`,
|
|
811
|
-
|
|
812
|
-
for (const r of bRows) orphanByChat[`${r.session_id}::${r.user_id}`] = r;
|
|
813
|
-
|
|
814
|
-
const cRows = await q(`
|
|
931
|
+
WHERE m.model IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
|
|
932
|
+
GROUP BY session_id, user_id`, p),
|
|
933
|
+
q(`
|
|
815
934
|
SELECT c.session_id AS id, c.user_id AS user_id, c.source AS source, c.name AS name, c.folder AS folder,
|
|
816
935
|
toUnixTimestamp64Milli(c.last_updated_at) AS last_updated_at,
|
|
817
936
|
toUnixTimestamp64Milli(c.created_at) AS created_at,
|
|
@@ -820,10 +939,22 @@ async function computePerChatCosts(f) {
|
|
|
820
939
|
c.input_tokens AS ti, c.output_tokens AS to_,
|
|
821
940
|
c.cache_read_tokens AS cr, c.cache_write_tokens AS cw,
|
|
822
941
|
formatDateTime(${TS}, '%Y-%m', 'UTC') AS month
|
|
823
|
-
FROM {{sessions_v}} AS c WHERE 1=1${f.and}`,
|
|
942
|
+
FROM {{sessions_v}} AS c WHERE 1=1${f.and}${scopeAnd(scope, 'c')}`, p),
|
|
943
|
+
sessionDominantMap(f, scope),
|
|
944
|
+
// NOT scoped, deliberately. This is the house-wide "which model does this editor
|
|
945
|
+
// usually run", used to price sessions that recorded no model at all. Computed from
|
|
946
|
+
// the fifty rows on the page it would answer differently on page 1 than on page 2,
|
|
947
|
+
// and the same session would report a different cost depending on where it was read.
|
|
948
|
+
// Its scope is the user's FILTER, which is what it has always been.
|
|
949
|
+
sourceDominantMap(f),
|
|
950
|
+
]);
|
|
951
|
+
const { sourceDominant, globalDominant } = sourceDominants;
|
|
824
952
|
|
|
825
|
-
const
|
|
826
|
-
const {
|
|
953
|
+
const byChatModel = {};
|
|
954
|
+
for (const r of aRows) { const k = `${r.session_id}::${r.user_id}`; (byChatModel[k] = byChatModel[k] || []).push(r); }
|
|
955
|
+
|
|
956
|
+
const orphanByChat = {};
|
|
957
|
+
for (const r of bRows) orphanByChat[`${r.session_id}::${r.user_id}`] = r;
|
|
827
958
|
|
|
828
959
|
const out = [];
|
|
829
960
|
for (const c of cRows) {
|
|
@@ -868,8 +999,7 @@ async function computePerChatCosts(f) {
|
|
|
868
999
|
|
|
869
1000
|
async function getCostAnalytics(opts = {}) {
|
|
870
1001
|
const f = filters(opts, { editorLike: true });
|
|
871
|
-
const overall = await estimateCosts(opts);
|
|
872
|
-
const perChat = await computePerChatCosts(f);
|
|
1002
|
+
const [overall, perChat] = await Promise.all([estimateCosts(opts), computePerChatCosts(f)]);
|
|
873
1003
|
|
|
874
1004
|
const editorAgg = {}, projectAgg = {}, monthCosts = {};
|
|
875
1005
|
const sessionCosts = [];
|
|
@@ -893,7 +1023,11 @@ async function getCostAnalytics(opts = {}) {
|
|
|
893
1023
|
const byEditor = Object.entries(editorAgg).map(([editor, d]) => ({ editor, cost: d.cost, models: d.models.size })).sort((a, b) => b.cost - a.cost);
|
|
894
1024
|
const byProject = Object.entries(projectAgg).map(([folder, cost]) => ({ folder, name: folder.split('/').pop(), cost })).sort((a, b) => b.cost - a.cost).slice(0, 20);
|
|
895
1025
|
const monthly = Object.entries(monthCosts).sort((a, b) => a[0].localeCompare(b[0])).map(([month, d]) => ({ month, cost: Math.round(d.cost * 100) / 100, sessions: d.sessions }));
|
|
896
|
-
|
|
1026
|
+
// Same total-order rule as getChats: a top-50 cut over ties must not depend on the
|
|
1027
|
+
// order the rows happened to arrive in. Every session in a synthetic house costs the
|
|
1028
|
+
// same, and the list changed identity between two equivalent sources without a
|
|
1029
|
+
// secondary key.
|
|
1030
|
+
sessionCosts.sort((a, b) => (b.cost - a.cost) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
897
1031
|
|
|
898
1032
|
const totalSessions = sessionCosts.length;
|
|
899
1033
|
const avgPerSession = totalSessions > 0 ? overall.totalCost / totalSessions : 0;
|
package/memhouse/shipper/ship.js
CHANGED
|
@@ -62,7 +62,9 @@ const selfUpdate = require('../self-update');
|
|
|
62
62
|
const selfSnap = selfUpdate.snapshot(__filename);
|
|
63
63
|
const {
|
|
64
64
|
resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN, createStatement,
|
|
65
|
-
SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor
|
|
65
|
+
SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor,
|
|
66
|
+
statCreateStatements, statRefreshStatements, statTableNames,
|
|
67
|
+
} = require('../house/house');
|
|
66
68
|
// Rooms plus the house's own record of itself. Every table the template declares, which
|
|
67
69
|
// is what the column healer and the drift warning have to cover — a column added to
|
|
68
70
|
// events would otherwise roll out to nobody.
|
|
@@ -463,6 +465,24 @@ async function ensureSchema(client) {
|
|
|
463
465
|
throw e;
|
|
464
466
|
}
|
|
465
467
|
}
|
|
468
|
+
// The precomputed rollups — additive, so they belong here and not in a migration (see
|
|
469
|
+
// migrate.js's rule). Three plain tables the shipper fills after every pass; the read
|
|
470
|
+
// layer probes for them at startup and falls back to the inline rollup when they are
|
|
471
|
+
// absent or empty, so a denied CREATE here degrades to slow, never to wrong. Why they
|
|
472
|
+
// are shipper-filled and not materialized views: house.js, above statBodies.
|
|
473
|
+
for (const q of statCreateStatements(rooms)) {
|
|
474
|
+
try {
|
|
475
|
+
await client.command({ query: q, clickhouse_settings: { async_insert: 0 } });
|
|
476
|
+
} catch (e) {
|
|
477
|
+
const m = e && e.message ? e.message : String(e);
|
|
478
|
+
if (/Not enough privileges|ACCESS_DENIED/i.test(m)) { denied++; continue; }
|
|
479
|
+
if (/ALREADY_EXISTS|already exists/i.test(m)) { continue; }
|
|
480
|
+
throw e;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
// First generation now, so install and update are fast at once instead of after the
|
|
484
|
+
// next pass. Best effort: a denied CREATE above means these fail too, quietly.
|
|
485
|
+
await refreshStats(client, rooms);
|
|
466
486
|
if (denied) {
|
|
467
487
|
// In a SHARED house this is the designed state, not a shortfall: the operator creates
|
|
468
488
|
// each member's rooms during `invite` precisely so the member cannot — a member who
|
|
@@ -903,8 +923,11 @@ function rowsForChat(chat, host) {
|
|
|
903
923
|
text,
|
|
904
924
|
project,
|
|
905
925
|
folder,
|
|
906
|
-
is_subagent: text.startsWith('[subagent]'),
|
|
907
|
-
|
|
926
|
+
is_subagent: text.startsWith('[subagent]') || Boolean(m._agent),
|
|
927
|
+
// Which subagent (Claude Code fork) a folded turn belongs to — id, the description
|
|
928
|
+
// the parent gave it, its type, and the turn's position within that subagent — so
|
|
929
|
+
// one fork's transcript is `WHERE extra.agent.id = '…' ORDER BY seq`. Not hashed.
|
|
930
|
+
extra: m._agent ? { agent: { id: String(m._agent.id || ''), description: String(m._agent.description || ''), type: String(m._agent.type || ''), turn: Number(m._agent.turn) || 0, ...(m._agent.workflow ? { workflow: String(m._agent.workflow) } : {}) } } : {},
|
|
908
931
|
};
|
|
909
932
|
// Hash the content fields only (not ts — it is synthesized and may drift).
|
|
910
933
|
row.line_hash = lineHash({
|
|
@@ -1051,6 +1074,38 @@ async function loadStoredParse(client, rooms, id, uid, epoch) {
|
|
|
1051
1074
|
return { hashes, tools };
|
|
1052
1075
|
}
|
|
1053
1076
|
|
|
1077
|
+
/**
|
|
1078
|
+
* Recompute the three stat tables — one generation, one INSERT each, all stamped with the
|
|
1079
|
+
* same refreshed_at. Runs at the end of every ship pass, which is exactly the cadence a
|
|
1080
|
+
* refreshable view would have had, and after --ensure-schema. Best effort: a house whose
|
|
1081
|
+
* tables were never created (a denied CREATE) fails every INSERT here, and the read layer
|
|
1082
|
+
* is already on the inline rollup.
|
|
1083
|
+
*/
|
|
1084
|
+
async function refreshStats(client, rooms) {
|
|
1085
|
+
// One timestamp for all three, taken once — three now64() calls would be three
|
|
1086
|
+
// generations, and the read layer would pair session_stats from one with model_stats
|
|
1087
|
+
// from another.
|
|
1088
|
+
let at;
|
|
1089
|
+
try {
|
|
1090
|
+
const rs = await client.query({ query: 'SELECT toString(now64(3)) AS t', format: 'JSONEachRow' });
|
|
1091
|
+
at = `toDateTime64('${(await rs.json())[0].t}', 3, 'UTC')`;
|
|
1092
|
+
} catch { return 0; }
|
|
1093
|
+
let done = 0;
|
|
1094
|
+
for (const q of statRefreshStatements(rooms, at)) {
|
|
1095
|
+
try {
|
|
1096
|
+
await client.command({ query: q, clickhouse_settings: { ...READ_SETTINGS, async_insert: 0 } });
|
|
1097
|
+
done++;
|
|
1098
|
+
} catch (e) {
|
|
1099
|
+
const m = e && e.message ? e.message : String(e);
|
|
1100
|
+
if (!/Not enough privileges|ACCESS_DENIED|UNKNOWN_TABLE|doesn't exist|does not exist/i.test(m)) {
|
|
1101
|
+
console.error(`[memhouse] stats refresh: ${m.split('\n')[0]}`);
|
|
1102
|
+
}
|
|
1103
|
+
return done;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
return done;
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1054
1109
|
// One shipping pass. Incremental unless opts.full: a chat is skipped when the house
|
|
1055
1110
|
// already has it at least as fresh (last_updated_at) and at least as large
|
|
1056
1111
|
// (extra.bubbleCount) — both readable without parsing the chat.
|
|
@@ -1198,6 +1253,9 @@ async function runShip(client, opts = {}) {
|
|
|
1198
1253
|
for (const r of rows.toolRows) { r.epoch = epoch; await push('tool_calls', scrub(r)); toolRows++; }
|
|
1199
1254
|
}
|
|
1200
1255
|
for (const table of Object.keys(batches)) await flush(table);
|
|
1256
|
+
// The dashboard's tables, recomputed from what this pass just wrote. Skipped on a pass
|
|
1257
|
+
// that shipped nothing: the previous generation is still exact.
|
|
1258
|
+
if (sessions > 0) await refreshStats(client, rooms);
|
|
1201
1259
|
// Anything that only failed while reading messages — the sink is reset by the
|
|
1202
1260
|
// next getAllChats(), so unreported here means never reported at all.
|
|
1203
1261
|
reportAdapterErrors(warned);
|
|
@@ -1352,7 +1410,7 @@ async function main() {
|
|
|
1352
1410
|
}
|
|
1353
1411
|
}
|
|
1354
1412
|
|
|
1355
|
-
module.exports = { runShip, ensureSchema, templateColumns, decideEpoch };
|
|
1413
|
+
module.exports = { runShip, ensureSchema, refreshStats, templateColumns, decideEpoch };
|
|
1356
1414
|
|
|
1357
1415
|
if (require.main === module) {
|
|
1358
1416
|
main().catch((e) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "memhouse",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.3",
|
|
4
4
|
"description": "memhouse \u2014 agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, kernel-agency install, and the agentlytics dashboard",
|
|
5
5
|
"main": "bin/memhouse.js",
|
|
6
6
|
"bin": {
|
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
"dev": "cd ui && npm run dev",
|
|
29
29
|
"test": "node misc/syntax-check.js && node misc/unit-test.js",
|
|
30
30
|
"test:house": "node misc/house-test.js",
|
|
31
|
-
"prepack": "cd ui && npm install --no-audit --no-fund && npm run build"
|
|
31
|
+
"prepack": "cd ui && npm install --no-audit --no-fund && npm run build",
|
|
32
|
+
"prepublishOnly": "node misc/publish-guard.js"
|
|
32
33
|
},
|
|
33
34
|
"keywords": [
|
|
34
35
|
"memory",
|