memhouse 0.17.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "mem",
3
3
  "description": "Conversation memory for AI agents over the memhouse typed ClickHouse store: recall past sessions and answer from them, run analytics, check the house, bring people in, and administer the server. Skills install as /mem:recall, /mem:sql, /mem:house, /mem:access, and /mem:admin.",
4
- "version": "0.17.0",
4
+ "version": "0.17.1",
5
5
  "author": {
6
6
  "name": "Ramazan Polat"
7
7
  },
@@ -288,6 +288,97 @@ function sessionsRollup({ sessions, messages }) {
288
288
  )`;
289
289
  }
290
290
 
291
+ // The precomputed rollup's table name. One place, because the migration, the runtime
292
+ // probe and the read layer must all agree on it.
293
+ const SESSION_STATS = 'session_stats';
294
+ // The per-(session, model) and per-(session, tool) rollups — what lets the cost, model
295
+ // and tool queries stop scanning the messages/tool_calls rooms entirely. Same refresh
296
+ // mechanism as SESSION_STATS; same fallback rule: absent tables mean the legacy SQL.
297
+ const MODEL_STATS = 'session_model_stats';
298
+ const TOOL_STATS = 'session_tool_stats';
299
+
300
+ /**
301
+ * The DDL that materializes the rollup, as [target, view].
302
+ *
303
+ * A REFRESHABLE materialized view, not an ordinary one, and the distinction is not a
304
+ * preference — an ordinary MV is an insert trigger, and all three of this schema's
305
+ * defining properties break it:
306
+ *
307
+ * DUPLICATES. The rooms are ReplacingMergeTree; the engine collapses re-inserted rows
308
+ * at MERGE time, long after a trigger has already added them to a sum(). On a real
309
+ * house that is 1,226,770 stored rows for 685,649 real ones — token totals ~1.8x high,
310
+ * with nothing to indicate it.
311
+ *
312
+ * EPOCHS. A trigger firing at insert cannot know a later parse will supersede the rows
313
+ * it is aggregating, so superseded parses would stay in the totals forever.
314
+ *
315
+ * THE JOIN. A trigger fires on one source table; the rollup joins sessions to messages.
316
+ *
317
+ * A refreshable view has none of those problems because it is not a trigger: it runs
318
+ * THIS ALREADY-CORRECT QUERY on a schedule and atomically swaps the target. FINAL, the
319
+ * epoch filter and the join all work exactly as they do in the subquery, because it IS
320
+ * the subquery. The price is staleness bounded by the interval — and the shipper's own
321
+ * default cadence is 300s, so a 5-minute refresh adds no lag to data arriving every 5.
322
+ *
323
+ * The target's columns are inferred with EMPTY AS, so the table and the rollup cannot
324
+ * drift: change sessionsRollup and the next migration rebuilds the table to match.
325
+ */
326
+ function sessionStatsStatements(rooms, { intervalMinutes = 5 } = {}) {
327
+ const rollup = sessionsRollup(rooms);
328
+ // sessionsRollup wraps itself in parentheses for the FROM position; strip them.
329
+ const body = rollup.replace(/^\s*\(/, '').replace(/\)\s*$/, '');
330
+ return [
331
+ `CREATE TABLE IF NOT EXISTS ${SESSION_STATS} ENGINE = MergeTree `
332
+ + `ORDER BY (session_id, user_id) EMPTY AS ${body}`,
333
+ `CREATE MATERIALIZED VIEW IF NOT EXISTS ${SESSION_STATS}_mv `
334
+ + `REFRESH EVERY ${intervalMinutes} MINUTE TO ${SESSION_STATS} AS ${body}`,
335
+ ];
336
+ }
337
+
338
+ /**
339
+ * The two fine-grained rollups, same shape as sessionStatsStatements.
340
+ *
341
+ * MODEL_STATS one row per (session, user, source, folder, model), token sums included.
342
+ * Every model/cost aggregate the dashboard runs is a sum or argMax over these rows —
343
+ * a few thousand of them — instead of a scan of the full messages room. The ORPHAN
344
+ * (any-token) predicates in the read layer survive unchanged: a message row with zero
345
+ * tokens contributes zero to every sum, so filtering it out before summing and summing
346
+ * over everything produce the same number.
347
+ *
348
+ * TOOL_STATS one row per (session, user, source, folder, tool_name) with the call
349
+ * count. NOTE the epoch source: currentParse(tool_calls, 'messages') — the tool room
350
+ * takes its epoch from MESSAGES, because a parse that keeps its messages but emits no
351
+ * tool calls writes nothing here at the new epoch, and this room's own max(epoch)
352
+ * would resurrect the superseded parse's calls.
353
+ */
354
+ function sessionModelStatsStatements(rooms, { intervalMinutes = 5 } = {}) {
355
+ const body = `SELECT session_id, user_id, any(source) AS source, any(folder) AS folder, model,
356
+ count() AS msgs,
357
+ sum(input_tokens) AS input_tokens, sum(output_tokens) AS output_tokens,
358
+ sum(cache_read_tokens) AS cache_read_tokens, sum(cache_write_tokens) AS cache_write_tokens
359
+ FROM ${rooms.messages} AS m
360
+ GROUP BY session_id, user_id, model`;
361
+ return [
362
+ `CREATE TABLE IF NOT EXISTS ${MODEL_STATS} ENGINE = MergeTree `
363
+ + `ORDER BY (session_id, user_id, model) EMPTY AS ${body}`,
364
+ `CREATE MATERIALIZED VIEW IF NOT EXISTS ${MODEL_STATS}_mv `
365
+ + `REFRESH EVERY ${intervalMinutes} MINUTE TO ${MODEL_STATS} AS ${body}`,
366
+ ];
367
+ }
368
+
369
+ function sessionToolStatsStatements(rooms, { intervalMinutes = 5 } = {}) {
370
+ const body = `SELECT session_id, user_id, any(source) AS source, any(folder) AS folder, tool_name,
371
+ count() AS calls
372
+ FROM ${rooms.tool_calls} AS tc
373
+ GROUP BY session_id, user_id, tool_name`;
374
+ return [
375
+ `CREATE TABLE IF NOT EXISTS ${TOOL_STATS} ENGINE = MergeTree `
376
+ + `ORDER BY (session_id, user_id, tool_name) EMPTY AS ${body}`,
377
+ `CREATE MATERIALIZED VIEW IF NOT EXISTS ${TOOL_STATS}_mv `
378
+ + `REFRESH EVERY ${intervalMinutes} MINUTE TO ${TOOL_STATS} AS ${body}`,
379
+ ];
380
+ }
381
+
291
382
  /**
292
383
  * The settings every read needs. `final` collapses ReplacingMergeTree versions;
293
384
  * `join_use_nulls` is what the rollup's coalesce depends on.
@@ -333,8 +424,65 @@ async function currentUser(client) {
333
424
  }
334
425
 
335
426
  /** Resolve the rooms for whoever this client is connected as. */
427
+ /**
428
+ * The precomputed rollup, when the house has one.
429
+ *
430
+ * `sessions_v` is a SAVED QUERY that rebuilds the whole rollup — sessions FINAL joined
431
+ * to the current parse of messages, grouped over the entire house — on every call. A
432
+ * house that has run the session_stats migration carries the same rows as a flat table,
433
+ * refreshed on a timer, and the read layer can name that instead. Measured on a 714k
434
+ * message house: 1,413,624 rows read per query becomes 1,230, and p50 query time 68ms
435
+ * becomes 4ms.
436
+ *
437
+ * Resolved at RUNTIME rather than by version, and that is the point: one binary has to
438
+ * serve a migrated house, a house whose migration has not run, and a friend's house
439
+ * shared read-only that was never migrated at all. Asking the server what exists is the
440
+ * only answer that is correct in all three.
441
+ */
442
+ async function statTables(client) {
443
+ try {
444
+ const rs = await client.query({
445
+ query: `SELECT name FROM system.tables
446
+ WHERE database = currentDatabase()
447
+ AND name IN ('${SESSION_STATS}', '${MODEL_STATS}', '${TOOL_STATS}')`,
448
+ format: 'JSONEachRow',
449
+ });
450
+ const present = (await rs.json()).map((r) => r.name);
451
+ // Present is not enough: a refreshable view populates its target on its FIRST
452
+ // refresh, which is scheduled, not immediate. Between `install` and that refresh the
453
+ // table exists and is empty — and a dashboard reading it would show a house with no
454
+ // sessions at all (caught by house-test the first time). An empty stat table reads
455
+ // as absent, so the inline rollup answers until the table can.
456
+ const usable = new Set();
457
+ for (const name of present) {
458
+ try {
459
+ const c = await client.query({ query: `SELECT count() AS n FROM ${name}`, format: 'JSONEachRow' });
460
+ const rows = await c.json();
461
+ if (Number(rows[0] && rows[0].n) > 0) usable.add(name);
462
+ } catch { /* unreadable: treat as absent */ }
463
+ }
464
+ return usable;
465
+ } catch {
466
+ // No grant on system.tables is not an error — it means "assume none", and the
467
+ // subquery path is always correct.
468
+ return new Set();
469
+ }
470
+ }
471
+
472
+ async function hasSessionStats(client) {
473
+ return (await statTables(client)).has(SESSION_STATS);
474
+ }
475
+
336
476
  async function resolveRooms(client) {
337
- return roomNames(await currentUser(client));
477
+ const rooms = roomNames(await currentUser(client));
478
+ const have = await statTables(client);
479
+ if (have.has(SESSION_STATS)) rooms.sessions_v = SESSION_STATS;
480
+ // Null when absent: the read layer branches to the legacy scan-the-room SQL. Never a
481
+ // token substitution, because the legacy SQL has a different shape, not just a
482
+ // different table name.
483
+ rooms.model_stats = have.has(MODEL_STATS) ? MODEL_STATS : null;
484
+ rooms.tool_stats = have.has(TOOL_STATS) ? TOOL_STATS : null;
485
+ return rooms;
338
486
  }
339
487
 
340
488
  module.exports = {
@@ -343,4 +491,6 @@ module.exports = {
343
491
  READ_SETTINGS, MEMBER_PIN,
344
492
  installCommand, assertUsableName,
345
493
  sessionsRollup, currentParse, createStatement, roomNames, currentUser, resolveRooms,
494
+ SESSION_STATS, MODEL_STATS, TOOL_STATS, hasSessionStats, statTables,
495
+ sessionStatsStatements, sessionModelStatsStatements, sessionToolStatsStatements,
346
496
  };
@@ -140,6 +140,28 @@ function inSessions(f, col) {
140
140
  return ` AND (${alias}.session_id, ${alias}.user_id) IN (SELECT session_id, user_id FROM {{sessions_v}} AS c WHERE 1=1${f.and})`;
141
141
  }
142
142
 
143
+ /**
144
+ * Narrow a query to an explicit set of (session_id, user_id) pairs.
145
+ *
146
+ * Distinct from inSessions(), which restricts to whatever the USER filtered to. This
147
+ * restricts to the rows a caller is actually going to use — the fifty sessions on the
148
+ * page, not the twelve hundred in the house.
149
+ *
150
+ * arrayZip, not two INs: the rooms are keyed on the PAIR, and matching the columns
151
+ * independently would pull in another member's rows for a colliding adapter-local
152
+ * session_id. Same idiom getDeepAnalytics already uses.
153
+ */
154
+ function scopeAnd(scope, alias) {
155
+ if (!scope || !scope.ids || !scope.ids.length) return '';
156
+ const a = alias ? `${alias}.` : '';
157
+ return ` AND (${a}session_id, ${a}user_id) IN arrayZip({scopeIds:Array(String)}, {scopeUsers:Array(String)})`;
158
+ }
159
+
160
+ function scopeParams(f, scope) {
161
+ if (!scope || !scope.ids || !scope.ids.length) return f.params;
162
+ return { ...f.params, scopeIds: scope.ids, scopeUsers: scope.users };
163
+ }
164
+
143
165
  // Fold a [{name|model, cnt}] list through normalizeModelName into a freq map.
144
166
  function normalizedModelFreq(rows) {
145
167
  const freq = {};
@@ -151,37 +173,52 @@ function normalizedModelFreq(rows) {
151
173
  }
152
174
  // The same exclusions as EXCLUDED_MODELS, usable in JS.
153
175
  const EXCLUDED_MODEL_SET = new Set(['', '<synthetic>']);
154
- const topN = (freq, n) => Object.entries(freq).sort((a, b) => b[1] - a[1]).slice(0, n)
176
+ // Tiebreak on the name. A top-N cut over counts alone is not a total order: two tools
177
+ // tied at the cut line resolve by arrival order, so the same house can show a different
178
+ // tenth tool on consecutive loads. Same rule as getChats' ORDER BY tiebreaker.
179
+ const topN = (freq, n) => Object.entries(freq).sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)).slice(0, n)
155
180
  .map(([name, count]) => ({ name, count }));
156
181
 
157
182
  // ── overview ────────────────────────────────────────────────────────────────────
158
183
  async function getOverview(opts = {}) {
159
184
  const f = filters(opts);
160
- // Same definition as getChats(). The nav bar read this and the Sessions page read
161
- // getChats, so a house with one placeholder session showed "9 sessions" beside
162
- // "8 sessions" in the same header, and the depth histogram summed to 7. A session row
163
- // with no name and no messages is a placeholder the shipper keeps to absorb the
164
- // incremental skip; it is not a conversation, and only one of the two counters knew.
165
- const totalChats = Number((await q1(
166
- `SELECT count() AS cnt FROM {{sessions_v}} AS c WHERE 1=1${f.and} AND (c.name != '' OR c.total_msgs > 0)`,
167
- f.params)).cnt);
168
-
169
- // Root parity WAS: without a folder filter the editor breakdown is global — which put
170
- // "34 + 3" chips under a header reading "3 sessions" when an editor filter was applied,
171
- // and 12 under a header of 11 with no filter at all. Parity with a wrong number is not a
172
- // feature.
173
- const editors = await q(
174
- `SELECT source, count() AS count FROM {{sessions_v}} AS c
175
- WHERE (c.name != '' OR c.total_msgs > 0)${f.and} GROUP BY source ORDER BY count DESC`, f.params);
176
-
177
- const modes = await q(`SELECT mode, count() AS count FROM {{sessions_v}} AS c WHERE mode != ''${f.and} GROUP BY mode`, f.params);
185
+ // Six independent reads, issued together. They share nothing but the filter, and every
186
+ // one of them is a full round trip: awaited in sequence they cost 6 x RTT before any
187
+ // result exists. Measured against a ClickHouse 6,000 km away, that sequencing — not the
188
+ // database — was roughly two thirds of what the browser waited for.
189
+ //
190
+ // The comments on each query below are the reasons its SHAPE is what it is; none of
191
+ // them is a reason for it to wait on the one above.
192
+ const [totalRow, editors, modes, rows, projects, range] = await Promise.all([
193
+ // Same definition as getChats(). The nav bar read this and the Sessions page read
194
+ // getChats, so a house with one placeholder session showed "9 sessions" beside
195
+ // "8 sessions" in the same header, and the depth histogram summed to 7. A session row
196
+ // with no name and no messages is a placeholder the shipper keeps to absorb the
197
+ // incremental skip; it is not a conversation, and only one of the two counters knew.
198
+ q1(`SELECT count() AS cnt FROM {{sessions_v}} AS c WHERE 1=1${f.and} AND (c.name != '' OR c.total_msgs > 0)`,
199
+ f.params),
200
+ // Root parity WAS: without a folder filter the editor breakdown is global — which put
201
+ // "34 + 3" chips under a header reading "3 sessions" when an editor filter was applied,
202
+ // and 12 under a header of 11 with no filter at all. Parity with a wrong number is not a
203
+ // feature.
204
+ q(`SELECT source, count() AS count FROM {{sessions_v}} AS c
205
+ WHERE (c.name != '' OR c.total_msgs > 0)${f.and} GROUP BY source ORDER BY count DESC`, f.params),
206
+ q(`SELECT mode, count() AS count FROM {{sessions_v}} AS c WHERE mode != ''${f.and} GROUP BY mode`, f.params),
207
+ q(`
208
+ SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
209
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and}
210
+ GROUP BY month, source ORDER BY month`, f.params),
211
+ q(`
212
+ SELECT folder, count() AS count FROM {{sessions_v}} AS c
213
+ WHERE folder != ''${f.and} GROUP BY folder ORDER BY count DESC LIMIT 20`, f.params),
214
+ q1(`SELECT min(${MS}) AS oldest, max(${MS}) AS newest FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params),
215
+ ]);
216
+
217
+ const totalChats = Number(totalRow.cnt);
218
+
178
219
  const byMode = {};
179
220
  for (const m of modes) byMode[m.mode] = Number(m.count);
180
221
 
181
- const rows = await q(`
182
- SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
183
- FROM {{sessions_v}} AS c WHERE 1=1${f.and}
184
- GROUP BY month, source ORDER BY month`, f.params);
185
222
  const monthMap = {};
186
223
  for (const r of rows) {
187
224
  if (!monthMap[r.month]) monthMap[r.month] = { count: 0, editors: {} };
@@ -190,17 +227,12 @@ async function getOverview(opts = {}) {
190
227
  }
191
228
  const byMonth = Object.keys(monthMap).sort().map(m => ({ month: m, ...monthMap[m] }));
192
229
 
193
- const projects = await q(`
194
- SELECT folder, count() AS count FROM {{sessions_v}} AS c
195
- WHERE folder != ''${f.and} GROUP BY folder ORDER BY count DESC LIMIT 20`, f.params);
196
230
  const topProjects = projects.map(p => ({
197
231
  name: p.folder.split(/[/\\]/).slice(-2).join('/'),
198
232
  fullPath: p.folder,
199
233
  count: Number(p.count),
200
234
  }));
201
235
 
202
- const range = await q1(`SELECT min(${MS}) AS oldest, max(${MS}) AS newest FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params);
203
-
204
236
  return {
205
237
  totalChats,
206
238
  editors: editors.map(e => ({ id: e.source, count: Number(e.count) })),
@@ -238,21 +270,62 @@ async function getDailyActivity(opts = {}) {
238
270
  async function getDashboardStats(opts = {}) {
239
271
  const f = filters(opts);
240
272
  const p = { ...f.params, tz: TZ };
273
+ const r = await rooms();
241
274
 
242
- const hourlyRows = await q(`
275
+ // Nine independent reads, one round trip. Sequential, this endpoint paid 9 x RTT
276
+ // before any result existed — the largest fan-out on the dashboard. Each query's
277
+ // shape is unchanged; only the waiting is gone.
278
+ const [hourlyRows, weekdayRows, depthRows, tokenRow, streakRows,
279
+ monthEditorRows, velocityRows, modelRows, toolRows] = await Promise.all([
280
+ q(`
243
281
  SELECT toHour(${TS}, {tz:String}) AS hour, count() AS count
244
- FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY hour ORDER BY hour`, p);
282
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY hour ORDER BY hour`, p),
283
+ q(`
284
+ SELECT toDayOfWeek(${TS}, 0, {tz:String}) % 7 AS dow, count() AS count
285
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY dow ORDER BY dow`, p),
286
+ q(`
287
+ SELECT total_msgs AS msgs FROM {{sessions_v}} AS c WHERE total_msgs > 0${f.and}`, f.params),
288
+ q1(`
289
+ SELECT COALESCE(sum(input_tokens), 0) AS input, COALESCE(sum(output_tokens), 0) AS output,
290
+ COALESCE(sum(cache_read_tokens), 0) AS cacheRead, COALESCE(sum(cache_write_tokens), 0) AS cacheWrite,
291
+ COALESCE(sum(user_chars), 0) AS userChars, COALESCE(sum(assistant_chars), 0) AS assistantChars,
292
+ COALESCE(sum(total_msgs), 0) AS messages,
293
+ -- The SAME definition the KPI beside it uses. A plain count() here put a
294
+ -- fourth session number on a screen that already had three: nav 11, chips 12,
295
+ -- avg depth 40/12, histogram 10.
296
+ countIf(c.name != '' OR c.total_msgs > 0) AS sessions
297
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params),
298
+ q(`
299
+ SELECT DISTINCT formatDateTime(${TS}, '%Y-%m-%d', {tz:String}) AS day
300
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY day`, p),
301
+ q(`
302
+ SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
303
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY month, source ORDER BY month`, f.params),
304
+ q(`
305
+ SELECT formatDateTime(c.last_updated_at, '%Y-%m', 'UTC') AS month,
306
+ avg(total_msgs) AS avgMsgs, avg(input_tokens + output_tokens) AS avgTokens
307
+ FROM {{sessions_v}} AS c WHERE c.last_updated_at IS NOT NULL${f.and}
308
+ GROUP BY month ORDER BY month`, f.params),
309
+ q(r.model_stats ? `
310
+ SELECT m.model AS model, sum(m.msgs) AS cnt FROM {{model_stats}} AS m
311
+ WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
312
+ GROUP BY model` : `
313
+ SELECT m.model AS model, count() AS cnt FROM {{messages}} AS m
314
+ WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
315
+ GROUP BY model`, f.params),
316
+ q(r.tool_stats ? `
317
+ SELECT tool_name, sum(tc.calls) AS cnt FROM {{tool_stats}} AS tc
318
+ WHERE 1=1${inSessions(f, 'tc.session_id')} GROUP BY tool_name` : `
319
+ SELECT tool_name, count() AS cnt FROM {{tool_calls}} AS tc
320
+ WHERE 1=1${inSessions(f, 'tc.session_id')} GROUP BY tool_name`, f.params),
321
+ ]);
322
+
245
323
  const hourly = new Array(24).fill(0);
246
324
  for (const r of hourlyRows) hourly[Number(r.hour)] = Number(r.count);
247
325
 
248
- const weekdayRows = await q(`
249
- SELECT toDayOfWeek(${TS}, 0, {tz:String}) % 7 AS dow, count() AS count
250
- FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY dow ORDER BY dow`, p);
251
326
  const weekdays = new Array(7).fill(0);
252
327
  for (const r of weekdayRows) weekdays[Number(r.dow)] = Number(r.count);
253
328
 
254
- const depthRows = await q(`
255
- SELECT total_msgs AS msgs FROM {{sessions_v}} AS c WHERE total_msgs > 0${f.and}`, f.params);
256
329
  const depthBuckets = { '1': 0, '2-5': 0, '6-10': 0, '11-20': 0, '21-50': 0, '51-100': 0, '100+': 0 };
257
330
  for (const r of depthRows) {
258
331
  const m = Number(r.msgs);
@@ -265,20 +338,6 @@ async function getDashboardStats(opts = {}) {
265
338
  else depthBuckets['100+']++;
266
339
  }
267
340
 
268
- const tokenRow = await q1(`
269
- SELECT COALESCE(sum(input_tokens), 0) AS input, COALESCE(sum(output_tokens), 0) AS output,
270
- COALESCE(sum(cache_read_tokens), 0) AS cacheRead, COALESCE(sum(cache_write_tokens), 0) AS cacheWrite,
271
- COALESCE(sum(user_chars), 0) AS userChars, COALESCE(sum(assistant_chars), 0) AS assistantChars,
272
- COALESCE(sum(total_msgs), 0) AS messages,
273
- -- The SAME definition the KPI beside it uses. A plain count() here put a
274
- -- fourth session number on a screen that already had three: nav 11, chips 12,
275
- -- avg depth 40/12, histogram 10.
276
- countIf(c.name != '' OR c.total_msgs > 0) AS sessions
277
- FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params);
278
-
279
- const streakRows = await q(`
280
- SELECT DISTINCT formatDateTime(${TS}, '%Y-%m-%d', {tz:String}) AS day
281
- FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY day`, p);
282
341
  let currentStreak = 0, longestStreak = 0, tempStreak = 1;
283
342
  const today = new Date().toISOString().split('T')[0];
284
343
  for (let i = 1; i < streakRows.length; i++) {
@@ -299,9 +358,6 @@ async function getDashboardStats(opts = {}) {
299
358
  }
300
359
  if (streakRows.length === 0) longestStreak = 0;
301
360
 
302
- const monthEditorRows = await q(`
303
- SELECT formatDateTime(${TS}, '%Y-%m', 'UTC') AS month, source, count() AS count
304
- FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY month, source ORDER BY month`, f.params);
305
361
  const monthEditors = {};
306
362
  const allSources = new Set();
307
363
  for (const r of monthEditorRows) {
@@ -310,21 +366,8 @@ async function getDashboardStats(opts = {}) {
310
366
  allSources.add(r.source);
311
367
  }
312
368
 
313
- const velocityRows = await q(`
314
- SELECT formatDateTime(c.last_updated_at, '%Y-%m', 'UTC') AS month,
315
- avg(total_msgs) AS avgMsgs, avg(input_tokens + output_tokens) AS avgTokens
316
- FROM {{sessions_v}} AS c WHERE c.last_updated_at IS NOT NULL${f.and}
317
- GROUP BY month ORDER BY month`, f.params);
318
-
319
- const modelRows = await q(`
320
- SELECT m.model AS model, count() AS cnt FROM {{messages}} AS m
321
- WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
322
- GROUP BY model`, f.params);
323
369
  const topModels = topN(normalizedModelFreq(modelRows), 10);
324
370
 
325
- const toolRows = await q(`
326
- SELECT tool_name, count() AS cnt FROM {{tool_calls}} AS tc
327
- WHERE 1=1${inSessions(f, 'tc.session_id')} GROUP BY tool_name`, f.params);
328
371
  let totalToolCalls = 0;
329
372
  const toolFreq = {};
330
373
  for (const r of toolRows) { toolFreq[r.tool_name] = Number(r.cnt); totalToolCalls += Number(r.cnt); }
@@ -373,7 +416,13 @@ async function getChats(opts = {}) {
373
416
  FROM {{sessions_v}} AS c WHERE 1=1${f.and}`;
374
417
  const params = { ...f.params };
375
418
  if (opts.named !== false) sql += " AND (c.name != '' OR c.total_msgs > 0)";
376
- sql += ` ORDER BY ${MS} DESC`;
419
+ // Tiebreak on the session key. `ORDER BY <ts> DESC` alone is not a total order —
420
+ // sessions that stopped in the same millisecond tie, and ClickHouse is free to return
421
+ // ties in any order, INCLUDING A DIFFERENT ONE PER QUERY. With LIMIT/OFFSET on top,
422
+ // that means page 2 can repeat or skip a row that page 1 already showed. Found by
423
+ // diffing this endpoint's response between two equivalent sources: identical rows,
424
+ // different order, 129 distinct timestamps across 300 sessions.
425
+ sql += ` ORDER BY ${MS} DESC, c.session_id, c.user_id`;
377
426
  if (opts.limit) { sql += ' LIMIT {limit:UInt64}'; params.limit = opts.limit; }
378
427
  if (opts.offset) { sql += ' OFFSET {offset:UInt64}'; params.offset = opts.offset; }
379
428
  const rows = await q(sql, params);
@@ -388,7 +437,13 @@ async function getChats(opts = {}) {
388
437
  // and left the next one. computePerChatCosts already handles all of them — the orphan
389
438
  // bucket, the source-dominant and global-dominant fallbacks, the char estimate — so call
390
439
  // it instead of growing a second copy of the same logic.
391
- const perChat = await computePerChatCosts(f);
440
+ // Price the fifty rows on the page, not the twelve hundred in the house. This query
441
+ // set was the most expensive thing the dashboard ran — 1,413,624 rows read and 272 MiB
442
+ // per call, three endpoints calling it — and getChats used ~4% of what it asked for.
443
+ const perChat = await computePerChatCosts(f, {
444
+ ids: rows.map((r) => String(r.id)),
445
+ users: rows.map((r) => String(r.user_id)),
446
+ });
392
447
  const costBySession = {};
393
448
  const topModelBySession = {};
394
449
  for (const r of perChat) {
@@ -455,14 +510,15 @@ async function getChat(id) {
455
510
  // Sessions are keyed (session_id, user_id): constrain the row reloads to the
456
511
  // selected rollup's writer, or an owner/no-RLS reader viewing a house where two
457
512
  // members share an adapter-local session_id would see their rows merged.
458
- const messages = await q(`
513
+ const [messages, toolCalls] = await Promise.all([
514
+ q(`
459
515
  SELECT role, text AS content, model, input_tokens, output_tokens
460
516
  FROM {{messages}} WHERE session_id = {cid:String} AND user_id = {uid:String} ORDER BY seq`,
461
- { cid: chat.id, uid: chat.user_id });
462
-
463
- const toolCalls = await q(`
517
+ { cid: chat.id, uid: chat.user_id }),
518
+ q(`
464
519
  SELECT tool_name, args FROM {{tool_calls}} WHERE session_id = {cid:String} AND user_id = {uid:String} ORDER BY idx`,
465
- { cid: chat.id, uid: chat.user_id });
520
+ { cid: chat.id, uid: chat.user_id }),
521
+ ]);
466
522
  const toolCallDetails = toolCalls.map(tc => ({ name: tc.tool_name, args: safeParseJson(tc.args) }));
467
523
 
468
524
  let toolMessages = 0, systemMessages = 0;
@@ -498,15 +554,37 @@ async function getChat(id) {
498
554
 
499
555
  // ── projects ────────────────────────────────────────────────────────────────────
500
556
  async function getProjects(opts = {}) {
557
+ const r = await rooms();
501
558
  const df = filters({
502
559
  hiddenFolders: opts.includeHidden ? [] : opts.hiddenFolders,
503
560
  dateFrom: opts.dateFrom, dateTo: opts.dateTo,
504
561
  });
505
562
 
506
- const perSource = await q(`
563
+ // Four independent reads, one round trip.
564
+ const [perSource, sums, modelRows, toolRows] = await Promise.all([
565
+ q(`
507
566
  SELECT folder, source, count() AS count, min(${MS}) AS first_seen, max(${MS}) AS last_seen
508
567
  FROM {{sessions_v}} AS c WHERE folder != ''${df.and}
509
- GROUP BY folder, source ORDER BY folder, count DESC`, df.params);
568
+ GROUP BY folder, source ORDER BY folder, count DESC`, df.params),
569
+ q(`
570
+ SELECT folder, sum(total_msgs) AS totalMessages,
571
+ sum(input_tokens) AS totalInputTokens, sum(output_tokens) AS totalOutputTokens,
572
+ sum(user_chars) AS totalUserChars, sum(assistant_chars) AS totalAssistantChars,
573
+ sum(cache_read_tokens) AS totalCacheRead, sum(cache_write_tokens) AS totalCacheWrite
574
+ FROM {{sessions_v}} AS c WHERE folder != ''${df.and} GROUP BY folder`, df.params),
575
+ q(r.model_stats ? `
576
+ SELECT m.folder AS folder, m.model AS model, sum(m.msgs) AS cnt FROM {{model_stats}} AS m
577
+ WHERE m.folder != '' AND m.model NOT IN ${EXCLUDED_MODELS}${inSessions(df, 'm.session_id')}
578
+ GROUP BY folder, model` : `
579
+ SELECT m.folder AS folder, m.model AS model, count() AS cnt FROM {{messages}} AS m
580
+ WHERE m.folder != '' AND m.model NOT IN ${EXCLUDED_MODELS}${inSessions(df, 'm.session_id')}
581
+ GROUP BY folder, model`, df.params),
582
+ q(r.tool_stats ? `
583
+ SELECT tc.folder AS folder, tc.tool_name AS tool_name, sum(tc.calls) AS cnt FROM {{tool_stats}} AS tc
584
+ WHERE tc.folder != ''${inSessions(df, 'tc.session_id')} GROUP BY folder, tool_name` : `
585
+ SELECT tc.folder AS folder, tc.tool_name AS tool_name, count() AS cnt FROM {{tool_calls}} AS tc
586
+ WHERE tc.folder != ''${inSessions(df, 'tc.session_id')} GROUP BY folder, tool_name`, df.params),
587
+ ]);
510
588
 
511
589
  const map = {};
512
590
  for (const r of perSource) {
@@ -517,19 +595,9 @@ async function getProjects(opts = {}) {
517
595
  if (r.last_seen != null && r.last_seen > map[r.folder].lastSeen) map[r.folder].lastSeen = r.last_seen;
518
596
  }
519
597
 
520
- const sums = await q(`
521
- SELECT folder, sum(total_msgs) AS totalMessages,
522
- sum(input_tokens) AS totalInputTokens, sum(output_tokens) AS totalOutputTokens,
523
- sum(user_chars) AS totalUserChars, sum(assistant_chars) AS totalAssistantChars,
524
- sum(cache_read_tokens) AS totalCacheRead, sum(cache_write_tokens) AS totalCacheWrite
525
- FROM {{sessions_v}} AS c WHERE folder != ''${df.and} GROUP BY folder`, df.params);
526
598
  const sumByFolder = {};
527
599
  for (const r of sums) sumByFolder[r.folder] = r;
528
600
 
529
- const modelRows = await q(`
530
- SELECT m.folder AS folder, m.model AS model, count() AS cnt FROM {{messages}} AS m
531
- WHERE m.folder != '' AND m.model NOT IN ${EXCLUDED_MODELS}${inSessions(df, 'm.session_id')}
532
- GROUP BY folder, model`, df.params);
533
601
  const modelsByFolder = {};
534
602
  for (const r of modelRows) {
535
603
  const k = normalizeModelName(r.model) || r.model;
@@ -537,9 +605,6 @@ async function getProjects(opts = {}) {
537
605
  modelsByFolder[r.folder][k] = (modelsByFolder[r.folder][k] || 0) + Number(r.cnt);
538
606
  }
539
607
 
540
- const toolRows = await q(`
541
- SELECT tc.folder AS folder, tc.tool_name AS tool_name, count() AS cnt FROM {{tool_calls}} AS tc
542
- WHERE tc.folder != ''${inSessions(df, 'tc.session_id')} GROUP BY folder, tool_name`, df.params);
543
608
  const toolsByFolder = {};
544
609
  for (const r of toolRows) {
545
610
  if (!toolsByFolder[r.folder]) toolsByFolder[r.folder] = {};
@@ -582,7 +647,7 @@ async function getDeepAnalytics(opts = {}) {
582
647
  let sql = `
583
648
  SELECT c.session_id AS id, c.user_id AS user_id, c.total_msgs AS msgs, c.user_chars AS uc, c.assistant_chars AS ac,
584
649
  c.input_tokens AS ti, c.output_tokens AS to_, c.cache_read_tokens AS cr, c.cache_write_tokens AS cw
585
- FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY ${MS} DESC`;
650
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and} ORDER BY ${MS} DESC, c.session_id, c.user_id`;
586
651
  const params = { ...f.params };
587
652
  if (opts.limit) { sql += ' LIMIT {limit:UInt64}'; params.limit = opts.limit; }
588
653
  const rows = await q(sql, params);
@@ -606,18 +671,26 @@ async function getDeepAnalytics(opts = {}) {
606
671
  let topTools = [], topModels = [], totalToolCalls = 0;
607
672
  if (ids.length > 0) {
608
673
  const pairs = { ids, users };
609
- const toolRows = await q(`
674
+ const r = await rooms();
675
+ const [toolRows, modelRows] = await Promise.all([
676
+ q(r.tool_stats ? `
677
+ SELECT tool_name, sum(calls) AS cnt FROM {{tool_stats}}
678
+ WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
679
+ GROUP BY tool_name` : `
610
680
  SELECT tool_name, count() AS cnt FROM {{tool_calls}}
611
681
  WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
612
- GROUP BY tool_name`, pairs);
682
+ GROUP BY tool_name`, pairs),
683
+ q(r.model_stats ? `
684
+ SELECT model, sum(msgs) AS cnt FROM {{model_stats}}
685
+ WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
686
+ AND model NOT IN ${EXCLUDED_MODELS} GROUP BY model` : `
687
+ SELECT model, count() AS cnt FROM {{messages}}
688
+ WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
689
+ AND model NOT IN ${EXCLUDED_MODELS} GROUP BY model`, pairs),
690
+ ]);
613
691
  const toolFreq = {};
614
692
  for (const r of toolRows) { toolFreq[r.tool_name] = Number(r.cnt); totalToolCalls += Number(r.cnt); }
615
693
  topTools = topN(toolFreq, 30);
616
-
617
- const modelRows = await q(`
618
- SELECT model, count() AS cnt FROM {{messages}}
619
- WHERE (session_id, user_id) IN arrayZip({ids:Array(String)}, {users:Array(String)})
620
- AND model NOT IN ${EXCLUDED_MODELS} GROUP BY model`, pairs);
621
694
  topModels = topN(normalizedModelFreq(modelRows), 20);
622
695
  }
623
696
 
@@ -674,13 +747,19 @@ async function getToolCalls(toolName, opts = {}) {
674
747
  const ORPHAN_TOKENS = '(m.input_tokens > 0 OR m.output_tokens > 0 OR m.cache_read_tokens > 0 OR m.cache_write_tokens > 0)';
675
748
 
676
749
  // session_id → dominant model (most frequent across the session's messages).
677
- async function sessionDominantMap(f) {
678
- const rows = await q(`
750
+ async function sessionDominantMap(f, scope = null) {
751
+ // model_stats already holds one row per (session, model) with the message count —
752
+ // argMax over a few thousand pre-aggregated rows instead of a scan of the room.
753
+ const r = await rooms();
754
+ const rows = await q(r.model_stats ? `
755
+ SELECT session_id, user_id, argMax(model, msgs) AS dominant
756
+ FROM {{model_stats}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
757
+ GROUP BY session_id, user_id` : `
679
758
  SELECT session_id, user_id, argMax(model, cnt) AS dominant
680
759
  FROM (SELECT m.session_id AS session_id, m.user_id AS user_id, m.model AS model, count() AS cnt
681
- FROM {{messages}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
760
+ FROM {{messages}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
682
761
  GROUP BY session_id, user_id, model)
683
- GROUP BY session_id, user_id`, f.params);
762
+ GROUP BY session_id, user_id`, scopeParams(f, scope));
684
763
  const map = {};
685
764
  for (const r of rows) map[`${r.session_id}::${r.user_id}`] = r.dominant;
686
765
  return map;
@@ -688,7 +767,11 @@ async function sessionDominantMap(f) {
688
767
 
689
768
  // source → dominant model + global dominant (for sessions with tokens but no model).
690
769
  async function sourceDominantMap(f) {
691
- const rows = await q(`
770
+ const r = await rooms();
771
+ const rows = await q(r.model_stats ? `
772
+ SELECT m.source AS source, m.model AS model, sum(m.msgs) AS cnt
773
+ FROM {{model_stats}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
774
+ GROUP BY source, model` : `
692
775
  SELECT m.source AS source, m.model AS model, count() AS cnt
693
776
  FROM {{messages}} AS m WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
694
777
  GROUP BY source, model`, f.params);
@@ -709,21 +792,50 @@ const CHARS_PER_TOKEN = 4;
709
792
  async function estimateCosts(opts = {}) {
710
793
  const f = filters(opts, { editorLike: true });
711
794
 
712
- const modelTokens = await q(`
795
+ // Five independent reads, one round trip. sourceDominantMap stays lazy below — it is
796
+ // only needed when unmodeled sessions exist, which most passes have none of.
797
+ //
798
+ // The ORPHAN_TOKENS predicate is dropped on the model_stats path on purpose: it
799
+ // filtered individual message rows to any-token ones before summing, and a row with
800
+ // zero tokens contributes zero to every sum — the predicate saved work, never changed
801
+ // a number. The stat rows are already summed, so there is no work left to save.
802
+ const r = await rooms();
803
+ const [modelTokens, orphanRows, dominantMap, charRows, unmodeledRows] = await Promise.all([
804
+ q(r.model_stats ? `
805
+ SELECT m.model AS model, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
806
+ sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
807
+ FROM {{model_stats}} AS m
808
+ WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
809
+ GROUP BY model` : `
713
810
  SELECT m.model AS model, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
714
811
  sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
715
812
  FROM {{messages}} AS m
716
813
  WHERE m.model NOT IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
717
- GROUP BY model`, f.params);
718
-
719
- const orphanRows = await q(`
814
+ GROUP BY model`, f.params),
815
+ q(r.model_stats ? `
816
+ SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
817
+ sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
818
+ FROM {{model_stats}} AS m
819
+ WHERE m.model IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}
820
+ GROUP BY session_id, user_id
821
+ HAVING sum(m.input_tokens) > 0 OR sum(m.output_tokens) > 0 OR sum(m.cache_read_tokens) > 0 OR sum(m.cache_write_tokens) > 0` : `
720
822
  SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS input, sum(m.output_tokens) AS output,
721
823
  sum(m.cache_read_tokens) AS cacheRead, sum(m.cache_write_tokens) AS cacheWrite
722
824
  FROM {{messages}} AS m
723
825
  WHERE m.model IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
724
- GROUP BY session_id, user_id`, f.params);
725
-
726
- const dominantMap = await sessionDominantMap(f);
826
+ GROUP BY session_id, user_id`, f.params),
827
+ sessionDominantMap(f),
828
+ q(`
829
+ SELECT c.session_id AS session_id, c.user_id AS user_id, c.user_chars AS userChars, c.assistant_chars AS asstChars
830
+ FROM {{sessions_v}} AS c
831
+ WHERE notEmpty(c.models) AND c.input_tokens = 0 AND c.output_tokens = 0
832
+ AND (c.user_chars > 0 OR c.assistant_chars > 0)${f.and}`, f.params),
833
+ q(`
834
+ SELECT c.source AS source, c.input_tokens AS input, c.output_tokens AS output,
835
+ c.cache_read_tokens AS cacheRead, c.cache_write_tokens AS cacheWrite
836
+ FROM {{sessions_v}} AS c
837
+ WHERE empty(c.models) AND (c.input_tokens > 0 OR c.output_tokens > 0)${f.and}`, f.params),
838
+ ]);
727
839
 
728
840
  const orphanByModel = {};
729
841
  const addOrphan = (dominant, input, output, cacheRead, cacheWrite) => {
@@ -739,22 +851,12 @@ async function estimateCosts(opts = {}) {
739
851
  }
740
852
 
741
853
  // Sessions that name models but report zero tokens → estimate from chars.
742
- const charRows = await q(`
743
- SELECT c.session_id AS session_id, c.user_id AS user_id, c.user_chars AS userChars, c.assistant_chars AS asstChars
744
- FROM {{sessions_v}} AS c
745
- WHERE notEmpty(c.models) AND c.input_tokens = 0 AND c.output_tokens = 0
746
- AND (c.user_chars > 0 OR c.assistant_chars > 0)${f.and}`, f.params);
747
854
  for (const r of charRows) {
748
855
  const dominant = dominantMap[`${r.session_id}::${r.user_id}`];
749
856
  if (dominant) addOrphan(dominant, Math.round((Number(r.userChars) || 0) / CHARS_PER_TOKEN), Math.round((Number(r.asstChars) || 0) / CHARS_PER_TOKEN), 0, 0);
750
857
  }
751
858
 
752
859
  // Sessions with tokens but NO model at all → the source's dominant model.
753
- const unmodeledRows = await q(`
754
- SELECT c.source AS source, c.input_tokens AS input, c.output_tokens AS output,
755
- c.cache_read_tokens AS cacheRead, c.cache_write_tokens AS cacheWrite
756
- FROM {{sessions_v}} AS c
757
- WHERE empty(c.models) AND (c.input_tokens > 0 OR c.output_tokens > 0)${f.and}`, f.params);
758
860
  if (unmodeledRows.length > 0) {
759
861
  const { sourceDominant, globalDominant } = await sourceDominantMap(f);
760
862
  for (const r of unmodeledRows) {
@@ -792,26 +894,43 @@ async function estimateCosts(opts = {}) {
792
894
 
793
895
  // Per-session cost attribution in a FIXED number of bulk queries (no per-session
794
896
  // round-trips). Mirrors root computePerChatCosts.
795
- async function computePerChatCosts(f) {
796
- const aRows = await q(`
897
+ /**
898
+ * @param {object} f the user's filter
899
+ * @param {object|null} scope {ids, users} — the sessions the caller will actually use.
900
+ * getChats passes the page it just fetched; the cost pages pass nothing, because they
901
+ * are pricing the whole house on purpose.
902
+ */
903
+ async function computePerChatCosts(f, scope = null) {
904
+ const p = scopeParams(f, scope);
905
+ const r = await rooms();
906
+ // Five independent reads, one round trip. This function is called by three endpoints;
907
+ // sequential, it alone cost 5 x RTT per call. On the model_stats path the ORPHAN
908
+ // predicate is dropped/HAVINGed for the reason written in estimateCosts.
909
+ const [aRows, bRows, cRows, dominantMap, sourceDominants] = await Promise.all([
910
+ q(r.model_stats ? `
911
+ SELECT m.session_id AS session_id, m.user_id AS user_id, m.model AS model, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
912
+ sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
913
+ FROM {{model_stats}} AS m
914
+ WHERE m.model NOT IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
915
+ GROUP BY session_id, user_id, model` : `
797
916
  SELECT m.session_id AS session_id, m.user_id AS user_id, m.model AS model, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
798
917
  sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
799
918
  FROM {{messages}} AS m
800
- WHERE m.model NOT IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
801
- GROUP BY session_id, user_id, model`, f.params);
802
- const byChatModel = {};
803
- for (const r of aRows) { const k = `${r.session_id}::${r.user_id}`; (byChatModel[k] = byChatModel[k] || []).push(r); }
804
-
805
- const bRows = await q(`
919
+ WHERE m.model NOT IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
920
+ GROUP BY session_id, user_id, model`, p),
921
+ q(r.model_stats ? `
922
+ SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
923
+ sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
924
+ FROM {{model_stats}} AS m
925
+ WHERE m.model IN ${EXCLUDED_MODELS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
926
+ GROUP BY session_id, user_id
927
+ HAVING sum(m.input_tokens) > 0 OR sum(m.output_tokens) > 0 OR sum(m.cache_read_tokens) > 0 OR sum(m.cache_write_tokens) > 0` : `
806
928
  SELECT m.session_id AS session_id, m.user_id AS user_id, sum(m.input_tokens) AS i, sum(m.output_tokens) AS o,
807
929
  sum(m.cache_read_tokens) AS cr, sum(m.cache_write_tokens) AS cw
808
930
  FROM {{messages}} AS m
809
- WHERE m.model IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}
810
- GROUP BY session_id, user_id`, f.params);
811
- const orphanByChat = {};
812
- for (const r of bRows) orphanByChat[`${r.session_id}::${r.user_id}`] = r;
813
-
814
- const cRows = await q(`
931
+ WHERE m.model IN ${EXCLUDED_MODELS} AND ${ORPHAN_TOKENS}${inSessions(f, 'm.session_id')}${scopeAnd(scope, 'm')}
932
+ GROUP BY session_id, user_id`, p),
933
+ q(`
815
934
  SELECT c.session_id AS id, c.user_id AS user_id, c.source AS source, c.name AS name, c.folder AS folder,
816
935
  toUnixTimestamp64Milli(c.last_updated_at) AS last_updated_at,
817
936
  toUnixTimestamp64Milli(c.created_at) AS created_at,
@@ -820,10 +939,22 @@ async function computePerChatCosts(f) {
820
939
  c.input_tokens AS ti, c.output_tokens AS to_,
821
940
  c.cache_read_tokens AS cr, c.cache_write_tokens AS cw,
822
941
  formatDateTime(${TS}, '%Y-%m', 'UTC') AS month
823
- FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params);
942
+ FROM {{sessions_v}} AS c WHERE 1=1${f.and}${scopeAnd(scope, 'c')}`, p),
943
+ sessionDominantMap(f, scope),
944
+ // NOT scoped, deliberately. This is the house-wide "which model does this editor
945
+ // usually run", used to price sessions that recorded no model at all. Computed from
946
+ // the fifty rows on the page it would answer differently on page 1 than on page 2,
947
+ // and the same session would report a different cost depending on where it was read.
948
+ // Its scope is the user's FILTER, which is what it has always been.
949
+ sourceDominantMap(f),
950
+ ]);
951
+ const { sourceDominant, globalDominant } = sourceDominants;
824
952
 
825
- const dominantMap = await sessionDominantMap(f);
826
- const { sourceDominant, globalDominant } = await sourceDominantMap(f);
953
+ const byChatModel = {};
954
+ for (const r of aRows) { const k = `${r.session_id}::${r.user_id}`; (byChatModel[k] = byChatModel[k] || []).push(r); }
955
+
956
+ const orphanByChat = {};
957
+ for (const r of bRows) orphanByChat[`${r.session_id}::${r.user_id}`] = r;
827
958
 
828
959
  const out = [];
829
960
  for (const c of cRows) {
@@ -868,8 +999,7 @@ async function computePerChatCosts(f) {
868
999
 
869
1000
  async function getCostAnalytics(opts = {}) {
870
1001
  const f = filters(opts, { editorLike: true });
871
- const overall = await estimateCosts(opts);
872
- const perChat = await computePerChatCosts(f);
1002
+ const [overall, perChat] = await Promise.all([estimateCosts(opts), computePerChatCosts(f)]);
873
1003
 
874
1004
  const editorAgg = {}, projectAgg = {}, monthCosts = {};
875
1005
  const sessionCosts = [];
@@ -893,7 +1023,11 @@ async function getCostAnalytics(opts = {}) {
893
1023
  const byEditor = Object.entries(editorAgg).map(([editor, d]) => ({ editor, cost: d.cost, models: d.models.size })).sort((a, b) => b.cost - a.cost);
894
1024
  const byProject = Object.entries(projectAgg).map(([folder, cost]) => ({ folder, name: folder.split('/').pop(), cost })).sort((a, b) => b.cost - a.cost).slice(0, 20);
895
1025
  const monthly = Object.entries(monthCosts).sort((a, b) => a[0].localeCompare(b[0])).map(([month, d]) => ({ month, cost: Math.round(d.cost * 100) / 100, sessions: d.sessions }));
896
- sessionCosts.sort((a, b) => b.cost - a.cost);
1026
+ // Same total-order rule as getChats: a top-50 cut over ties must not depend on the
1027
+ // order the rows happened to arrive in. Every session in a synthetic house costs the
1028
+ // same, and the list changed identity between two equivalent sources without a
1029
+ // secondary key.
1030
+ sessionCosts.sort((a, b) => (b.cost - a.cost) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
897
1031
 
898
1032
  const totalSessions = sessionCosts.length;
899
1033
  const avgPerSession = totalSessions > 0 ? overall.totalCost / totalSessions : 0;
@@ -63,6 +63,8 @@ const selfSnap = selfUpdate.snapshot(__filename);
63
63
  const {
64
64
  resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN,
65
65
  SUPPORTED_SCHEMAS, keyProblem,
66
+ sessionStatsStatements, sessionModelStatsStatements, sessionToolStatsStatements,
67
+ SESSION_STATS, MODEL_STATS, TOOL_STATS,
66
68
  } = require('../house/house');
67
69
  // Rooms plus the house's own record of itself. Every table the template declares, which
68
70
  // is what the column healer and the drift warning have to cover — a column added to
@@ -448,6 +450,36 @@ async function ensureSchema(client) {
448
450
  throw e;
449
451
  }
450
452
  }
453
+ // The precomputed rollups — additive, so they belong here and not in a migration (see
454
+ // migrate.js's rule). Three plain tables plus the refreshable views that fill them;
455
+ // the read layer probes for them at startup and falls back to the inline rollup when
456
+ // they are absent, so a denied CREATE here degrades to slow, never to wrong. Built from
457
+ // the SAME rollup text the read layer would otherwise run, so the two cannot drift.
458
+ const statStmts = [
459
+ ...sessionStatsStatements(rooms), ...sessionModelStatsStatements(rooms), ...sessionToolStatsStatements(rooms),
460
+ ];
461
+ for (const q of statStmts) {
462
+ try {
463
+ await client.command({
464
+ query: q,
465
+ // 26.x defaults the flag on; 25.x wants it stated. Query-scoped, like the
466
+ // text-index flag above.
467
+ clickhouse_settings: { async_insert: 0, allow_experimental_refreshable_materialized_view: 1 },
468
+ });
469
+ } catch (e) {
470
+ const m = e && e.message ? e.message : String(e);
471
+ if (/Not enough privileges|ACCESS_DENIED/i.test(m)) { denied++; continue; }
472
+ if (/ALREADY_EXISTS|already exists/i.test(m)) { continue; }
473
+ throw e;
474
+ }
475
+ }
476
+ // Kick the first population now rather than waiting for the schedule: a refreshable
477
+ // view's first refresh lands at its next interval boundary, and until then the read
478
+ // layer (correctly) ignores the empty table and runs the slow rollup. Best effort —
479
+ // the schedule gets there on its own, this just makes install/update fast at once.
480
+ for (const mv of [`${SESSION_STATS}_mv`, `${MODEL_STATS}_mv`, `${TOOL_STATS}_mv`]) {
481
+ try { await client.command({ query: `SYSTEM REFRESH VIEW ${mv}` }); } catch { /* no grant, or not there */ }
482
+ }
451
483
  if (denied) {
452
484
  console.error(`[memhouse] ${denied} schema statement(s) needed rights you do not hold — continuing with what you can do.`);
453
485
  console.error('[memhouse] creating or replacing a ROOM is the house owner\'s job; adding a missing COLUMN is not.');
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "memhouse",
3
- "version": "0.17.0",
3
+ "version": "0.17.1",
4
4
  "description": "memhouse \u2014 agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, kernel-agency install, and the agentlytics dashboard",
5
5
  "main": "bin/memhouse.js",
6
6
  "bin": {