memhouse 0.4.3 → 0.7.0

package/{mem-house → memhouse}/server/queries.js

@@ -1,12 +1,12 @@
-// queries.js — mem-house read/query layer.
+// queries.js — memhouse read/query layer.
 //
 // Serves the SAME response shapes as the root cache.js (the agentlytics API
-// contract) but computed over the mem-house typed schema — one set of rooms per member,
+// contract) but computed over the memhouse typed schema — one set of rooms per member,
 // `sessions_<m>` / `messages_<m>` / `tool_calls_<m>` (../per-member/schema-member.sql.tpl),
 // plus the session rollup, which is a SAVED QUERY and not an object
 // (../per-member/rooms.js). The dashboard SPA consumes these shapes unchanged.
 //
-// Schema mapping notes (root cache schema → mem-house):
+// Schema mapping notes (root cache schema → memhouse):
 //   chats            → sessions_<m> / the rollup  (id → session_id, bubble_count → total_msgs)
 //   chat_stats       → rollup aggregates          (token/char sums, models array)
 //   messages.content → messages_<m>.text          (cache_read → cache_read_tokens, …)
@@ -16,7 +16,7 @@
 // All reads run with the `final: 1` setting so ReplacingMergeTree collapses
 // duplicate versions at read time (see DESIGN.md data-plane rules).
 
-const { createClient } = require('@clickhouse/client');
+const { createClient, ClickHouseLogLevel } = require('@clickhouse/client');
 const { calculateCost, normalizeModelName } = require('../../pricing');
 const { resolveRooms, READ_SETTINGS } = require('../per-member/rooms');
 
@@ -31,6 +31,9 @@ let _client = null;
 function getClient() {
   if (_client) return _client;
   _client = createClient({
+    // See the note in shipper/ship.js: the driver's ERROR-level dump lands ahead of every
+    // message we print, so a handled refusal reads like a crash. MEMHOUSE_DEBUG=1 restores it.
+    log: { level: process.env.MEMHOUSE_DEBUG ? ClickHouseLogLevel.DEBUG : ClickHouseLogLevel.OFF },
     url: config.url,
     username: config.username,
     password: config.password,
@@ -74,9 +77,10 @@ function applyRooms(sql, r) {
   });
 }
 
-async function q(query, params = {}) {
+async function q(query, params = {}, settings = undefined) {
   const rs = await getClient().query({
     query: applyRooms(query, await rooms()), query_params: params, format: 'JSONEachRow',
+    ...(settings ? { clickhouse_settings: settings } : {}),
   });
   return rs.json();
 }
@@ -145,18 +149,30 @@ function normalizedModelFreq(rows) {
   }
   return freq;
 }
+// The same exclusions as EXCLUDED_MODELS, usable in JS.
+const EXCLUDED_MODEL_SET = new Set(['', '<synthetic>']);
 const topN = (freq, n) => Object.entries(freq).sort((a, b) => b[1] - a[1]).slice(0, n)
   .map(([name, count]) => ({ name, count }));
 
 // ── overview ────────────────────────────────────────────────────────────────────
 async function getOverview(opts = {}) {
   const f = filters(opts);
-  const totalChats = Number((await q1(`SELECT count() AS cnt FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params)).cnt);
-
-  // Root parity: without a folder filter the editor breakdown is global.
-  const editors = opts.folder
-    ? await q(`SELECT source, count() AS count FROM {{sessions_v}} AS c WHERE 1=1${f.and} GROUP BY source ORDER BY count DESC`, f.params)
-    : await q('SELECT source, count() AS count FROM {{sessions_v}} GROUP BY source ORDER BY count DESC');
+  // Same definition as getChats(). The nav bar read this and the Sessions page read
+  // getChats, so a house with one placeholder session showed "9 sessions" beside
+  // "8 sessions" in the same header, and the depth histogram summed to 7. A session row
+  // with no name and no messages is a placeholder the shipper keeps to absorb the
+  // incremental skip; it is not a conversation, and only one of the two counters knew.
+  const totalChats = Number((await q1(
+    `SELECT count() AS cnt FROM {{sessions_v}} AS c WHERE 1=1${f.and} AND (c.name != '' OR c.total_msgs > 0)`,
+    f.params)).cnt);
+
+  // Root parity WAS: without a folder filter the editor breakdown is global — which put
+  // "34 + 3" chips under a header reading "3 sessions" when an editor filter was applied,
+  // and 12 under a header of 11 with no filter at all. Parity with a wrong number is not a
+  // feature.
+  const editors = await q(
+    `SELECT source, count() AS count FROM {{sessions_v}} AS c
+     WHERE (c.name != '' OR c.total_msgs > 0)${f.and} GROUP BY source ORDER BY count DESC`, f.params);
 
   const modes = await q(`SELECT mode, count() AS count FROM {{sessions_v}} AS c WHERE mode != ''${f.and} GROUP BY mode`, f.params);
   const byMode = {};
@@ -253,7 +269,11 @@ async function getDashboardStats(opts = {}) {
     SELECT COALESCE(sum(input_tokens), 0) AS input, COALESCE(sum(output_tokens), 0) AS output,
            COALESCE(sum(cache_read_tokens), 0) AS cacheRead, COALESCE(sum(cache_write_tokens), 0) AS cacheWrite,
            COALESCE(sum(user_chars), 0) AS userChars, COALESCE(sum(assistant_chars), 0) AS assistantChars,
-           count() AS sessions
+           COALESCE(sum(total_msgs), 0) AS messages,
+           -- The SAME definition the KPI beside it uses. A plain count() here put a
+           -- fourth session number on a screen that already had three: nav 11, chips 12,
+           -- avg depth 40/12, histogram 10.
+           countIf(c.name != '' OR c.total_msgs > 0) AS sessions
     FROM {{sessions_v}} AS c WHERE 1=1${f.and}`, f.params);
 
   const streakRows = await q(`
@@ -323,7 +343,7 @@ async function getDashboardStats(opts = {}) {
     tokens: {
       input: inputTokens, output: outputTokens, cacheRead: Number(tokenRow.cacheRead), cacheWrite: Number(tokenRow.cacheWrite),
       userChars: Number(tokenRow.userChars), assistantChars: Number(tokenRow.assistantChars),
-      sessions: Number(tokenRow.sessions), estimated: tokensEstimated,
+      sessions: Number(tokenRow.sessions), messages: Number(tokenRow.messages), estimated: tokensEstimated,
     },
     streaks: { current: currentStreak, longest: longestStreak, totalDays: streakRows.length },
     monthlyTrend: { months: Object.keys(monthEditors).sort(), sources: [...allSources], data: monthEditors },
@@ -359,26 +379,30 @@ async function getChats(opts = {}) {
   const rows = await q(sql, params);
   if (rows.length === 0) return [];
 
-  // Top model per rollup = most frequent model across THAT WRITER's messages
-  // (session_id alone would blend colliding sessions across members).
-  const ids = rows.map(r => r.id);
-  const tmRows = await q(`
-    SELECT session_id, user_id, argMax(model, cnt) AS top_model
-    FROM (SELECT session_id, user_id, model, count() AS cnt FROM {{messages}}
-          WHERE session_id IN {ids:Array(String)} AND model NOT IN ${EXCLUDED_MODELS}
-          GROUP BY session_id, user_id, model)
-    GROUP BY session_id, user_id`, { ids });
+  // ONE cost implementation, shared with the Costs page.
+  //
+  // getChats used to compute its own, and the two disagreed three times running: first the
+  // whole session priced at its dominant model ($10.02 against $150), then sessions whose
+  // rows carry no model at all ($0 against $10.50), then sessions with zero tokens but
+  // non-zero chars ($0 against $0.033). Each round fixed the case that had been measured
+  // and left the next one. computePerChatCosts already handles all of them — the orphan
+  // bucket, the source-dominant and global-dominant fallbacks, the char estimate — so call
+  // it instead of growing a second copy of the same logic.
+  const perChat = await computePerChatCosts(f);
+  const costBySession = {};
   const topModelBySession = {};
-  for (const r of tmRows) topModelBySession[`${r.session_id}::${r.user_id}`] = r.top_model;
+  for (const r of perChat) {
+    // computePerChatCosts already emits the composite key as `id`.
+    const k = r.id;
+    costBySession[k] = { total: r.totalCost, unpriced: !!r.hasUnpriced };
+    if (r.byModel[0]) topModelBySession[k] = r.byModel[0].model;
+  }
 
   return rows.map(r => {
-    const topModel = topModelBySession[`${r.id}::${r.user_id}`] || null;
-    let inTok = Number(r._inTok) || 0, outTok = Number(r._outTok) || 0;
-    if (inTok === 0 && outTok === 0 && ((r._uChars || 0) > 0 || (r._aChars || 0) > 0)) {
-      inTok = Math.round((r._uChars || 0) / 4);
-      outTok = Math.round((r._aChars || 0) / 4);
-    }
-    const cost = topModel ? (calculateCost(topModel, inTok, outTok, Number(r._cacheR) || 0, Number(r._cacheW) || 0) || 0) : 0;
+    const k = `${r.id}::${r.user_id}`;
+    const topModel = topModelBySession[k] || null;
+    const summed = costBySession[k];
+    const cost = summed ? summed.total : 0;
     return {
       // Composite API id: rollups are keyed (session_id, user_id), so the id a
       // client clicks must pin BOTH — otherwise opening one of two colliding
@@ -388,6 +412,9 @@ async function getChats(opts = {}) {
       source: r.source, name: r.name, mode: r.mode, folder: r.folder,
       createdAt: r.created_at, lastUpdatedAt: r.last_updated_at,
       encrypted: false, bubbleCount: Number(r.bubble_count), topModel, cost,
+      // Surfaced, not just computed. A session containing an unpriced model reports a cost
+      // that silently omits part of itself; the caller can now say so.
+      costPartial: !!(summed && summed.unpriced),
     };
   });
 }
@@ -643,7 +670,7 @@ async function getToolCalls(toolName, opts = {}) {
 // ── cost estimation ─────────────────────────────────────────────────────────────
 // Messages with model '' or '<synthetic>' are "orphans": their tokens are
 // attributed to the session's dominant model (root cache.js treats model IS NULL
-// the same way; '' is the mem-house null and synthetic models have no pricing).
+// the same way; '' is the memhouse null and synthetic models have no pricing).
 const ORPHAN_TOKENS = '(m.input_tokens > 0 OR m.output_tokens > 0 OR m.cache_read_tokens > 0 OR m.cache_write_tokens > 0)';
 
 // session_id → dominant model (most frequent across the session's messages).
@@ -821,15 +848,19 @@ async function computePerChatCosts(f) {
       if (srcDom) add(srcDom, c.ti, c.to_, c.cr, c.cw);
     }
     let totalCost = 0; const byModel = [];
+    // A model with no price contributes nothing to totalCost, so the number a session
+    // reports silently omits part of itself. Carry the fact alongside the number.
+    let hasUnpriced = false;
     for (const [model, tok] of Object.entries(tokenMap)) {
       const cost = calculateCost(model, tok.input, tok.output, tok.cacheRead, tok.cacheWrite);
       if (cost !== null) { totalCost += cost; byModel.push({ model, cost }); }
+      else hasUnpriced = true;
     }
     byModel.sort((a, b) => b.cost - a.cost);
     out.push({
       id: key, user: c.user_id, source: c.source, name: c.name, folder: c.folder,
       last_updated_at: c.last_updated_at, created_at: c.created_at,
-      msgs: Number(c.msgs), month: c.month, totalCost, byModel,
+      msgs: Number(c.msgs), month: c.month, totalCost, byModel, hasUnpriced,
     });
   }
   return out;
@@ -881,10 +912,22 @@ async function getCostAnalytics(opts = {}) {
 }
 
 // ── raw SQL / schema (SqlViewer) ────────────────────────────────────────────────
+// Bounded at the SERVER, not in JS. `q()` buffers every row into memory, so
+// `SELECT number FROM numbers(2000000000)` killed the whole dashboard process —
+// "FATAL ERROR: Ineffective mark-compacts near heap limit", unauthenticated, on the bound
+// port. Capping after the rows arrive is too late; ClickHouse has to stop sending.
+const RAW_MAX_ROWS = 10000;
+const RAW_MAX_BYTES = 64 * 1024 * 1024;
 async function rawQuery(sql) {
-  const rows = await q(sql);
+  const rows = await q(sql, {}, {
+    max_result_rows: RAW_MAX_ROWS,
+    max_result_bytes: RAW_MAX_BYTES,
+    // Cut the result off rather than failing a query that was legitimately large.
+    result_overflow_mode: 'break',
+    max_execution_time: 30,
+  });
   const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
-  return { columns, rows, count: rows.length };
+  return { columns, rows, count: rows.length, truncated: rows.length >= RAW_MAX_ROWS };
 }
 
 async function schema() {
@@ -925,4 +968,7 @@ module.exports = {
   getProjects, getDeepAnalytics, getToolCalls,
   estimateCosts, getCostAnalytics,
   rawQuery, schema,
+  // Used only by the server's own guard to learn which database names are real. Not
+  // reachable from any route.
+  rawQueryUnguarded: (sql) => q(sql),
 };