memhouse 0.0.1 → 0.3.0

package/mem-house/server/server.js

@@ -0,0 +1,228 @@
+// mem-house server — the dashboard/API entrypoint.
+//
+// Speaks the agentlytics REST contract (same routes, same response shapes) over the
+// mem-house typed schema, so the UNMODIFIED agentlytics React SPA is the dashboard:
+// queries.js computes every response; this file is routing + config + stubs. Serves
+// the built SPA from the repo root's public/.
+//
+//   node mem-house/server/server.js        (env: MEMHOUSE_* per DESIGN.md; port 4640)
+
+const express = require('express');
+const path = require('path');
+const fs = require('fs');
+const os = require('os');
+const qy = require('./queries');
+
+const PORT = parseInt(process.env.MEMHOUSE_PORT || '4640', 10);
+// Localhost by default: the dashboard exposes full transcripts and a SQL console
+// with no auth. Remote access is an explicit opt-in (MEMHOUSE_HOST=0.0.0.0).
+const HOST = process.env.MEMHOUSE_HOST || '127.0.0.1';
+const PUBLIC_DIR = path.join(__dirname, '..', '..', 'public');
+const CONFIG_PATH = path.join(os.homedir(), '.memhouse', 'config.json');
+
+const app = express();
+app.use(express.json());
+if (fs.existsSync(PUBLIC_DIR)) app.use(express.static(PUBLIC_DIR));
+
+function readConfig() {
+  try { return JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf-8')); } catch { return {}; }
+}
+function writeConfig(config) {
+  fs.mkdirSync(path.dirname(CONFIG_PATH), { recursive: true });
+  fs.writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2));
+}
+const hiddenFolders = () => readConfig().hiddenProjects || [];
+
+function parseDateOpts(query) {
+  const opts = {};
+  if (query.dateFrom) opts.dateFrom = parseInt(query.dateFrom) || null;
+  if (query.dateTo) opts.dateTo = parseInt(query.dateTo) || null;
+  return opts;
+}
+
+// Route helper: async handler with the uniform 500 shape.
+const route = (fn) => async (req, res) => {
+  try { await fn(req, res); } catch (err) { res.status(500).json({ error: err.message }); }
+};
+
+app.get('/api/ping', (req, res) => res.json({ app: 'agentlytics', pid: process.pid }));
+app.get('/api/mode', (req, res) => res.json({ mode: 'local' }));
+
+app.get('/api/overview', route(async (req, res) => {
+  res.json(await qy.getOverview({ editor: req.query.editor || null, ...parseDateOpts(req.query), hiddenFolders: hiddenFolders() }));
+}));
+
+app.get('/api/daily-activity', route(async (req, res) => {
+  res.json(await qy.getDailyActivity({ editor: req.query.editor || null, ...parseDateOpts(req.query), hiddenFolders: hiddenFolders() }));
+}));
+
+app.get('/api/dashboard-stats', route(async (req, res) => {
+  res.json(await qy.getDashboardStats({ editor: req.query.editor || null, ...parseDateOpts(req.query), hiddenFolders: hiddenFolders() }));
+}));
+
+app.get('/api/chats', route(async (req, res) => {
+  const opts = {
+    editor: req.query.editor || null,
+    folder: req.query.folder || null,
+    named: req.query.named !== 'false',
+    limit: req.query.limit ? parseInt(req.query.limit) : 200,
+    offset: req.query.offset ? parseInt(req.query.offset) : 0,
+    ...parseDateOpts(req.query),
+    hiddenFolders: hiddenFolders(),
+  };
+  const [total, chats] = await Promise.all([qy.countChats(opts), qy.getChats(opts)]);
+  res.json({ total, chats });
+}));
+
+app.get('/api/chats/:id', route(async (req, res) => {
+  const result = await qy.getChat(req.params.id);
+  if (!result) return res.status(404).json({ error: 'Chat not found' });
+  res.json(result);
+}));
+
+app.get('/api/chats/:id/markdown', route(async (req, res) => {
+  const result = await qy.getChat(req.params.id);
+  if (!result) return res.status(404).json({ error: 'Chat not found' });
+  const lines = [];
+  const title = result.name || 'Untitled Session';
+  lines.push(`# ${title}\n`);
+  const meta = [];
+  if (result.source) meta.push(`**Editor:** ${result.source}`);
+  if (result.mode) meta.push(`**Mode:** ${result.mode}`);
+  if (result.folder) meta.push(`**Project:** ${result.folder}`);
+  if (result.createdAt) meta.push(`**Created:** ${new Date(result.createdAt).toISOString()}`);
+  if (result.lastUpdatedAt) meta.push(`**Updated:** ${new Date(result.lastUpdatedAt).toISOString()}`);
+  if (result.stats) {
+    meta.push(`**Messages:** ${result.stats.totalMessages}`);
+    if (result.stats.totalInputTokens) meta.push(`**Input Tokens:** ${result.stats.totalInputTokens}`);
+    if (result.stats.totalOutputTokens) meta.push(`**Output Tokens:** ${result.stats.totalOutputTokens}`);
+    const models = [...new Set(result.stats.models || [])];
+    if (models.length > 0) meta.push(`**Models:** ${models.join(', ')}`);
+  }
+  if (meta.length > 0) lines.push(meta.join('  \n') + '\n');
+  lines.push('---\n');
+  for (const msg of result.messages) {
+    const label = msg.role === 'user' ? '## User' : msg.role === 'assistant' ? '## Assistant' : `## ${msg.role.charAt(0).toUpperCase() + msg.role.slice(1)}`;
+    const modelTag = msg.model ? ` *(${msg.model})*` : '';
+    lines.push(`${label}${modelTag}\n`);
+    lines.push(msg.content + '\n');
+  }
+  const filename = title.replace(/[^a-zA-Z0-9_-]/g, '_').substring(0, 80) + '.md';
+  res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
+  res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+  res.send(lines.join('\n'));
+}));
+
+app.get('/api/projects', route(async (req, res) => {
+  res.json(await qy.getProjects({ ...parseDateOpts(req.query), hiddenFolders: hiddenFolders() }));
+}));
+
+app.get('/api/all-projects', route(async (req, res) => {
+  res.json(await qy.getProjects({ ...parseDateOpts(req.query), includeHidden: true }));
+}));
+
+app.get('/api/deep-analytics', route(async (req, res) => {
+  res.json(await qy.getDeepAnalytics({
+    editor: req.query.editor || null,
+    folder: req.query.folder || null,
+    limit: Math.min(parseInt(req.query.limit) || 500, 5000),
+    ...parseDateOpts(req.query),
+    hiddenFolders: hiddenFolders(),
+  }));
+}));
+
+app.get('/api/costs', route(async (req, res) => {
+  res.json(await qy.estimateCosts({
+    editor: req.query.editor || null,
+    folder: req.query.folder || null,
+    chatId: req.query.chatId || null,
+    ...parseDateOpts(req.query),
+    hiddenFolders: hiddenFolders(),
+  }));
+}));
+
+app.get('/api/cost-analytics', route(async (req, res) => {
+  res.json(await qy.getCostAnalytics({ editor: req.query.editor || null, ...parseDateOpts(req.query), hiddenFolders: hiddenFolders() }));
+}));
+
+app.get('/api/tool-calls', route(async (req, res) => {
+  const name = req.query.name;
+  if (!name) return res.status(400).json({ error: 'name query param required' });
+  // Same filter set as the analytics the drill-down is opened from.
+  res.json(await qy.getToolCalls(name, {
+    limit: Math.min(parseInt(req.query.limit) || 200, 1000),
+    folder: req.query.folder || null,
+    editor: req.query.editor || null,
+    ...parseDateOpts(req.query),
+    hiddenFolders: hiddenFolders(),
+  }));
+}));
+
+app.post('/api/query', route(async (req, res) => {
+  const { sql } = req.body;
+  if (!sql || typeof sql !== 'string') return res.status(400).json({ error: 'sql string required' });
+  const first = sql.trim().replace(/^--.*$/gm, '').trim().split(/\s+/)[0].toUpperCase();
+  if (!['SELECT', 'WITH', 'EXPLAIN', 'DESCRIBE', 'SHOW'].includes(first)) {
+    return res.status(403).json({ error: 'Only SELECT queries are allowed' });
+  }
+  try { res.json(await qy.rawQuery(sql)); }
+  catch (err) { res.status(400).json({ error: err.message }); }
+}));
+
+app.get('/api/schema', route(async (req, res) => res.json(await qy.schema())));
+
+app.get('/api/config', (req, res) => res.json(readConfig()));
+app.put('/api/config', route(async (req, res) => {
+  const config = readConfig();
+  Object.assign(config, req.body);
+  writeConfig(config);
+  res.json(config);
+}));
+
+// Real refetch: run one full shipper pass, streaming SSE like the root contract.
+app.get('/api/refetch', async (req, res) => {
+  res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' });
+  try {
+    const { runShip } = require('../shipper/ship');
+    res.write(`data: ${JSON.stringify({ scanned: 0, analyzed: 0, skipped: 0, total: 0 })}\n\n`);
+    const r = await runShip(qy.getClient(), { full: true });
+    res.write(`data: ${JSON.stringify({ done: true, total: r.sessions + r.skipped, analyzed: r.sessions })}\n\n`);
+  } catch (err) {
+    res.write(`data: ${JSON.stringify({ error: err.message })}\n\n`);
+  }
+  res.end();
+});
+
+// Not applicable to mem-house (dashboard renders these pages empty/absent).
+app.get('/api/usage', (req, res) => res.json([]));
+app.get('/api/artifacts', (req, res) => res.json([]));
+app.get('/api/artifact-content', (req, res) => res.status(404).json({ error: 'not available in mem-house' }));
+app.get('/api/mcps', (req, res) => res.json({
+  servers: [], toolCalls: [], matchedTools: {}, topSessions: [], projectMcps: [],
+  summary: { totalServers: 0, totalToolCalls: 0, uniqueTools: 0, sessionsWithTools: 0, editorsWithServers: [] },
+}));
+app.get('/api/gsd/projects', (req, res) => res.json([]));
+app.get('/api/gsd/phases', (req, res) => res.json([]));
+app.get('/api/gsd/plan', (req, res) => res.status(404).json({ error: 'not available in mem-house' }));
+app.get('/api/gsd/overview', (req, res) => res.json({ totalProjects: 0, totalPhases: 0, completedPhases: 0, activePhases: [], executingPhases: 0, plannedPhases: 0 }));
+app.get('/api/gsd/config', (req, res) => res.json(null));
+app.get('/api/gsd/phase-tokens', (req, res) => res.json([]));
+app.get('/api/gsd/file', (req, res) => res.json({ content: null }));
+app.get('/api/share-image', (req, res) => res.status(501).json({ error: 'not available in mem-house' }));
+app.get('/api/check-ai', (req, res) => res.status(501).json({ error: 'not available in mem-house' }));
+
+// SPA fallback
+app.get('*', (req, res) => {
+  const index = path.join(PUBLIC_DIR, 'index.html');
+  if (fs.existsSync(index)) res.sendFile(index);
+  else res.status(503).json({ error: 'UI not built — run: cd ui && npm install && npm run build' });
+});
+
+if (require.main === module) {
+  app.listen(PORT, HOST, () => {
+    console.log(`[mem-house] dashboard → http://localhost:${PORT} (house '${qy.config.database}' @ ${qy.config.url.replace(/\/\/.*@/, '//')})`
+      + (HOST !== '127.0.0.1' ? ` [bound to ${HOST} — remotely reachable]` : ''));
+  });
+}
+
+module.exports = app;

package/mem-house/shipper/ship.js

@@ -0,0 +1,402 @@
+#!/usr/bin/env node
+// mem-house SHIPPER — parse-on-client (bet #1 in ../DESIGN.md). Runs the 17 editor
+// adapters locally and ships TYPED rows into the house (sessions / messages /
+// tool_calls, ../schema.sql). The typed evolution of agency/ingest.js: same host id,
+// same ts interpolation, same batching discipline — but rows land in physical
+// columns instead of a raw JSON blob.
+//
+// Data-plane rules (binding, see DESIGN.md):
+//   - JSONEachRow, batches <= 2000 rows, async_insert=0 on EVERY insert — the house
+//     stamps user_id MATERIALIZED currentUser(), which is empty during async flushes.
+//   - DateTime64 values travel as 'YYYY-MM-DD HH:MM:SS.mmm' UTC strings (plain
+//     format; ISO 'T'/'Z' forms parse unreliably under JSONEachRow). Nullable → null.
+//   - Int64-bound values are integer-coerced (some adapters emit fractional ms).
+//   - Re-shipping is always safe: ReplacingMergeTree(ingested_at) keyed
+//     (session_id, user_id, seq) collapses to latest-wins at FINAL.
+//
+// CLI:  node ship.js                one incremental pass
+//       node ship.js --loop [sec]   repeat every sec seconds (default 300)
+//       node ship.js --full         ignore the incremental skip (re-ship everything)
+//       node ship.js --ensure-schema  apply ../schema.sql and exit
+//       node ship.js --stats        per-source counts from sessions_v and exit
+//
+// Env (DESIGN.md contract): MEMHOUSE_URL / MEMHOUSE_USER / MEMHOUSE_PASSWORD /
+// MEMHOUSE_DB — defaults http://localhost:8123 / memhouse_root / '' / memhouse.
+
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { createClient } = require('@clickhouse/client');
+const { getAllChats, getMessages, resetCaches } = require('../../editors');
+
+const BATCH_ROWS = 2000;   // insert batch ceiling (binding)
+const TEXT_MAX = 50000;    // messages.text truncation
+const ARGS_MAX = 20000;    // tool_calls.args truncation
+
+// Stable host id: <shorthostname>-<8hex sha256(machine seed)> (memory-house convention).
+function hostId() {
+  const short = (os.hostname() || 'host').split('.')[0];
+  const seed = `${os.hostname()}|${os.platform()}|${os.arch()}`;
+  const h = crypto.createHash('sha256').update(seed).digest('hex').slice(0, 8);
+  return `${short}-${h}`;
+}
+
+// UInt64 (as decimal string) = first 8 bytes big-endian of sha256 over the row's
+// content fields. Decimal string survives JSONEachRow → UInt64 without precision loss.
+function lineHash(obj) {
+  return crypto.createHash('sha256').update(JSON.stringify(obj)).digest().readBigUInt64BE(0).toString();
+}
+
+// DateTime64(3) wire format: 'YYYY-MM-DD HH:MM:SS.mmm' in UTC (see header note).
+function chTs(ms) {
+  const d = new Date(ms);
+  const p = (n, w = 2) => String(n).padStart(w, '0');
+  return `${d.getUTCFullYear()}-${p(d.getUTCMonth() + 1)}-${p(d.getUTCDate())} ` +
+    `${p(d.getUTCHours())}:${p(d.getUTCMinutes())}:${p(d.getUTCSeconds())}.${p(d.getUTCMilliseconds(), 3)}`;
+}
+
+// Per-message timestamp interpolated across [createdAt, lastUpdatedAt] — adapters
+// don't expose per-message times, so we synthesize monotonic-by-seq ones that make
+// sessions_v started/ended line up with the session bounds (same approximation as
+// agency/ingest.js; documented in DESIGN.md).
+function messageTs(chat, seq, total) {
+  const start = chat.createdAt || chat.lastUpdatedAt || Date.now();
+  const end = chat.lastUpdatedAt || chat.createdAt || start;
+  if (total <= 1) return chTs(start);
+  return chTs(start + Math.round((end - start) * (seq / (total - 1))));
+}
+
+// UInt64-bound coercion: integers only, never negative, garbage → 0.
+function toInt(v) {
+  const n = Math.trunc(Number(v));
+  return Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+function makeClient() {
+  return createClient({
+    url: process.env.MEMHOUSE_URL || 'http://localhost:8123',
+    username: process.env.MEMHOUSE_USER || 'memhouse_root',
+    password: process.env.MEMHOUSE_PASSWORD || '',
+    database: process.env.MEMHOUSE_DB || 'memhouse',
+    request_timeout: 300000, // full re-ships move tens of MB; don't cut inserts short
+    clickhouse_settings: {
+      // Int64/UInt64 back as JSON numbers — our values (counts, tokens) are < 2^53.
+      output_format_json_quote_64bit_integers: 0,
+      // With a 300s request_timeout the client warns unless progress headers
+      // keep long requests alive through proxies/load balancers — enable them
+      // (also silences the startup WARN on every ship/install run).
+      // The interval MUST sit below the smallest idle timeout in the path:
+      // 60s is the default for AWS ALB and nginx proxy_read_timeout, so a
+      // longer interval emits nothing before the socket is dropped and the
+      // setting only silences the warning. 30s leaves headroom for both.
+      send_progress_in_http_headers: 1,
+      http_headers_progress_interval_ms: '30000',
+    },
+  });
+}
+
+// Apply ../schema.sql statement by statement. Comments are stripped BEFORE the ';'
+// split — schema comments legitimately contain semicolons.
+async function ensureSchema(client) {
+  const sql = fs.readFileSync(path.join(__dirname, '..', 'schema.sql'), 'utf-8');
+  const stripped = sql.split('\n').map((l) => l.replace(/--.*$/, '')).join('\n');
+  const stmts = stripped.split(';').map((s) => s.trim()).filter(Boolean);
+  for (const q of stmts) {
+    await client.command({
+      query: q,
+      // allow_experimental_full_text_index: on 25.x the messages text indexes
+      // are gated behind this flag (SUPPORT_IS_DISABLED without it); 26.x+
+      // accepts it as a no-op. Query-scoped, so no server config or admin
+      // rights are needed. Verified on 25.11 and 26.7.
+      clickhouse_settings: { async_insert: 0, allow_experimental_full_text_index: 1 },
+    });
+  }
+  return stmts.length;
+}
+
+// Incremental state: what the house already holds, keyed by session_id. We compare
+// against extra.bubbleCount (the adapter's own cheap count, stored at ship time) —
+// NOT message_count — because parsed-message count and bubbleCount are different
+// units in several adapters (claude folds subagents, codex reports 0), and skipping
+// must be decidable WITHOUT calling getMessages on every chat.
+async function loadExisting(client) {
+  const rs = await client.query({
+    query: 'SELECT session_id, last_updated_at, message_count, extra FROM sessions FINAL WHERE user_id = currentUser()',
+    format: 'JSONEachRow',
+  });
+  // Actual message rows per session: an interrupted re-ship (crash between the
+  // clear-DELETE and the inserts, or mid-flush on a large transcript) leaves a
+  // fresh-looking session row with a missing or PARTIAL transcript — the skip
+  // must compare the real row count against the recorded message_count, not
+  // merely check that some row exists.
+  const mr = await client.query({
+    query: 'SELECT session_id, count() AS n FROM messages FINAL WHERE user_id = currentUser() GROUP BY session_id',
+    format: 'JSONEachRow',
+  });
+  const msgCounts = new Map();
+  for (const r of await mr.json()) msgCounts.set(r.session_id, toInt(r.n));
+  const map = new Map();
+  for (const r of await rs.json()) {
+    // DateTime64 comes back as 'YYYY-MM-DD HH:MM:SS.mmm' — re-parse as UTC.
+    const ms = r.last_updated_at ? Date.parse(r.last_updated_at.replace(' ', 'T') + 'Z') : null;
+    const bc = toInt(r.extra && r.extra.bubbleCount);
+    const count = toInt(r.message_count);
+    map.set(r.session_id, { ms, count, bc, intact: (msgCounts.get(r.session_id) || 0) === count });
+  }
+  return map;
+}
+
+// Regex fallback for adapters that render tool calls into text but don't expose
+// m._toolCalls: matches '[tool-call: Name(' and '[tool-call: Name]' forms.
+const TOOL_CALL_RE = /\[tool-call: ([^(\]]+)/g;
+
+// Build the typed rows for one chat. Returns null when the chat is unreadable
+// (adapter threw — e.g. a partially-written or locked session file): the caller
+// must write NOTHING for it, so the next incremental pass retries. A chat that
+// parses to zero messages still gets its session row (deliberate: the stored row
+// absorbs the incremental skip for stable-empty chats).
+function rowsForChat(chat, host) {
+  let messages;
+  try { messages = getMessages(chat) || []; } catch { return null; }
+
+  const source = chat.source;
+  // Canonical globally-unique session id: '<source>:<adapter-local id>'.
+  // composerId is only unique WITHIN one editor; two editors emitting the same
+  // id must never collide in keys, incremental state, deletes, or the API.
+  // Sources are colon-free, so the prefix parses back unambiguously.
+  const id = `${source}:${String(chat.composerId)}`;
+  const folder = chat.folder || '';
+  const project = folder ? path.basename(folder) : '';
+  const total = messages.length;
+
+  const session = {
+    session_id: id,
+    source,
+    host,
+    name: chat.name || '',
+    mode: chat.mode || '',
+    folder,
+    project,
+    git_branch: chat._gitBranch || '',
+    created_at: chat.createdAt ? chTs(chat.createdAt) : null,
+    last_updated_at: chat.lastUpdatedAt ? chTs(chat.lastUpdatedAt) : null,
+    message_count: total,
+    path: chat._fullPath || chat._dbPath || id,
+    extra: { bubbleCount: chat.bubbleCount || 0 },
+  };
+
+  const msgRows = [];
+  const toolRows = [];
+  let idx = 0; // session-wide tool-call index
+  for (let seq = 0; seq < total; seq++) {
+    const m = messages[seq];
+    const raw = typeof m.content === 'string' ? m.content : JSON.stringify(m.content) ?? '';
+    const text = raw.length > TEXT_MAX ? raw.slice(0, TEXT_MAX) : raw;
+    const ts = messageTs(chat, seq, total);
+    const row = {
+      session_id: id,
+      seq,
+      source,
+      host,
+      ts,
+      role: m.role || '',
+      model: m._model || '',
+      input_tokens: toInt(m._inputTokens),
+      output_tokens: toInt(m._outputTokens),
+      cache_read_tokens: toInt(m._cacheRead),
+      cache_write_tokens: toInt(m._cacheWrite),
+      text,
+      project,
+      folder,
+      is_subagent: text.startsWith('[subagent]'),
+      extra: {},
+    };
+    // Hash the content fields only (not ts — it is synthesized and may drift).
+    row.line_hash = lineHash({
+      session_id: id, seq, role: row.role, model: row.model,
+      input_tokens: row.input_tokens, output_tokens: row.output_tokens,
+      cache_read_tokens: row.cache_read_tokens, cache_write_tokens: row.cache_write_tokens,
+      text,
+    });
+    msgRows.push(row);
+
+    if (m.role !== 'assistant') continue;
+    let calls;
+    if (Array.isArray(m._toolCalls) && m._toolCalls.length) {
+      calls = m._toolCalls.map((tc) => ({ name: String(tc.name || 'unknown'), args: tc.args || {} }));
+    } else {
+      calls = [];
+      for (const match of text.matchAll(TOOL_CALL_RE)) {
+        calls.push({ name: match[1].trim(), args: {} });
+      }
+    }
+    for (const tc of calls) {
+      const args = JSON.stringify(tc.args);
+      toolRows.push({
+        session_id: id,
+        seq,
+        idx: idx++,
+        source,
+        host,
+        tool_name: tc.name,
+        args: args.length > ARGS_MAX ? args.slice(0, ARGS_MAX) : args,
+        ts,
+        project,
+        folder,
+      });
+    }
+  }
+  return { session, msgRows, toolRows };
+}
+
+// One shipping pass. Incremental unless opts.full: a chat is skipped when the house
+// already has it at least as fresh (last_updated_at) and at least as large
+// (extra.bubbleCount) — both readable without parsing the chat.
+async function runShip(client, opts = {}) {
+  const { full = false } = opts;
+  const host = hostId();
+  // Always load what the house holds — even with --full. The skip decision uses it
+  // only in incremental mode, but re-shipping a KNOWN session must clear its old
+  // message/tool rows first (a shrunken re-parse leaves stale higher-seq rows
+  // otherwise: ReplacingMergeTree collapses same-key rows only).
+  const existing = await loadExisting(client);
+  const chats = getAllChats();
+
+  const batches = { sessions: [], messages: [], tool_calls: [] };
+  const flush = async (table) => {
+    if (!batches[table].length) return;
+    await client.insert({
+      table,
+      values: batches[table],
+      format: 'JSONEachRow',
+      clickhouse_settings: { async_insert: 0 }, // binding: user_id stamping breaks otherwise
+    });
+    batches[table] = [];
+  };
+  const push = async (table, row) => {
+    batches[table].push(row);
+    if (batches[table].length >= BATCH_ROWS) await flush(table);
+  };
+
+  const seen = new Set(); // adapters must not double-ship a session id within a pass
+  let sessions = 0, skipped = 0, msgRows = 0, toolRows = 0, unreadable = 0;
+  for (const chat of chats) {
+    if (chat.encrypted) continue;
+    // Same canonical '<source>:<adapter-local id>' as rowsForChat: dedup and
+    // incremental state are per (source, id) — two editors sharing an id must
+    // not skip or overwrite each other.
+    const id = `${chat.source}:${String(chat.composerId)}`;
+    if (seen.has(id)) continue;
+    seen.add(id);
+
+    const prev = existing.get(id);
+    if (!full && prev) {
+      // floor: some adapters (codex) emit fractional-ms timestamps, but DateTime64(3)
+      // stores whole ms — without it every such session loses by <1ms and re-ships.
+      const chatLast = Math.floor(chat.lastUpdatedAt || chat.createdAt || 0);
+      const notNewer = chatLast === 0 || (prev.ms !== null && prev.ms >= chatLast);
+      // Freshness alone is not enough: the transcript must be COMPLETE (stored
+      // row count == recorded message_count) so an interrupted re-ship — even one
+      // that died mid-flush leaving a partial transcript — repairs itself on the
+      // next pass instead of being skipped forever.
+      if (notNewer && prev.bc >= (chat.bubbleCount || 0) && prev.intact) { skipped++; continue; }
+    }
+
+    const rows = rowsForChat(chat, host);
+    if (!rows) { unreadable++; continue; } // write nothing → retried next pass
+    if (prev) {
+      // Known session being re-shipped: clear its old rows BEFORE inserting so a
+      // shorter re-parse can't leave stale seq/idx tails. A crash between the
+      // delete and the inserts is repaired by the next pass: the skip predicate
+      // refuses to skip a non-empty session whose message rows are missing.
+      for (const t of ['messages', 'tool_calls']) {
+        await client.command({
+          query: `DELETE FROM ${t} WHERE session_id = {id:String} AND user_id = currentUser()`,
+          query_params: { id },
+          clickhouse_settings: { async_insert: 0 },
+        });
+      }
+    }
+    await push('sessions', rows.session);
+    sessions++;
+    for (const r of rows.msgRows) { await push('messages', r); msgRows++; }
+    for (const r of rows.toolRows) { await push('tool_calls', r); toolRows++; }
+  }
+  for (const table of Object.keys(batches)) await flush(table);
+  return { sessions, skipped, msgRows, toolRows, unreadable };
+}
+
+// Per-source rollup straight from sessions_v (final=1 so ReplacingMergeTree collapses).
+async function printStats(client) {
+  const rs = await client.query({
+    query: `
+      SELECT source,
+             count() AS sessions,
+             sum(total_msgs) AS messages,
+             sum(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) AS tokens
+      FROM sessions_v
+      GROUP BY source
+      ORDER BY sessions DESC`,
+    format: 'JSONEachRow',
+    clickhouse_settings: { final: 1 },
+  });
+  const rows = await rs.json();
+  if (!rows.length) { console.log('[mem-house] house is empty'); return; }
+  const w = [Math.max(6, ...rows.map((r) => r.source.length)), 8, 8, 14];
+  const line = (a, b, c, d) =>
+    console.log(`${String(a).padEnd(w[0])}  ${String(b).padStart(w[1])}  ${String(c).padStart(w[2])}  ${String(d).padStart(w[3])}`);
+  line('source', 'sessions', 'messages', 'tokens');
+  let ts = 0, tm = 0, tt = 0;
+  for (const r of rows) {
+    line(r.source, r.sessions, r.messages, r.tokens);
+    ts += Number(r.sessions); tm += Number(r.messages); tt += Number(r.tokens);
+  }
+  line('total', ts, tm, tt);
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  const client = makeClient();
+  try {
+    if (argv.includes('--ensure-schema')) {
+      const n = await ensureSchema(client);
+      console.log(`[mem-house] schema ensured (${n} statements)`);
+      return;
+    }
+    if (argv.includes('--stats')) { await printStats(client); return; }
+
+    const loopIdx = argv.indexOf('--loop');
+    const loop = loopIdx !== -1;
+    let intervalSec = 300;
+    if (loop) {
+      const n = Number(argv[loopIdx + 1]);
+      if (Number.isFinite(n) && n > 0) intervalSec = n;
+    }
+    let full = argv.includes('--full');
+    do {
+      const t0 = Date.now();
+      try {
+        const r = await runShip(client, { full });
+        console.log(`[mem-house] shipped ${r.sessions} sessions (${r.skipped} skipped${r.unreadable ? `, ${r.unreadable} unreadable-will-retry` : ''}) → ` +
+          `${r.msgRows} msg rows, ${r.toolRows} tool rows in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
+      } catch (e) {
+        console.error(`[mem-house] pass failed: ${e.message}`);
+        if (!loop) process.exitCode = 1;
+      }
+      full = false; // --full applies to the first pass only; loop passes stay incremental
+      if (loop) {
+        await new Promise((r) => setTimeout(r, intervalSec * 1000));
+        resetCaches(); // adapters cache chat lists; drop them so new sessions surface
+      }
+    } while (loop);
+  } finally {
+    await client.close();
+  }
+}
+
+module.exports = { runShip, ensureSchema };
+
+if (require.main === module) {
+  main().catch((e) => { console.error(`[mem-house] fatal: ${e.message}`); process.exit(1); });
+}

package/misc/syntax-check.js

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+// Repo-wide JS syntax gate (`npm test`). Walks every .js file outside
+// node_modules / public / ui (the UI has its own lint + build gates and uses
+// JSX, which `node --check` cannot parse) and fails on the first parse error.
+// A deliberately thin gate: it catches the "broken require tree / stray token"
+// class of regression across the two storage implementations and 17 adapters
+// until a real test suite exists.
+
+const { execFileSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const SKIP = new Set(['node_modules', 'public', 'ui', '.git']);
+
+function walk(dir, out) {
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (e.isDirectory()) {
+      if (!SKIP.has(e.name)) walk(path.join(dir, e.name), out);
+    } else if (e.name.endsWith('.js')) {
+      out.push(path.join(dir, e.name));
+    }
+  }
+  return out;
+}
+
+const files = walk(ROOT, []);
+let failed = 0;
+for (const f of files) {
+  try {
+    execFileSync(process.execPath, ['--check', f], { stdio: 'pipe' });
+  } catch (e) {
+    failed++;
+    console.error(`SYNTAX ERROR: ${path.relative(ROOT, f)}\n${e.stderr}`);
+  }
+}
+console.log(`${files.length - failed}/${files.length} files parse clean`);
+process.exit(failed ? 1 : 0);