memhouse 0.0.1 → 0.3.0

package/mcp-server.js

@@ -0,0 +1,294 @@
+const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js');
+const { SSEServerTransport } = require('@modelcontextprotocol/sdk/server/sse.js');
+const { StreamableHTTPServerTransport } = require('@modelcontextprotocol/sdk/server/streamableHttp.js');
+const { z } = require('zod');
+const crypto = require('crypto');
+
+/**
+ * Creates an MCP server instance wired to the relay database.
+ * Returns { mcpServer, transports } — caller wires SSE endpoints into Express.
+ */
+function createMcpServer(getDb) {
+  const mcpServer = new McpServer({
+    name: 'agentlytics-relay',
+    version: '1.0.0',
+  });
+
+  // ── Tool: list_users ──
+  mcpServer.tool(
+    'list_users',
+    'List all connected users and their shared projects',
+    {},
+    async () => {
+      const db = getDb();
+      if (!db) return { content: [{ type: 'text', text: 'Relay database not initialized' }] };
+      const users = db.prepare('SELECT username, last_seen, projects FROM users ORDER BY last_seen DESC').all();
+      const result = users.map(u => ({
+        username: u.username,
+        lastSeen: new Date(u.last_seen).toISOString(),
+        projects: JSON.parse(u.projects || '[]'),
+      }));
+      return {
+        content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+      };
+    }
+  );
+
+  // ── Tool: search_sessions ──
+  mcpServer.tool(
+    'search_sessions',
+    'Search across all users\' chat messages by keyword. Use this to find what someone worked on, or find discussions about a specific file or topic.',
+    {
+      query: z.string().describe('Search query — keyword, file name, or topic'),
+      username: z.string().optional().describe('Filter by specific username'),
+      project: z.string().optional().describe('Filter by project folder path (partial match)'),
+      limit: z.number().optional().describe('Max results (default 20)'),
+    },
+    async ({ query, username, project, limit }) => {
+      const db = getDb();
+      if (!db) return { content: [{ type: 'text', text: 'Relay database not initialized' }] };
+
+      let sql = `
+        SELECT rm.chat_id, rm.username, rm.role, rm.content, rm.model, rm.seq,
+               rc.name as chat_name, rc.source, rc.folder, rc.last_updated_at
+        FROM relay_messages rm
+        JOIN relay_chats rc ON rm.chat_id = rc.id AND rm.username = rc.username
+        WHERE rm.content LIKE ?`;
+      const params = [`%${query}%`];
+
+      if (username) { sql += ' AND rm.username = ?'; params.push(username); }
+      if (project) { sql += ' AND rc.folder LIKE ?'; params.push(`%${project}%`); }
+      sql += ' ORDER BY rc.last_updated_at DESC LIMIT ?';
+      params.push(limit || 20);
+
+      const rows = db.prepare(sql).all(...params);
+
+      if (rows.length === 0) {
+        return { content: [{ type: 'text', text: `No results found for "${query}"` }] };
+      }
+
+      const results = rows.map(r => ({
+        chatId: r.chat_id,
+        chatName: r.chat_name,
+        username: r.username,
+        role: r.role,
+        source: r.source,
+        folder: r.folder,
+        lastUpdated: r.last_updated_at ? new Date(r.last_updated_at).toISOString() : null,
+        model: r.model,
+        content: r.content.length > 300 ? r.content.substring(0, 300) + '...' : r.content,
+      }));
+
+      return {
+        content: [{ type: 'text', text: JSON.stringify(results, null, 2) }],
+      };
+    }
+  );
+
+  // ── Tool: get_user_activity ──
+  mcpServer.tool(
+    'get_user_activity',
+    'Get recent activity for a specific user — their recent sessions, what they worked on, which editors and models they used.',
+    {
+      username: z.string().describe('Username to look up'),
+      project: z.string().optional().describe('Filter by project folder (partial match)'),
+      file_path: z.string().optional().describe('Filter by file path mentioned in messages'),
+      limit: z.number().optional().describe('Max sessions to return (default 20)'),
+    },
+    async ({ username, project, file_path, limit }) => {
+      const db = getDb();
+      if (!db) return { content: [{ type: 'text', text: 'Relay database not initialized' }] };
+
+      // First get sessions
+      let sql = `
+        SELECT rc.*, rcs.total_messages, rcs.models, rcs.tool_calls,
+               rcs.total_input_tokens, rcs.total_output_tokens
+        FROM relay_chats rc
+        LEFT JOIN relay_chat_stats rcs ON rc.id = rcs.chat_id AND rc.username = rcs.username
+        WHERE rc.username = ?`;
+      const params = [username];
+
+      if (project) { sql += ' AND rc.folder LIKE ?'; params.push(`%${project}%`); }
+      sql += ' ORDER BY rc.last_updated_at DESC LIMIT ?';
+      params.push(limit || 20);
+
+      let sessions = db.prepare(sql).all(...params);
+
+      // If file_path filter, narrow down to sessions mentioning that file
+      if (file_path && sessions.length > 0) {
+        const chatIds = sessions.map(s => s.id);
+        const placeholders = chatIds.map(() => '?').join(',');
+        const fileMatches = db.prepare(`
+          SELECT DISTINCT chat_id FROM relay_messages
+          WHERE chat_id IN (${placeholders}) AND username = ? AND content LIKE ?
+        `).all(...chatIds, username, `%${file_path}%`);
+        const matchingIds = new Set(fileMatches.map(m => m.chat_id));
+        sessions = sessions.filter(s => matchingIds.has(s.id));
+      }
+
+      if (sessions.length === 0) {
+        return { content: [{ type: 'text', text: `No activity found for user "${username}"` }] };
+      }
+
+      const result = sessions.map(s => ({
+        id: s.id,
+        name: s.name,
+        source: s.source,
+        mode: s.mode,
+        folder: s.folder,
+        lastUpdated: s.last_updated_at ? new Date(s.last_updated_at).toISOString() : null,
+        totalMessages: s.total_messages,
+        models: s.models ? [...new Set(JSON.parse(s.models))].slice(0, 5) : [],
+        toolCalls: s.tool_calls ? JSON.parse(s.tool_calls).length : 0,
+        totalInputTokens: s.total_input_tokens,
+        totalOutputTokens: s.total_output_tokens,
+      }));
+
+      return {
+        content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+      };
+    }
+  );
+
+  // ── Tool: get_session_detail ──
+  mcpServer.tool(
+    'get_session_detail',
+    'Get the full conversation messages for a specific session. Use the session ID from search_sessions or get_user_activity results.',
+    {
+      session_id: z.string().describe('The chat/session ID'),
+      username: z.string().optional().describe('Username who owns the session (optional, auto-detected if unique)'),
+    },
+    async ({ session_id, username }) => {
+      const db = getDb();
+      if (!db) return { content: [{ type: 'text', text: 'Relay database not initialized' }] };
+
+      let chatSql = 'SELECT * FROM relay_chats WHERE id = ?';
+      const chatParams = [session_id];
+      if (username) { chatSql += ' AND username = ?'; chatParams.push(username); }
+      chatSql += ' LIMIT 1';
+
+      const chat = db.prepare(chatSql).get(...chatParams);
+      if (!chat) {
+        return { content: [{ type: 'text', text: `Session "${session_id}" not found` }] };
+      }
+
+      const messages = db.prepare(
+        'SELECT seq, role, content, model FROM relay_messages WHERE chat_id = ? AND username = ? ORDER BY seq'
+      ).all(chat.id, chat.username);
+
+      const formatted = messages.map(m => {
+        const label = m.role === 'user' ? 'User' : m.role === 'assistant' ? 'Assistant' : m.role;
+        const modelTag = m.model ? ` (${m.model})` : '';
+        const content = m.content.length > 2000 ? m.content.substring(0, 2000) + '\n... [truncated]' : m.content;
+        return `## ${label}${modelTag}\n\n${content}`;
+      }).join('\n\n---\n\n');
+
+      const header = `# ${chat.name || 'Untitled Session'}\n**User:** ${chat.username} | **Editor:** ${chat.source} | **Project:** ${chat.folder || 'N/A'}\n\n---\n\n`;
+
+      return {
+        content: [{ type: 'text', text: header + formatted }],
+      };
+    }
+  );
+
+  return mcpServer;
+}
+
+/**
+ * Wire MCP SSE transport into an Express app.
+ * GET /mcp → establishes SSE connection
+ * POST /mcp → receives messages from MCP client
+ */
+function wireMcpToExpress(app, getDb) {
+  const sseTransports = {};
+  const httpTransports = {};
+
+  // Same auth gate as the relay's /relay/* routes (same token derivation as
+  // createRelayApp): when RELAY_PASSWORD is set, /mcp must not expose
+  // search_sessions / get_session_detail to unauthenticated clients.
+  const RELAY_PASSWORD = process.env.RELAY_PASSWORD || null;
+  const AUTH_TOKEN = RELAY_PASSWORD
+    ? require('crypto').createHmac('sha256', 'agentlytics-relay').update(RELAY_PASSWORD).digest('hex')
+    : null;
+  function requireAuth(req, res, next) {
+    if (!AUTH_TOKEN) return next();
+    const header = req.headers.authorization || '';
+    const token = header.startsWith('Bearer ') ? header.slice(7) : null;
+    if (token === AUTH_TOKEN) return next();
+    res.status(401).json({ error: 'Unauthorized' });
+  }
+
+  // SSE: GET /mcp establishes SSE stream
+  app.get('/mcp', requireAuth, async (req, res) => {
+    const sessionId = req.headers['mcp-session-id'];
+
+    // Streamable HTTP GET for SSE stream resumption
+    if (sessionId && httpTransports[sessionId]) {
+      const transport = httpTransports[sessionId];
+      await transport.handleRequest(req, res);
+      return;
+    }
+
+    // Legacy SSE transport
+    const transport = new SSEServerTransport('/mcp', res);
+    sseTransports[transport.sessionId] = transport;
+
+    const mcpServer = createMcpServer(getDb);
+
+    res.on('close', () => {
+      delete sseTransports[transport.sessionId];
+      mcpServer.close().catch(() => {});
+    });
+
+    await mcpServer.connect(transport);
+  });
+
+  // POST /mcp handles both SSE messages and Streamable HTTP
+  app.post('/mcp', requireAuth, async (req, res) => {
+    // Check for SSE session first
+    const sseSessionId = req.query.sessionId;
+    if (sseSessionId && sseTransports[sseSessionId]) {
+      await sseTransports[sseSessionId].handlePostMessage(req, res);
+      return;
+    }
+
+    // Streamable HTTP: check for existing session
+    const sessionId = req.headers['mcp-session-id'];
+    if (sessionId && httpTransports[sessionId]) {
+      await httpTransports[sessionId].handleRequest(req, res, req.body);
+      return;
+    }
+
+    // New Streamable HTTP session (initialization request)
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: () => crypto.randomUUID(),
+      onsessioninitialized: (id) => {
+        httpTransports[id] = transport;
+      },
+    });
+
+    transport.onclose = () => {
+      if (transport.sessionId) {
+        delete httpTransports[transport.sessionId];
+      }
+    };
+
+    const mcpServer = createMcpServer(getDb);
+    await mcpServer.connect(transport);
+    await transport.handleRequest(req, res, req.body);
+  });
+
+  // DELETE /mcp for session cleanup
+  app.delete('/mcp', requireAuth, async (req, res) => {
+    const sessionId = req.headers['mcp-session-id'];
+    if (sessionId && httpTransports[sessionId]) {
+      await httpTransports[sessionId].handleRequest(req, res);
+      return;
+    }
+    res.status(404).end();
+  });
+
+  return { sseTransports, httpTransports };
+}
+
+module.exports = { createMcpServer, wireMcpToExpress };

package/mem-house/DESIGN.md

@@ -0,0 +1,119 @@
+# mem-house — design
+
+**mem-house** is agent conversation memory as a product: every coding-agent session
+on your machines — across **all 17 editors** agentlytics supports — parsed locally,
+shipped to one typed ClickHouse store, shareable with a team, installable on the
+ultimagent kernel as an **agency**, and visible through the agentlytics dashboard.
+
+**Agency, precisely.** In constellation terms (`../TERMINOLOGY.md`) mem-house is a
+**house** — the `memhouse` database — **plus a resident**: the shipper. That pairing
+is what the word *agency* means; a house on its own only holds.
+
+The test is mechanical — **residents write, routines read**. The shipper fires on a
+trigger no query supplies (its own loop) and produces writes that outlive the call:
+
+```toml
+[[resident]]
+kind = "worker"    # deterministic code, no LLM
+on   = "loop"      # a daemon — not an insert trigger, not a schedule
+```
+
+`sessions_v` is a plain view: computed during your query, for your query, writing
+nothing. It is a **routine** — house machinery, not a second resident. mem-house has
+**no materialized views at all**, so the shipper is not merely a resident, it is the
+only candidate in the tree. Strip it and every remaining moving part is a routine
+over rows nobody is writing any more.
+
+Note that `user_id MATERIALIZED currentUser()` is a materialized *column*: it computes
+during your own insert and is part of the table's definition. It makes own-only RLS
+work and it records who wrote each row, but it is an identity property, not what makes
+this an agency.
+
+**Positioning:** an alternative agency **competing with memory-house**. It borrows
+memory-house's proven ideas (server-stamped identity, own-only RLS, idempotent
+shipping, skills/plugin delivery) and agentlytics' proven assets (adapters, UI,
+cost engine). If it wins, it can become memory-house v4; until then the two run
+side by side as separate agencies, each in its own house. Naming: the product is
+**mem-house**; ClickHouse identifiers use **`memhouse`** (agency/house name, owner
+`memhouse_root`) since identifiers can't carry a dash.
+
+## The four bets
+
+1. **Parse on the client.** memory-house ships raw transcript lines and parses in
+   views — workable only because its 4 agents store sessions as JSON/JSONL files.
+   agentlytics' breadth includes sqlite-backed editors (Cursor, VS Code, Zed,
+   Devin…) where no "raw line" exists; the adapter must crack a DB. So client-side
+   parsing is not a style choice — it is what unlocks 17-editor coverage. The
+   shipper runs the adapters (`editors/`) and ships **typed rows**. On the canon's
+   write axis this bet is precisely *moving work from routine to resident*:
+   memory-house parses when you query, mem-house parses before anyone asks and
+   writes the result down.
+2. **Typed common schema** (`schema.sql`). Physical typed columns (what
+   memory-house derives in views) + `tool_calls` (memory-house has none) + one
+   `extra JSON` escape hatch per table so unnormalized adapter fields are never
+   lost. `ReplacingMergeTree(ingested_at)`; `messages` keyed `(session_id, seq)` so
+   a re-ship of a grown/changed session replaces stale rows (latest-wins). FTS text
+   indexes built in (CH ≥ 26.2).
+3. **Kernel-installable agency.** Same install path proven for agentlytics-agency:
+   `install-agency{memhouse}` → house + `memhouse_root` + credential; members via
+   `register-member` + owner `GRANT`; own-only visibility via the row policy
+   (`rls.sql`) applied by the ego (the `kernel` user) or the mayor. The shipper is
+   the resident that lands with it. Identity is `user_id MATERIALIZED currentUser()`
+   (requires `async_insert=0`). Sharing modes: own-only (policy TO member) or
+   team-pool (no policy).
+4. **Borrowed UI, zero fork.** The React SPA consumes REST JSON, not tables. The
+   mem-house server implements the **same `/api/*` contract** as agentlytics'
+   `server.js` (same routes, same response shapes) over the mem-house schema, and
+   serves the same built `public/` bundle. UI changes: none.
+
+## Components (this directory)
+
+| Path | What | Notes |
+|---|---|---|
+| `schema.sql` | the house schema (typed) | unqualified names; owner applies in its house |
+| `rls.sql` | own-only row policies | the ego (`kernel`) or the mayor applies (owner lacks ACCESS MANAGEMENT) |
+| `shipper/ship.js` | parse-on-client shipper CLI — the resident (`worker`) | reuses `../../editors`; incremental; idempotent |
+| `server/server.js` | REST API + dashboard | same API contract as agentlytics; serves `../../public` |
+| `delivery/` | delivery kit | installer, skills, plugin, AGENT-INSTALL.md, prompt |
+
+## Environment contract
+
+| Var | Default | Meaning |
+|---|---|---|
+| `MEMHOUSE_URL` | `http://localhost:8123` | ClickHouse HTTP(S) endpoint |
+| `MEMHOUSE_USER` | `memhouse_root` | CH user (owner, or a member's own credential) |
+| `MEMHOUSE_PASSWORD` | *(empty)* | credential |
+| `MEMHOUSE_DB` | `memhouse` | the house |
+| `MEMHOUSE_PORT` | `4640` | dashboard/API port |
+
+Config file: `~/.memhouse/config.json` (`hiddenProjects`, future prefs).
+
+## Data-plane rules (binding for the shipper and server)
+
+- Inserts: `JSONEachRow`, batches ≤ 2000 rows, **`async_insert=0` always** (else
+  `user_id` stamping breaks). Reads: set `final: 1` so ReplacingMergeTree collapses.
+- All Int64-bound values coerced to integers (some adapters emit fractional ms).
+- Per-message `ts`: interpolated across the session's `[createdAt, lastUpdatedAt]`
+  (adapters don't expose per-message timestamps; monotonic by `seq`; documented
+  approximation, refine per-adapter later).
+- `seq` is the message index within the session (0-based). `tool_calls.idx` is the
+  call index within the session. `is_subagent` = message content begins with
+  `[subagent]` (the fold marker from the Claude adapter).
+- Incremental shipping: read existing `(session_id, last_updated_at, message_count)`
+  (FINAL) and skip chats that haven't grown/changed; a full re-ship must remain
+  safe (ReplacingMergeTree collapses).
+- Costs are computed in the server from `pricing.js` (repo root) over per-model
+  token sums; `<synthetic>` and empty models excluded from model lists.
+
+## Delivery kit (mirrors memory-house)
+
+`install.sh` (standalone), `AGENT-INSTALL.md` (agent-facing install steps),
+`skills/` (Claude Code skills to query/search mem-house), `plugin/` (Claude Code
+plugin wrapping the skills), `PROMPT.md` (system-prompt snippet teaching an agent
+to use mem-house as memory), `kernel-install.md` (provision as agency `memhouse`).
+
+## Non-goals (v0)
+
+Relay/team-server (the kernel + RLS already covers sharing), launchd daemonization
+(ship `--loop` instead), migration tooling from memory-house's `memory` db, and
+per-adapter true message timestamps.

package/mem-house/delivery/AGENT-INSTALL.md

@@ -0,0 +1,75 @@
+# mem-house — agent install
+
+Imperative steps for an AI coding agent installing mem-house on a machine. Execute
+in order; every command is copy-pasteable. Prereqs: Node >= 20.19, a reachable
+ClickHouse >= 26.2 (local server, Cloud, or a kernel house — see
+`kernel-install.md` for the kernel path), and this repo checked out.
+
+## 1. Install
+
+```bash
+cd <repo>            # the memhouse checkout
+npm install --no-audit --no-fund
+```
+
+## 2. Configure the connection
+
+Write `~/.memhouse/env` (the installer can do this interactively — `bash
+mem-house/delivery/install.sh` — or write it yourself):
+
+```bash
+mkdir -p ~/.memhouse
+cat > ~/.memhouse/env <<'EOF'
+MEMHOUSE_URL=http://localhost:8123
+MEMHOUSE_USER=memhouse_root
+MEMHOUSE_PASSWORD=<credential>
+MEMHOUSE_DB=memhouse
+EOF
+chmod 600 ~/.memhouse/env
+```
+
+## 3. Create the schema and run the first ship
+
+```bash
+set -a; source ~/.memhouse/env; set +a
+node mem-house/shipper/ship.js --ensure-schema
+node mem-house/shipper/ship.js            # first full parse-on-client ship
+```
+
+Expected output shape: `[mem-house] shipped N sessions (0 skipped) → M msg rows, T
+tool rows in Xs` with N in the hundreds on a machine with real agent usage.
+
+## 4. Verify
+
+```bash
+node mem-house/shipper/ship.js --stats
+```
+
+Expect one row per source (claude-code, codex, gemini-cli, …) with non-zero
+sessions/messages. Or verify by SQL:
+
+```bash
+curl -s -u "$MEMHOUSE_USER:$MEMHOUSE_PASSWORD" "$MEMHOUSE_URL/?database=$MEMHOUSE_DB" \
+  --data-binary "SELECT source, count() FROM sessions_v GROUP BY source SETTINGS final=1 FORMAT PrettyCompact"
+```
+
+## 5. Start the dashboard (optional)
+
+```bash
+cd ui && npm install && npm run build && cd ..   # first time only (builds public/)
+node mem-house/server/server.js                  # → http://localhost:4640
+```
+
+## 6. Keep it fresh
+
+Re-ship on a schedule (incremental — cheap):
+
+```bash
+node mem-house/shipper/ship.js --loop 300 &
+```
+
+## 7. Install the skills (optional, for Claude Code)
+
+Copy `mem-house/delivery/skills/*` into the target agent's skills directory
+(e.g. `$CLAUDE_CONFIG_DIR/skills/`), or install `mem-house/delivery/plugin/` as a
+Claude Code plugin. The skills read the connection from `~/.memhouse/env`.

package/mem-house/delivery/PROMPT.md

@@ -0,0 +1,42 @@
+# mem-house — system-prompt snippet
+
+Paste (or import) this into an agent's system prompt to make mem-house its
+long-term conversation memory.
+
+---
+
+## Conversation memory (mem-house)
+
+You have persistent memory of past agent sessions — every conversation this user
+has had with coding agents (Claude Code, Codex, Gemini CLI, Cursor, and other
+editors), across machines, stored in a ClickHouse database called **mem-house**.
+Connection: read `~/.memhouse/env` (`MEMHOUSE_URL/USER/PASSWORD/DB`); query over
+HTTP with `curl -u "$MEMHOUSE_USER:$MEMHOUSE_PASSWORD" "$MEMHOUSE_URL/?database=$MEMHOUSE_DB"`
+and always add `SETTINGS final=1` to reads.
+
+**When to reach for it — before claiming ignorance.** If the user refers to past
+work that is not in your current context ("that session where…", "how did I solve
+X before", "did we ever…", "what was I working on last week"), search mem-house
+FIRST. Only say you don't know after a search comes back empty.
+
+**How to query.** Prefer the installed skills when present: `/memhouse:search`
+(full-text over messages), `/memhouse:sessions` (list/filter sessions),
+`/memhouse:sql` (free-form read-only SQL). Without skills, query directly:
+
+- Find sessions about a topic (FTS, lowercase your terms):
+  `SELECT DISTINCT session_id, any(project), min(ts) FROM messages
+   WHERE hasToken(text_word, 'clickhouse') GROUP BY session_id
+   ORDER BY 3 DESC LIMIT 10 SETTINGS final=1 FORMAT PrettyCompact`
+- Recent sessions: `SELECT session_id, source, project, started, first_prompt
+   FROM sessions_v ORDER BY started DESC LIMIT 20 SETTINGS final=1`
+- Replay one session: `SELECT role, text FROM messages
+   WHERE session_id = '<id>' ORDER BY seq SETTINGS final=1`
+
+**Rules.**
+- Memory is READ-ONLY for you. Never INSERT/ALTER/DROP — ingestion belongs to the
+  mem-house shipper alone.
+- Quote retrieved content as *the user's past sessions*, and cite the session_id
+  when the user may want to dig deeper.
+- Other members' sessions may be invisible to you (row-level security). An empty
+  result means "nothing visible", not "nothing ever happened".
+- Do not paste credentials from `~/.memhouse/env` into responses, commits, or logs.