@jhizzard/termdeck 0.2.0

package/packages/server/src/config.js

@@ -0,0 +1,308 @@
+// TermDeck config loader with secrets.env support (F2.2).
+//
+// Secrets live in ~/.termdeck/secrets.env (dotenv format) so that
+// ~/.termdeck/config.yaml can be committed / shared / reviewed without
+// carrying plaintext API keys. Precedence (highest wins):
+//
+//   1. process.env (as it was at launch)
+//   2. ~/.termdeck/secrets.env (loaded here, merged INTO process.env)
+//   3. ${VAR} substitutions inside config.yaml
+//   4. Inline values in config.yaml (legacy, triggers a deprecation warning)
+//   5. Built-in defaults
+//
+// Never print secret values. The deprecation warning must not echo the leaked
+// key — it just names the config.yaml field that still holds an inline secret.
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const CONFIG_DIR = path.join(os.homedir(), '.termdeck');
+const CONFIG_PATH = path.join(CONFIG_DIR, 'config.yaml');
+const SECRETS_PATH = path.join(CONFIG_DIR, 'secrets.env');
+
+// Fields in config.yaml that historically held plaintext secrets. Used to
+// emit the one-time deprecation warning on startup.
+const LEGACY_SECRET_PATHS = [
+  ['rag', 'supabaseKey'],
+  ['rag', 'openaiApiKey'],
+  ['rag', 'anthropicApiKey']
+];
+
+// Very small dotenv parser so we don't take on a dependency for 40 lines of code.
+// Supports: KEY=value, KEY="quoted value", KEY='single', blank lines, #comments.
+// Does NOT support variable expansion inside values (not needed here).
+function parseDotenv(raw) {
+  const out = {};
+  const lines = raw.split(/\r?\n/);
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eq = trimmed.indexOf('=');
+    if (eq === -1) continue;
+    const key = trimmed.slice(0, eq).trim();
+    if (!key) continue;
+    let val = trimmed.slice(eq + 1).trim();
+    if ((val.startsWith('"') && val.endsWith('"')) ||
+        (val.startsWith("'") && val.endsWith("'"))) {
+      val = val.slice(1, -1);
+    }
+    out[key] = val;
+  }
+  return out;
+}
+
+function loadSecretsEnv() {
+  if (!fs.existsSync(SECRETS_PATH)) return { loaded: false, keys: [] };
+  try {
+    const raw = fs.readFileSync(SECRETS_PATH, 'utf-8');
+    const parsed = parseDotenv(raw);
+    const keys = [];
+    for (const [k, v] of Object.entries(parsed)) {
+      // Do not clobber pre-set process env; shell wins.
+      if (process.env[k] === undefined) {
+        process.env[k] = v;
+      }
+      keys.push(k);
+    }
+    return { loaded: true, keys };
+  } catch (err) {
+    console.warn('[config] Failed to read secrets.env:', err.message);
+    return { loaded: false, keys: [] };
+  }
+}
+
+// Walk a parsed YAML tree and replace ${VAR} / ${VAR:-default} tokens in any
+// string leaf using process.env. Unknown vars → empty string (matches dotenv
+// conventions), unless a default is supplied via :-.
+function substituteEnv(value) {
+  if (value == null) return value;
+  if (typeof value === 'string') {
+    return value.replace(/\$\{([A-Z0-9_]+)(?::-([^}]*))?\}/gi, (_m, key, def) => {
+      const v = process.env[key];
+      if (v !== undefined && v !== '') return v;
+      return def !== undefined ? def : '';
+    });
+  }
+  if (Array.isArray(value)) return value.map(substituteEnv);
+  if (typeof value === 'object') {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = substituteEnv(v);
+    return out;
+  }
+  return value;
+}
+
+function getPath(obj, segs) {
+  let cur = obj;
+  for (const s of segs) {
+    if (cur == null || typeof cur !== 'object') return undefined;
+    cur = cur[s];
+  }
+  return cur;
+}
+
+// A "secret-shaped" inline value is a non-empty string that is NOT an ${ENV}
+// reference. We treat `${FOO}` and `${FOO:-bar}` as safe (just a template).
+function looksLikeInlineSecret(v) {
+  if (typeof v !== 'string') return false;
+  if (!v) return false;
+  if (/^\$\{[A-Z0-9_]+(?::-[^}]*)?\}$/i.test(v.trim())) return false;
+  return true;
+}
+
+function warnIfLegacyInlineSecrets(parsed, secretsLoaded) {
+  if (secretsLoaded) return; // secrets.env exists — user has migrated, don't nag.
+  const hits = [];
+  for (const segs of LEGACY_SECRET_PATHS) {
+    const v = getPath(parsed, segs);
+    if (looksLikeInlineSecret(v)) hits.push(segs.join('.'));
+  }
+  if (hits.length > 0) {
+    console.warn(
+      `[config] WARNING: secrets in config.yaml are deprecated — move them to ~/.termdeck/secrets.env ` +
+      `(see config/secrets.env.example). Fields still inline: ${hits.join(', ')}`
+    );
+  }
+}
+
+function defaultConfig() {
+  return {
+    port: 3000,
+    host: '127.0.0.1',
+    shell: process.env.SHELL || '/bin/bash',
+    defaultTheme: 'tokyo-night',
+    projects: {},
+    rag: {
+      enabled: false,
+      supabaseUrl: null,
+      supabaseKey: null,
+      openaiApiKey: null,
+      anthropicApiKey: null,
+      developerId: os.userInfo().username,
+      syncIntervalMs: 10000,
+      engramMode: 'direct',
+      engramWebhookUrl: 'http://localhost:37778/engram',
+      tables: {
+        session: 'engram_session_memory',
+        project: 'engram_project_memory',
+        developer: 'engram_developer_memory',
+        commands: 'engram_commands'
+      }
+    },
+    sessionLogs: {
+      enabled: false,
+      summaryModel: 'claude-haiku-4-5'
+    }
+  };
+}
+
+function loadConfig() {
+  if (!fs.existsSync(CONFIG_DIR)) {
+    fs.mkdirSync(CONFIG_DIR, { recursive: true });
+  }
+
+  const secrets = loadSecretsEnv();
+  if (secrets.loaded) {
+    console.log(`[config] Loaded secrets from ${SECRETS_PATH} (${secrets.keys.length} key${secrets.keys.length === 1 ? '' : 's'})`);
+  }
+
+  const defaults = defaultConfig();
+
+  // Auto-create default config.yaml on first run (unchanged behavior).
+  if (!fs.existsSync(CONFIG_PATH)) {
+    const defaultYaml = `# TermDeck Configuration
+# Secrets belong in ~/.termdeck/secrets.env, not here.
+
+port: 3000
+host: 127.0.0.1
+
+shell: ${process.env.SHELL || '/bin/bash'}
+defaultTheme: tokyo-night
+
+# projects:
+#   my-project:
+#     path: ~/code/my-project
+#     defaultTheme: catppuccin-mocha
+#     defaultCommand: claude
+
+rag:
+  enabled: false
+  # supabaseUrl and secrets come from ~/.termdeck/secrets.env
+  supabaseUrl: \${SUPABASE_URL}
+  supabaseKey: \${SUPABASE_SERVICE_ROLE_KEY}
+  openaiApiKey: \${OPENAI_API_KEY}
+  anthropicApiKey: \${ANTHROPIC_API_KEY}
+  syncIntervalMs: 10000
+  engramMode: direct
+  engramWebhookUrl: http://localhost:37778/engram
+
+sessionLogs:
+  enabled: false
+  summaryModel: claude-haiku-4-5
+`;
+    fs.writeFileSync(CONFIG_PATH, defaultYaml, 'utf-8');
+    console.log('[config] Created default config at', CONFIG_PATH);
+  }
+
+  let parsed = {};
+  try {
+    const yaml = require('yaml');
+    const raw = fs.readFileSync(CONFIG_PATH, 'utf-8');
+    parsed = yaml.parse(raw) || {};
+    console.log('[config] Loaded from', CONFIG_PATH);
+  } catch (err) {
+    console.warn('[config] Could not load config.yaml, using defaults:', err.message);
+    parsed = {};
+  }
+
+  // Warn about inline secrets BEFORE substitution so the diagnostic matches
+  // what the user actually has on disk.
+  warnIfLegacyInlineSecrets(parsed, secrets.loaded);
+
+  const substituted = substituteEnv(parsed);
+
+  return {
+    ...defaults,
+    ...substituted,
+    rag: { ...defaults.rag, ...(substituted?.rag || {}) },
+    sessionLogs: { ...defaults.sessionLogs, ...(substituted?.sessionLogs || {}) }
+  };
+}
+
+// Add a project to ~/.termdeck/config.yaml and return the updated projects map.
+// Writes a timestamped .bak of the existing file before overwriting so the
+// user can always recover manually. Comments in the original yaml WILL be lost
+// on rewrite — yaml.stringify does not round-trip comments. That's acceptable
+// for a v0.2 convenience feature; permanent editing still belongs in a text
+// editor for users who care about file comments.
+function addProject({ name, path: projectPath, defaultTheme, defaultCommand }) {
+  if (!name || !/^[A-Za-z0-9_.-]+$/.test(name)) {
+    throw new Error('Project name must be non-empty and contain only letters, digits, . _ or -');
+  }
+  if (!projectPath || typeof projectPath !== 'string') {
+    throw new Error('Project path is required');
+  }
+
+  // Expand ~ for validation but keep tilde form in the stored config so it
+  // remains portable across machines.
+  const expanded = projectPath.replace(/^~/, os.homedir());
+  const resolved = path.resolve(expanded);
+  if (!fs.existsSync(resolved)) {
+    throw new Error(`Project path does not exist: ${resolved}`);
+  }
+  const stat = fs.statSync(resolved);
+  if (!stat.isDirectory()) {
+    throw new Error(`Project path is not a directory: ${resolved}`);
+  }
+
+  const yaml = require('yaml');
+  let parsed = {};
+  if (fs.existsSync(CONFIG_PATH)) {
+    const raw = fs.readFileSync(CONFIG_PATH, 'utf-8');
+    try {
+      parsed = yaml.parse(raw) || {};
+    } catch (err) {
+      throw new Error(`config.yaml is not valid YAML — cannot safely rewrite: ${err.message}`);
+    }
+  }
+
+  if (!parsed.projects || typeof parsed.projects !== 'object') {
+    parsed.projects = {};
+  }
+  if (parsed.projects[name]) {
+    throw new Error(`Project "${name}" already exists`);
+  }
+
+  parsed.projects[name] = {
+    path: projectPath,
+    ...(defaultTheme ? { defaultTheme } : {}),
+    ...(defaultCommand ? { defaultCommand } : {})
+  };
+
+  // Backup before overwrite.
+  if (fs.existsSync(CONFIG_PATH)) {
+    const ts = new Date().toISOString().replace(/[:.]/g, '-');
+    const bak = `${CONFIG_PATH}.${ts}.bak`;
+    try {
+      fs.copyFileSync(CONFIG_PATH, bak);
+    } catch (err) {
+      console.warn('[config] Could not write backup before adding project:', err.message);
+    }
+  }
+
+  const out = yaml.stringify(parsed);
+  fs.writeFileSync(CONFIG_PATH, out, 'utf-8');
+  console.log(`[config] Added project "${name}" → ${projectPath}`);
+
+  return parsed.projects;
+}
+
+module.exports = {
+  loadConfig,
+  addProject,
+  // exported for tests / introspection
+  _parseDotenv: parseDotenv,
+  _substituteEnv: substituteEnv,
+  _paths: { CONFIG_DIR, CONFIG_PATH, SECRETS_PATH }
+};

package/packages/server/src/database.js

@@ -0,0 +1,130 @@
+// Database layer - SQLite for session persistence and command history
+
+const path = require('path');
+const os = require('os');
+
+function initDatabase(Database) {
+  const dbPath = path.join(os.homedir(), '.termdeck', 'termdeck.db');
+
+  // Ensure directory exists
+  const fs = require('fs');
+  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+
+  const db = new Database(dbPath);
+
+  // Enable WAL mode for better concurrent read performance
+  db.pragma('journal_mode = WAL');
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS sessions (
+      id TEXT PRIMARY KEY,
+      type TEXT NOT NULL DEFAULT 'shell',
+      project TEXT,
+      label TEXT,
+      command TEXT,
+      cwd TEXT,
+      created_at TEXT NOT NULL,
+      exited_at TEXT,
+      exit_code INTEGER,
+      reason TEXT,
+      theme TEXT DEFAULT 'tokyo-night'
+    );
+
+    CREATE TABLE IF NOT EXISTS command_history (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL,
+      command TEXT NOT NULL,
+      output_snippet TEXT,
+      timestamp TEXT NOT NULL,
+      source TEXT NOT NULL DEFAULT 'user',
+      FOREIGN KEY (session_id) REFERENCES sessions(id)
+    );
+
+    CREATE TABLE IF NOT EXISTS rag_events (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL,
+      event_type TEXT NOT NULL,
+      payload TEXT NOT NULL,
+      project TEXT,
+      timestamp TEXT NOT NULL,
+      synced INTEGER DEFAULT 0,
+      FOREIGN KEY (session_id) REFERENCES sessions(id)
+    );
+
+    CREATE TABLE IF NOT EXISTS projects (
+      name TEXT PRIMARY KEY,
+      path TEXT NOT NULL,
+      default_theme TEXT DEFAULT 'tokyo-night',
+      default_command TEXT DEFAULT 'bash',
+      rag_namespace TEXT,
+      created_at TEXT NOT NULL
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project);
+    CREATE INDEX IF NOT EXISTS idx_commands_session ON command_history(session_id);
+    CREATE INDEX IF NOT EXISTS idx_rag_synced ON rag_events(synced);
+    CREATE INDEX IF NOT EXISTS idx_rag_project ON rag_events(project);
+  `);
+
+  // Migration: add command_history.source on existing databases
+  try {
+    const cols = db.prepare(`PRAGMA table_info(command_history)`).all();
+    const hasSource = cols.some((c) => c.name === 'source');
+    if (!hasSource) {
+      db.exec(`ALTER TABLE command_history ADD COLUMN source TEXT NOT NULL DEFAULT 'user'`);
+      db.exec(`UPDATE command_history SET source = 'user' WHERE source IS NULL`);
+      console.log("[db] Migrated command_history: added 'source' column");
+    }
+  } catch (err) {
+    console.warn('[db] command_history.source migration failed:', err.message);
+  }
+
+  return db;
+}
+
+function logCommand(db, sessionId, command, outputSnippet, source) {
+  db.prepare(`
+    INSERT INTO command_history (session_id, command, output_snippet, timestamp, source)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(sessionId, command, outputSnippet || null, new Date().toISOString(), source || 'user');
+}
+
+function logRagEvent(db, sessionId, eventType, payload, project) {
+  db.prepare(`
+    INSERT INTO rag_events (session_id, event_type, payload, project, timestamp)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(sessionId, eventType, JSON.stringify(payload), project, new Date().toISOString());
+}
+
+function getUnsyncedRagEvents(db, limit = 50) {
+  return db.prepare(`
+    SELECT * FROM rag_events WHERE synced = 0 ORDER BY timestamp ASC LIMIT ?
+  `).all(limit);
+}
+
+function markRagEventsSynced(db, ids) {
+  const placeholders = ids.map(() => '?').join(',');
+  db.prepare(`UPDATE rag_events SET synced = 1 WHERE id IN (${placeholders})`).run(...ids);
+}
+
+function getSessionHistory(db, sessionId) {
+  return db.prepare(`
+    SELECT * FROM command_history WHERE session_id = ? ORDER BY timestamp DESC LIMIT 50
+  `).all(sessionId);
+}
+
+function getProjectSessions(db, project) {
+  return db.prepare(`
+    SELECT * FROM sessions WHERE project = ? ORDER BY created_at DESC
+  `).all(project);
+}
+
+module.exports = {
+  initDatabase,
+  logCommand,
+  logRagEvent,
+  getUnsyncedRagEvents,
+  markRagEventsSynced,
+  getSessionHistory,
+  getProjectSessions
+};

package/packages/server/src/engram-bridge/index.js

@@ -0,0 +1,232 @@
+// Engram bridge — routes TermDeck memory queries through one of three backends:
+//   - direct:  talk to Supabase + OpenAI from the server (pre-bridge behavior)
+//   - webhook: POST to Engram's HTTP webhook server (T3.1) at rag.engramWebhookUrl
+//   - mcp:     spawn the @jhizzard/engram binary and talk JSON-RPC over stdio
+//
+// All three modes return the same shape:
+//   { memories: Array<{ content, source_type, project, similarity, created_at }>, total }
+//
+// Errors are thrown as plain Error objects; the caller maps them to HTTP responses.
+
+const { spawn } = require('child_process');
+
+function createBridge(config) {
+  const mode = config.rag?.engramMode || 'direct';
+  const state = { mcpChild: null, mcpQueue: [], mcpNextId: 1, mcpBuffer: '' };
+
+  async function queryDirect({ question, project, searchAll }) {
+    const supabaseUrl = config.rag?.supabaseUrl;
+    const supabaseKey = config.rag?.supabaseKey;
+    const openaiKey = config.rag?.openaiApiKey || process.env.OPENAI_API_KEY;
+
+    if (!supabaseUrl || !supabaseKey) {
+      throw new Error('RAG not configured — add supabaseUrl and supabaseKey to ~/.termdeck/config.yaml');
+    }
+    if (!openaiKey) {
+      throw new Error('OPENAI_API_KEY not configured');
+    }
+
+    const embeddingRes = await fetch('https://api.openai.com/v1/embeddings', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${openaiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: 'text-embedding-3-large',
+        input: question,
+        dimensions: 1536
+      })
+    });
+    if (!embeddingRes.ok) {
+      const err = await embeddingRes.text();
+      console.error('[engram-bridge:direct] embedding failed:', err);
+      throw new Error('Embedding generation failed');
+    }
+    const embeddingData = await embeddingRes.json();
+    const embedding = embeddingData.data[0].embedding;
+
+    const searchRes = await fetch(`${supabaseUrl}/rest/v1/rpc/memory_hybrid_search`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'apikey': supabaseKey,
+        'Authorization': `Bearer ${supabaseKey}`
+      },
+      body: JSON.stringify({
+        query_text: question,
+        query_embedding: `[${embedding.join(',')}]`,
+        match_count: 10,
+        full_text_weight: 1.0,
+        semantic_weight: 1.0,
+        rrf_k: 60,
+        filter_project: searchAll ? null : (project || null),
+        filter_source_type: null,
+        recency_weight: 0.15,
+        decay_days: 30.0
+      })
+    });
+    if (!searchRes.ok) {
+      const err = await searchRes.text();
+      console.error('[engram-bridge:direct] supabase search failed:', err);
+      throw new Error('Memory search failed');
+    }
+    const rows = await searchRes.json();
+    return {
+      memories: rows.map((m) => ({
+        content: m.content,
+        source_type: m.source_type,
+        project: m.project,
+        similarity: m.similarity,
+        created_at: m.created_at
+      })),
+      total: rows.length
+    };
+  }
+
+  async function queryWebhook({ question, project, searchAll }) {
+    const url = config.rag?.engramWebhookUrl || 'http://localhost:37778/engram';
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        op: 'recall',
+        question,
+        project: searchAll ? null : (project || null),
+        min_results: 5
+      })
+    });
+    if (!res.ok) {
+      const err = await res.text();
+      console.error('[engram-bridge:webhook] request failed:', err);
+      throw new Error(`Engram webhook returned ${res.status}`);
+    }
+    const data = await res.json();
+    const rows = data.memories || [];
+    return {
+      memories: rows.map((m) => ({
+        content: m.content,
+        source_type: m.source_type,
+        project: m.project,
+        similarity: m.similarity ?? m.score ?? null,
+        created_at: m.created_at
+      })),
+      total: rows.length
+    };
+  }
+
+  function ensureMcpChild() {
+    if (state.mcpChild && !state.mcpChild.killed) return state.mcpChild;
+
+    const bin = config.rag?.engramBinary || 'engram';
+    const child = spawn(bin, ['serve', '--stdio'], { stdio: ['pipe', 'pipe', 'pipe'] });
+    state.mcpChild = child;
+    state.mcpBuffer = '';
+
+    child.stdout.on('data', (chunk) => {
+      state.mcpBuffer += chunk.toString('utf-8');
+      let idx;
+      while ((idx = state.mcpBuffer.indexOf('\n')) >= 0) {
+        const line = state.mcpBuffer.slice(0, idx).trim();
+        state.mcpBuffer = state.mcpBuffer.slice(idx + 1);
+        if (!line) continue;
+        try {
+          const msg = JSON.parse(line);
+          const pending = state.mcpQueue.find((p) => p.id === msg.id);
+          if (pending) {
+            state.mcpQueue = state.mcpQueue.filter((p) => p !== pending);
+            if (msg.error) pending.reject(new Error(msg.error.message || 'Engram MCP error'));
+            else pending.resolve(msg.result);
+          }
+        } catch (err) {
+          console.error('[engram-bridge:mcp] parse error:', err.message, line);
+        }
+      }
+    });
+
+    child.stderr.on('data', (chunk) => {
+      console.error('[engram-bridge:mcp]', chunk.toString('utf-8').trim());
+    });
+
+    child.on('exit', (code, signal) => {
+      console.warn(`[engram-bridge:mcp] child exited (code=${code}, signal=${signal}); will respawn on next call`);
+      state.mcpChild = null;
+      for (const pending of state.mcpQueue) {
+        pending.reject(new Error('Engram MCP child exited'));
+      }
+      state.mcpQueue = [];
+    });
+
+    return child;
+  }
+
+  function mcpCall(method, params) {
+    const child = ensureMcpChild();
+    const id = state.mcpNextId++;
+    const req = { jsonrpc: '2.0', id, method, params };
+    return new Promise((resolve, reject) => {
+      state.mcpQueue.push({ id, resolve, reject });
+      try {
+        child.stdin.write(JSON.stringify(req) + '\n');
+      } catch (err) {
+        state.mcpQueue = state.mcpQueue.filter((p) => p.id !== id);
+        reject(err);
+      }
+      // Safety timeout
+      setTimeout(() => {
+        const pending = state.mcpQueue.find((p) => p.id === id);
+        if (pending) {
+          state.mcpQueue = state.mcpQueue.filter((p) => p !== pending);
+          pending.reject(new Error('Engram MCP call timed out'));
+        }
+      }, 15000);
+    });
+  }
+
+  async function queryMcp({ question, project, searchAll }) {
+    try {
+      const result = await mcpCall('tools/call', {
+        name: 'memory_recall',
+        arguments: {
+          query: question,
+          project: searchAll ? null : (project || null),
+          match_count: 10
+        }
+      });
+      const rows = (result && (result.memories || result.content || [])) || [];
+      return {
+        memories: rows.map((m) => ({
+          content: m.content,
+          source_type: m.source_type,
+          project: m.project,
+          similarity: m.similarity ?? m.score ?? null,
+          created_at: m.created_at
+        })),
+        total: rows.length
+      };
+    } catch (err) {
+      // Kill child so it respawns next call
+      if (state.mcpChild) {
+        try { state.mcpChild.kill(); } catch {}
+        state.mcpChild = null;
+      }
+      throw err;
+    }
+  }
+
+  async function queryEngram({ question, project, searchAll }) {
+    switch (mode) {
+      case 'webhook':
+        return queryWebhook({ question, project, searchAll });
+      case 'mcp':
+        return queryMcp({ question, project, searchAll });
+      case 'direct':
+      default:
+        return queryDirect({ question, project, searchAll });
+    }
+  }
+
+  return { mode, queryEngram };
+}
+
+module.exports = { createBridge };