@jhizzard/termdeck 0.2.0 → 0.2.2

package/packages/server/src/{engram-bridge → mnestra-bridge}/index.js

@@ -1,7 +1,7 @@
-// Engram bridge — routes TermDeck memory queries through one of three backends:
+// Mnestra bridge — routes TermDeck memory queries through one of three backends:
 //   - direct:  talk to Supabase + OpenAI from the server (pre-bridge behavior)
-//   - webhook: POST to Engram's HTTP webhook server (T3.1) at rag.engramWebhookUrl
-//   - mcp:     spawn the @jhizzard/engram binary and talk JSON-RPC over stdio
+//   - webhook: POST to Mnestra's HTTP webhook server (T3.1) at rag.mnestraWebhookUrl
+//   - mcp:     spawn the @jhizzard/mnestra binary and talk JSON-RPC over stdio
 //
 // All three modes return the same shape:
 //   { memories: Array<{ content, source_type, project, similarity, created_at }>, total }
@@ -11,7 +11,7 @@
 const { spawn } = require('child_process');
 
 function createBridge(config) {
-  const mode = config.rag?.engramMode || 'direct';
+  const mode = config.rag?.mnestraMode || 'direct';
   const state = { mcpChild: null, mcpQueue: [], mcpNextId: 1, mcpBuffer: '' };
 
   async function queryDirect({ question, project, searchAll }) {
@@ -40,7 +40,7 @@ function createBridge(config) {
     });
     if (!embeddingRes.ok) {
       const err = await embeddingRes.text();
-      console.error('[engram-bridge:direct] embedding failed:', err);
+      console.error('[mnestra-bridge:direct] embedding failed:', err);
       throw new Error('Embedding generation failed');
     }
     const embeddingData = await embeddingRes.json();
@@ -68,7 +68,7 @@ function createBridge(config) {
     });
     if (!searchRes.ok) {
       const err = await searchRes.text();
-      console.error('[engram-bridge:direct] supabase search failed:', err);
+      console.error('[mnestra-bridge:direct] supabase search failed:', err);
       throw new Error('Memory search failed');
     }
     const rows = await searchRes.json();
@@ -85,7 +85,7 @@ function createBridge(config) {
   }
 
   async function queryWebhook({ question, project, searchAll }) {
-    const url = config.rag?.engramWebhookUrl || 'http://localhost:37778/engram';
+    const url = config.rag?.mnestraWebhookUrl || 'http://localhost:37778/mnestra';
     const res = await fetch(url, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
@@ -98,8 +98,8 @@ function createBridge(config) {
     });
     if (!res.ok) {
       const err = await res.text();
-      console.error('[engram-bridge:webhook] request failed:', err);
-      throw new Error(`Engram webhook returned ${res.status}`);
+      console.error('[mnestra-bridge:webhook] request failed:', err);
+      throw new Error(`Mnestra webhook returned ${res.status}`);
     }
     const data = await res.json();
     const rows = data.memories || [];
@@ -118,7 +118,7 @@ function createBridge(config) {
   function ensureMcpChild() {
     if (state.mcpChild && !state.mcpChild.killed) return state.mcpChild;
 
-    const bin = config.rag?.engramBinary || 'engram';
+    const bin = config.rag?.mnestraBinary || 'mnestra';
     const child = spawn(bin, ['serve', '--stdio'], { stdio: ['pipe', 'pipe', 'pipe'] });
     state.mcpChild = child;
     state.mcpBuffer = '';
@@ -135,24 +135,24 @@ function createBridge(config) {
           const pending = state.mcpQueue.find((p) => p.id === msg.id);
           if (pending) {
             state.mcpQueue = state.mcpQueue.filter((p) => p !== pending);
-            if (msg.error) pending.reject(new Error(msg.error.message || 'Engram MCP error'));
+            if (msg.error) pending.reject(new Error(msg.error.message || 'Mnestra MCP error'));
             else pending.resolve(msg.result);
           }
         } catch (err) {
-          console.error('[engram-bridge:mcp] parse error:', err.message, line);
+          console.error('[mnestra-bridge:mcp] parse error:', err.message, line);
         }
       }
     });
 
     child.stderr.on('data', (chunk) => {
-      console.error('[engram-bridge:mcp]', chunk.toString('utf-8').trim());
+      console.error('[mnestra-bridge:mcp]', chunk.toString('utf-8').trim());
     });
 
     child.on('exit', (code, signal) => {
-      console.warn(`[engram-bridge:mcp] child exited (code=${code}, signal=${signal}); will respawn on next call`);
+      console.warn(`[mnestra-bridge:mcp] child exited (code=${code}, signal=${signal}); will respawn on next call`);
       state.mcpChild = null;
       for (const pending of state.mcpQueue) {
-        pending.reject(new Error('Engram MCP child exited'));
+        pending.reject(new Error('Mnestra MCP child exited'));
       }
       state.mcpQueue = [];
     });
@@ -177,7 +177,7 @@ function createBridge(config) {
         const pending = state.mcpQueue.find((p) => p.id === id);
         if (pending) {
           state.mcpQueue = state.mcpQueue.filter((p) => p !== pending);
-          pending.reject(new Error('Engram MCP call timed out'));
+          pending.reject(new Error('Mnestra MCP call timed out'));
         }
       }, 15000);
     });
@@ -214,7 +214,7 @@ function createBridge(config) {
     }
   }
 
-  async function queryEngram({ question, project, searchAll }) {
+  async function queryMnestra({ question, project, searchAll }) {
     switch (mode) {
       case 'webhook':
         return queryWebhook({ question, project, searchAll });
@@ -226,7 +226,7 @@ function createBridge(config) {
     }
   }
 
-  return { mode, queryEngram };
+  return { mode, queryMnestra };
 }
 
 module.exports = { createBridge };

package/packages/server/src/rag.js

@@ -16,10 +16,10 @@ class RAGIntegration {
 
     // Table configuration matching Josh's multi-layer schema
     this.tables = {
-      sessionMemory: config.rag?.tables?.session || 'engram_session_memory',
-      projectMemory: config.rag?.tables?.project || 'engram_project_memory',
-      developerMemory: config.rag?.tables?.developer || 'engram_developer_memory',
-      commandLog: config.rag?.tables?.commands || 'engram_commands'
+      sessionMemory: config.rag?.tables?.session || 'mnestra_session_memory',
+      projectMemory: config.rag?.tables?.project || 'mnestra_project_memory',
+      developerMemory: config.rag?.tables?.developer || 'mnestra_developer_memory',
+      commandLog: config.rag?.tables?.commands || 'mnestra_commands'
     };
 
     if (this.enabled) {
@@ -120,7 +120,7 @@ class RAGIntegration {
       }
     } catch (err) {
       // Will be retried by sync loop
-      console.error('[engram] Push failed:', err.message);
+      console.error('[mnestra] Push failed:', err.message);
     }
   }
 
@@ -163,7 +163,7 @@ class RAGIntegration {
           markRagEventsSynced(this.db, synced);
         }
       } catch (err) {
-        console.error('[engram] Sync cycle error:', err.message);
+        console.error('[mnestra] Sync cycle error:', err.message);
       }
     }, this.syncInterval);
   }

package/packages/server/src/setup/dotenv-io.js

@@ -0,0 +1,116 @@
+// Merge-aware reader/writer for ~/.termdeck/secrets.env.
+//
+// The existing config.js module already parses the file at load time, but for
+// the `init` wizards we need to UPDATE the file without clobbering values the
+// user has already set. This helper preserves unknown keys, preserves order
+// of existing keys, and appends new keys at the bottom.
+//
+// File format (same subset as config.js parseDotenv):
+//   KEY=value
+//   KEY="quoted"
+//   KEY='single'
+//   # comments are preserved
+//   blank lines are preserved
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const SECRETS_PATH = path.join(os.homedir(), '.termdeck', 'secrets.env');
+
+function readSecretsRaw(filepath = SECRETS_PATH) {
+  if (!fs.existsSync(filepath)) return { exists: false, lines: [], keys: {} };
+  const raw = fs.readFileSync(filepath, 'utf-8');
+  const lines = raw.split(/\r?\n/);
+  const keys = {};
+  lines.forEach((line, idx) => {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) return;
+    const eq = trimmed.indexOf('=');
+    if (eq === -1) return;
+    const key = trimmed.slice(0, eq).trim();
+    let val = trimmed.slice(eq + 1).trim();
+    if ((val.startsWith('"') && val.endsWith('"')) ||
+        (val.startsWith("'") && val.endsWith("'"))) {
+      val = val.slice(1, -1);
+    }
+    keys[key] = { value: val, lineIndex: idx };
+  });
+  return { exists: true, lines, keys };
+}
+
+// Escape a value for safe re-serialization. Wraps in double quotes if the
+// value contains whitespace, `#`, or `"`. Always safe to wrap — we wrap when
+// in doubt to avoid ambiguity with the dotenv parser.
+function formatValue(value) {
+  if (value == null) return '';
+  const str = String(value);
+  if (str === '') return '';
+  const needsQuoting = /[\s#"'=]/.test(str);
+  if (!needsQuoting) return str;
+  const escaped = str.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+  return `"${escaped}"`;
+}
+
+// Write a merged secrets.env. `updates` is an object of key→value pairs. Pass
+// null/undefined value to delete a key. Existing lines are preserved for keys
+// not listed in `updates`. New keys get appended to the bottom with a header.
+function writeSecrets(updates, filepath = SECRETS_PATH) {
+  const dir = path.dirname(filepath);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+
+  const { exists, lines, keys } = readSecretsRaw(filepath);
+  const originalLines = exists ? lines.slice() : [
+    '# TermDeck secrets — loaded on server startup.',
+    '# Never commit this file.',
+    ''
+  ];
+  const workingLines = originalLines.slice();
+  const toAppend = [];
+
+  for (const [key, rawVal] of Object.entries(updates)) {
+    if (rawVal == null || rawVal === '') {
+      // Delete existing line (by overwriting with blank). Safer to mark for
+      // deletion than splice (splice shifts indices of subsequent keys).
+      if (keys[key]) workingLines[keys[key].lineIndex] = '';
+      continue;
+    }
+    const formatted = `${key}=${formatValue(rawVal)}`;
+    if (keys[key]) {
+      workingLines[keys[key].lineIndex] = formatted;
+    } else {
+      toAppend.push(formatted);
+    }
+  }
+
+  let out = workingLines.join('\n');
+  if (toAppend.length > 0) {
+    if (out.length > 0 && !out.endsWith('\n')) out += '\n';
+    // Add a header for the first append-pass on an empty file.
+    if (!exists) out = originalLines.join('\n') + toAppend.join('\n') + '\n';
+    else out = out.replace(/\n+$/, '') + '\n' + toAppend.join('\n') + '\n';
+  } else if (!out.endsWith('\n')) {
+    out += '\n';
+  }
+
+  // chmod 600 for secrets — owner read/write only.
+  fs.writeFileSync(filepath, out, { encoding: 'utf-8', mode: 0o600 });
+  try { fs.chmodSync(filepath, 0o600); } catch (_err) { /* best-effort */ }
+
+  return { path: filepath, wrote: Object.keys(updates).length, appended: toAppend.length };
+}
+
+// Convenience read — returns `{ SUPABASE_URL, ... }` object, no metadata.
+function readSecrets(filepath = SECRETS_PATH) {
+  const { keys } = readSecretsRaw(filepath);
+  const out = {};
+  for (const [k, v] of Object.entries(keys)) out[k] = v.value;
+  return out;
+}
+
+module.exports = {
+  SECRETS_PATH,
+  readSecrets,
+  writeSecrets,
+  _formatValue: formatValue
+};

package/packages/server/src/setup/engram-migrations/001_engram_tables.sql

@@ -0,0 +1,116 @@
+-- Engram v0.1 — core tables
+--
+-- Run against a Postgres 15+ database that has pgvector installed
+-- (Supabase already ships with it). Apply in order:
+--   001_engram_tables.sql
+--   002_engram_search_function.sql
+--   003_engram_event_webhook.sql
+
+create extension if not exists "vector";
+create extension if not exists "pg_trgm";
+create extension if not exists "pgcrypto";
+
+-- ── memory_items ──────────────────────────────────────────────────────────
+
+create table if not exists memory_items (
+  id              uuid primary key default gen_random_uuid(),
+  content         text not null,
+  embedding       vector(1536),
+  source_type     text not null default 'fact',
+  category        text,
+  project         text not null default 'global',
+  metadata        jsonb not null default '{}'::jsonb,
+  is_active       boolean not null default true,
+  archived        boolean not null default false,
+  superseded_by   uuid references memory_items(id) on delete set null,
+  created_at      timestamptz not null default now(),
+  updated_at      timestamptz not null default now()
+);
+
+create index if not exists memory_items_project_idx
+  on memory_items(project) where is_active = true and archived = false;
+
+create index if not exists memory_items_source_type_idx
+  on memory_items(source_type) where is_active = true and archived = false;
+
+create index if not exists memory_items_created_at_idx
+  on memory_items(created_at desc);
+
+create index if not exists memory_items_content_trgm_idx
+  on memory_items using gin (content gin_trgm_ops);
+
+-- HNSW vector index. If your Postgres/pgvector is too old for HNSW,
+-- swap to ivfflat:
+--   create index memory_items_embedding_idx on memory_items
+--     using ivfflat (embedding vector_cosine_ops) with (lists = 100);
+create index if not exists memory_items_embedding_hnsw_idx
+  on memory_items using hnsw (embedding vector_cosine_ops)
+  with (m = 16, ef_construction = 64);
+
+-- ── memory_sessions ───────────────────────────────────────────────────────
+
+create table if not exists memory_sessions (
+  id          uuid primary key default gen_random_uuid(),
+  project     text not null default 'global',
+  summary     text,
+  metadata    jsonb not null default '{}'::jsonb,
+  created_at  timestamptz not null default now()
+);
+
+create index if not exists memory_sessions_project_idx on memory_sessions(project);
+create index if not exists memory_sessions_created_at_idx on memory_sessions(created_at desc);
+
+-- ── memory_relationships ──────────────────────────────────────────────────
+
+create table if not exists memory_relationships (
+  id                uuid primary key default gen_random_uuid(),
+  source_id         uuid not null references memory_items(id) on delete cascade,
+  target_id         uuid not null references memory_items(id) on delete cascade,
+  relationship_type text not null,
+  created_at        timestamptz not null default now(),
+  unique (source_id, target_id, relationship_type),
+  check (source_id <> target_id),
+  check (relationship_type in ('supersedes','relates_to','contradicts','elaborates','caused_by'))
+);
+
+create index if not exists memory_relationships_source_idx on memory_relationships(source_id);
+create index if not exists memory_relationships_target_idx on memory_relationships(target_id);
+
+-- ── match_memories helper RPC ─────────────────────────────────────────────
+-- Used by remember.ts (dedup) and consolidate.ts (cluster seeding).
+
+create or replace function match_memories (
+  query_embedding   vector(1536),
+  match_threshold   float,
+  match_count       int,
+  filter_project    text default null
+)
+returns table (
+  id          uuid,
+  content     text,
+  source_type text,
+  category    text,
+  project     text,
+  metadata    jsonb,
+  similarity  float
+)
+language sql stable
+as $$
+  select
+    m.id,
+    m.content,
+    m.source_type,
+    m.category,
+    m.project,
+    m.metadata,
+    1 - (m.embedding <=> query_embedding) as similarity
+  from memory_items m
+  where m.is_active = true
+    and m.archived = false
+    and m.superseded_by is null
+    and m.embedding is not null
+    and (filter_project is null or m.project = filter_project)
+    and 1 - (m.embedding <=> query_embedding) >= match_threshold
+  order by m.embedding <=> query_embedding asc
+  limit match_count;
+$$;

package/packages/server/src/setup/engram-migrations/002_engram_search_function.sql

@@ -0,0 +1,141 @@
+-- Engram v0.1 — memory_hybrid_search
+--
+-- Reciprocal rank fusion over full-text and semantic search, with the
+-- three SQL-side fixes from RAG-MEMORY-IMPROVEMENTS-AND-TERMDECK-STRATEGY.md:
+--
+--   Fix 1 — Tiered recency decay by source_type. Architectural decisions
+--           decay on a one-year half-life; bug fixes on a 30-day half-life;
+--           session summaries and document chunks on a 14-day half-life.
+--
+--   Fix 3 — Source_type weighting. Decisions and architecture outrank
+--           raw document chunks in the final fused score.
+--
+--   Fix 5 — Project affinity scoring. Exact project match multiplies the
+--           score by 1.5x; mismatches are penalised 0.7x.
+
+create or replace function memory_hybrid_search (
+  query_text          text,
+  query_embedding     vector(1536),
+  match_count         int default 20,
+  full_text_weight    float default 1.0,
+  semantic_weight     float default 1.0,
+  rrf_k               int default 60,
+  filter_project      text default null,
+  filter_source_type  text default null
+)
+returns table (
+  id          uuid,
+  content     text,
+  source_type text,
+  category    text,
+  project     text,
+  metadata    jsonb,
+  score       float,
+  created_at  timestamptz
+)
+language sql stable
+as $$
+with candidates as (
+  select
+    m.id,
+    m.content,
+    m.source_type,
+    m.category,
+    m.project,
+    m.metadata,
+    m.created_at,
+    m.embedding,
+    ts_rank_cd(to_tsvector('english', m.content), plainto_tsquery('english', query_text))
+      as ft_rank,
+    1 - (m.embedding <=> query_embedding) as sem_rank,
+    extract(epoch from (now() - m.created_at))::float as age_seconds
+  from memory_items m
+  where m.is_active = true
+    and m.archived = false
+    and m.superseded_by is null
+    and m.embedding is not null
+    and (filter_project is null or m.project = filter_project)
+    and (filter_source_type is null or m.source_type = filter_source_type)
+),
+ft_ranked as (
+  select id, row_number() over (order by ft_rank desc nulls last) as rank
+  from candidates where ft_rank > 0
+),
+sem_ranked as (
+  select id, row_number() over (order by sem_rank desc nulls last) as rank
+  from candidates
+),
+fused as (
+  select
+    c.id,
+    c.content,
+    c.source_type,
+    c.category,
+    c.project,
+    c.metadata,
+    c.created_at,
+    c.age_seconds,
+    -- RRF base score
+    coalesce(full_text_weight / (rrf_k + ft.rank), 0.0) +
+    coalesce(semantic_weight  / (rrf_k + sr.rank), 0.0) as base_score
+  from candidates c
+  left join ft_ranked  ft on ft.id = c.id
+  left join sem_ranked sr on sr.id = c.id
+),
+scored as (
+  select
+    f.id,
+    f.content,
+    f.source_type,
+    f.category,
+    f.project,
+    f.metadata,
+    f.created_at,
+    f.base_score
+      -- Fix 1: tiered recency decay by source_type
+      * case f.source_type
+          when 'decision'        then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'architecture'    then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'preference'      then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'fact'            then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'convention'      then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'bug_fix'         then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'debugging'       then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'session_summary' then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'document_chunk'  then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'code_context'    then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          else                         1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+        end
+      -- Fix 3: source_type weighting
+      * case f.source_type
+          when 'decision'       then 1.5
+          when 'architecture'   then 1.4
+          when 'bug_fix'        then 1.3
+          when 'preference'     then 1.2
+          when 'fact'           then 1.0
+          when 'document_chunk' then 0.6
+          else                       1.0
+        end
+      -- Fix 5: project affinity scoring
+      * case
+          when filter_project is null then 1.0
+          when f.project = filter_project then 1.5
+          when f.project = 'global' then 1.0
+          else 0.7
+        end
+      as score
+  from fused f
+)
+select
+  s.id,
+  s.content,
+  s.source_type,
+  s.category,
+  s.project,
+  s.metadata,
+  s.score,
+  s.created_at
+from scored s
+order by s.score desc
+limit match_count;
+$$;

package/packages/server/src/setup/engram-migrations/003_engram_event_webhook.sql

@@ -0,0 +1,28 @@
+-- Engram v0.1 — real-time event webhook (Fix 6)
+--
+-- Fix 6 from RAG-MEMORY-IMPROVEMENTS-AND-TERMDECK-STRATEGY.md is a
+-- real-time event intake path so TermDeck (or any other client) can
+-- POST terminal events — "server started on :8080", "tests failing",
+-- "error detected" — and have them land in memory immediately.
+--
+-- That intake is implemented as an HTTP endpoint inside the Engram MCP
+-- server process, not as a SQL trigger. This file exists as a placeholder
+-- so the migration history is explicit and future database-side changes
+-- (e.g. an events queue table for async ingestion) have a home.
+--
+-- If you want to add a durable event queue later, add it below this line.
+
+-- Example future shape (commented out — not applied):
+--
+--   create table if not exists memory_events (
+--     id         uuid primary key default gen_random_uuid(),
+--     project    text not null,
+--     source     text not null,
+--     event_type text not null,
+--     payload    jsonb not null default '{}'::jsonb,
+--     processed  boolean not null default false,
+--     created_at timestamptz not null default now()
+--   );
+--
+--   create index if not exists memory_events_unprocessed_idx
+--     on memory_events(created_at) where processed = false;