@jhizzard/termdeck 0.2.0 → 0.2.1

package/packages/server/src/setup/engram-migrations/002_engram_search_function.sql

@@ -0,0 +1,141 @@
+-- Engram v0.1 — memory_hybrid_search
+--
+-- Reciprocal rank fusion over full-text and semantic search, with the
+-- three SQL-side fixes from RAG-MEMORY-IMPROVEMENTS-AND-TERMDECK-STRATEGY.md:
+--
+--   Fix 1 — Tiered recency decay by source_type. Architectural decisions
+--           decay on a one-year half-life; bug fixes on a 30-day half-life;
+--           session summaries and document chunks on a 14-day half-life.
+--
+--   Fix 3 — Source_type weighting. Decisions and architecture outrank
+--           raw document chunks in the final fused score.
+--
+--   Fix 5 — Project affinity scoring. Exact project match multiplies the
+--           score by 1.5x; mismatches are penalised 0.7x.
+
+create or replace function memory_hybrid_search (
+  query_text          text,
+  query_embedding     vector(1536),
+  match_count         int default 20,
+  full_text_weight    float default 1.0,
+  semantic_weight     float default 1.0,
+  rrf_k               int default 60,
+  filter_project      text default null,
+  filter_source_type  text default null
+)
+returns table (
+  id          uuid,
+  content     text,
+  source_type text,
+  category    text,
+  project     text,
+  metadata    jsonb,
+  score       float,
+  created_at  timestamptz
+)
+language sql stable
+as $$
+with candidates as (
+  select
+    m.id,
+    m.content,
+    m.source_type,
+    m.category,
+    m.project,
+    m.metadata,
+    m.created_at,
+    m.embedding,
+    ts_rank_cd(to_tsvector('english', m.content), plainto_tsquery('english', query_text))
+      as ft_rank,
+    1 - (m.embedding <=> query_embedding) as sem_rank,
+    extract(epoch from (now() - m.created_at))::float as age_seconds
+  from memory_items m
+  where m.is_active = true
+    and m.archived = false
+    and m.superseded_by is null
+    and m.embedding is not null
+    and (filter_project is null or m.project = filter_project)
+    and (filter_source_type is null or m.source_type = filter_source_type)
+),
+ft_ranked as (
+  select id, row_number() over (order by ft_rank desc nulls last) as rank
+  from candidates where ft_rank > 0
+),
+sem_ranked as (
+  select id, row_number() over (order by sem_rank desc nulls last) as rank
+  from candidates
+),
+fused as (
+  select
+    c.id,
+    c.content,
+    c.source_type,
+    c.category,
+    c.project,
+    c.metadata,
+    c.created_at,
+    c.age_seconds,
+    -- RRF base score
+    coalesce(full_text_weight / (rrf_k + ft.rank), 0.0) +
+    coalesce(semantic_weight  / (rrf_k + sr.rank), 0.0) as base_score
+  from candidates c
+  left join ft_ranked  ft on ft.id = c.id
+  left join sem_ranked sr on sr.id = c.id
+),
+scored as (
+  select
+    f.id,
+    f.content,
+    f.source_type,
+    f.category,
+    f.project,
+    f.metadata,
+    f.created_at,
+    f.base_score
+      -- Fix 1: tiered recency decay by source_type
+      * case f.source_type
+          when 'decision'        then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'architecture'    then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'preference'      then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'fact'            then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'convention'      then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'bug_fix'         then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'debugging'       then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'session_summary' then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'document_chunk'  then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'code_context'    then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          else                         1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+        end
+      -- Fix 3: source_type weighting
+      * case f.source_type
+          when 'decision'       then 1.5
+          when 'architecture'   then 1.4
+          when 'bug_fix'        then 1.3
+          when 'preference'     then 1.2
+          when 'fact'           then 1.0
+          when 'document_chunk' then 0.6
+          else                       1.0
+        end
+      -- Fix 5: project affinity scoring
+      * case
+          when filter_project is null then 1.0
+          when f.project = filter_project then 1.5
+          when f.project = 'global' then 1.0
+          else 0.7
+        end
+      as score
+  from fused f
+)
+select
+  s.id,
+  s.content,
+  s.source_type,
+  s.category,
+  s.project,
+  s.metadata,
+  s.score,
+  s.created_at
+from scored s
+order by s.score desc
+limit match_count;
+$$;

package/packages/server/src/setup/engram-migrations/003_engram_event_webhook.sql

@@ -0,0 +1,28 @@
+-- Engram v0.1 — real-time event webhook (Fix 6)
+--
+-- Fix 6 from RAG-MEMORY-IMPROVEMENTS-AND-TERMDECK-STRATEGY.md is a
+-- real-time event intake path so TermDeck (or any other client) can
+-- POST terminal events — "server started on :8080", "tests failing",
+-- "error detected" — and have them land in memory immediately.
+--
+-- That intake is implemented as an HTTP endpoint inside the Engram MCP
+-- server process, not as a SQL trigger. This file exists as a placeholder
+-- so the migration history is explicit and future database-side changes
+-- (e.g. an events queue table for async ingestion) have a home.
+--
+-- If you want to add a durable event queue later, add it below this line.
+
+-- Example future shape (commented out — not applied):
+--
+--   create table if not exists memory_events (
+--     id         uuid primary key default gen_random_uuid(),
+--     project    text not null,
+--     source     text not null,
+--     event_type text not null,
+--     payload    jsonb not null default '{}'::jsonb,
+--     processed  boolean not null default false,
+--     created_at timestamptz not null default now()
+--   );
+--
+--   create index if not exists memory_events_unprocessed_idx
+--     on memory_events(created_at) where processed = false;

package/packages/server/src/setup/engram-migrations/004_engram_match_count_cap_and_explain.sql

@@ -0,0 +1,176 @@
+-- Engram v0.2 — match_count cap + EXPLAIN variant
+--
+-- Two changes to the search surface:
+--
+--   1. memory_hybrid_search gains a configurable cap on `match_count`.
+--      Default cap: 200. The original function was unbounded, which risks
+--      runaway queries at scale (10k+ rows pulled per call).
+--
+--      Override per-database:   ALTER DATABASE your_db SET engram.max_match_count = 500;
+--      Override per-session:    SET engram.max_match_count = 500;
+--      Leave unset:             cap defaults to 200.
+--
+--   2. A new function `memory_hybrid_search_explain` that returns
+--      EXPLAIN (ANALYZE, BUFFERS) output for an equivalent call. Used by
+--      `engram diagnose` to troubleshoot slow recall queries.
+--
+-- Rerun-safe: CREATE OR REPLACE on both.
+
+-- ── memory_hybrid_search ─────────────────────────────────────────────────
+
+create or replace function memory_hybrid_search (
+  query_text          text,
+  query_embedding     vector(1536),
+  match_count         int default 20,
+  full_text_weight    float default 1.0,
+  semantic_weight     float default 1.0,
+  rrf_k               int default 60,
+  filter_project      text default null,
+  filter_source_type  text default null
+)
+returns table (
+  id          uuid,
+  content     text,
+  source_type text,
+  category    text,
+  project     text,
+  metadata    jsonb,
+  score       float,
+  created_at  timestamptz
+)
+language sql stable
+as $$
+with candidates as (
+  select
+    m.id,
+    m.content,
+    m.source_type,
+    m.category,
+    m.project,
+    m.metadata,
+    m.created_at,
+    m.embedding,
+    ts_rank_cd(to_tsvector('english', m.content), plainto_tsquery('english', query_text))
+      as ft_rank,
+    1 - (m.embedding <=> query_embedding) as sem_rank,
+    extract(epoch from (now() - m.created_at))::float as age_seconds
+  from memory_items m
+  where m.is_active = true
+    and m.archived = false
+    and m.superseded_by is null
+    and m.embedding is not null
+    and (filter_project is null or m.project = filter_project)
+    and (filter_source_type is null or m.source_type = filter_source_type)
+),
+ft_ranked as (
+  select id, row_number() over (order by ft_rank desc nulls last) as rank
+  from candidates where ft_rank > 0
+),
+sem_ranked as (
+  select id, row_number() over (order by sem_rank desc nulls last) as rank
+  from candidates
+),
+fused as (
+  select
+    c.id,
+    c.content,
+    c.source_type,
+    c.category,
+    c.project,
+    c.metadata,
+    c.created_at,
+    c.age_seconds,
+    coalesce(full_text_weight / (rrf_k + ft.rank), 0.0) +
+    coalesce(semantic_weight  / (rrf_k + sr.rank), 0.0) as base_score
+  from candidates c
+  left join ft_ranked  ft on ft.id = c.id
+  left join sem_ranked sr on sr.id = c.id
+),
+scored as (
+  select
+    f.id,
+    f.content,
+    f.source_type,
+    f.category,
+    f.project,
+    f.metadata,
+    f.created_at,
+    f.base_score
+      * case f.source_type
+          when 'decision'        then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'architecture'    then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'preference'      then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'fact'            then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'convention'      then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'bug_fix'         then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'debugging'       then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'session_summary' then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'document_chunk'  then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'code_context'    then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          else                         1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+        end
+      * case f.source_type
+          when 'decision'       then 1.5
+          when 'architecture'   then 1.4
+          when 'bug_fix'        then 1.3
+          when 'preference'     then 1.2
+          when 'fact'           then 1.0
+          when 'document_chunk' then 0.6
+          else                       1.0
+        end
+      * case
+          when filter_project is null then 1.0
+          when f.project = filter_project then 1.5
+          when f.project = 'global' then 1.0
+          else 0.7
+        end
+      as score
+  from fused f
+)
+select
+  s.id,
+  s.content,
+  s.source_type,
+  s.category,
+  s.project,
+  s.metadata,
+  s.score,
+  s.created_at
+from scored s
+order by s.score desc
+limit least(
+  greatest(match_count, 1),
+  coalesce(nullif(current_setting('engram.max_match_count', true), '')::int, 200)
+);
+$$;
+
+-- ── memory_hybrid_search_explain ─────────────────────────────────────────
+
+create or replace function memory_hybrid_search_explain (
+  query_text          text,
+  query_embedding     vector(1536),
+  match_count         int default 20,
+  full_text_weight    float default 1.0,
+  semantic_weight     float default 1.0,
+  rrf_k               int default 60,
+  filter_project      text default null,
+  filter_source_type  text default null
+)
+returns setof text
+language plpgsql
+as $$
+begin
+  return query execute
+    'explain (analyze, buffers, format text) '
+    || 'select * from memory_hybrid_search($1, $2, $3, $4, $5, $6, $7, $8)'
+  using
+    query_text,
+    query_embedding,
+    match_count,
+    full_text_weight,
+    semantic_weight,
+    rrf_k,
+    filter_project,
+    filter_source_type;
+end;
+$$;

package/packages/server/src/setup/engram-migrations/005_v0_1_to_v0_2_upgrade.sql

@@ -0,0 +1,23 @@
+-- Engram v0.2 — minimal additive delta for stores provisioned against the
+-- original rag-system schema.
+--
+-- This migration is idempotent and non-destructive:
+--   • adds only the single column (`archived`) that Engram v0.2 reads/writes
+--     but which is absent from the original rag-system `memory_items` table;
+--   • re-creates the two partial indexes under v2 names so they cannot
+--     collide with any pre-existing same-named index that uses a different
+--     `where` predicate;
+--   • does NOT replace `memory_hybrid_search`, `match_memories`, or any
+--     other SQL function — those remain on their production versions.
+--
+-- Apply once, in the Supabase SQL editor, against any existing store that
+-- pre-dates Engram v0.2's `archived` soft-delete column.
+
+alter table memory_items
+  add column if not exists archived boolean not null default false;
+
+create index if not exists memory_items_project_idx_v2
+  on memory_items(project) where is_active = true and archived = false;
+
+create index if not exists memory_items_source_type_idx_v2
+  on memory_items(source_type) where is_active = true and archived = false;

package/packages/server/src/setup/engram-migrations/006_memory_status_rpc.sql

@@ -0,0 +1,58 @@
+-- Engram migration 006 — memory_status_aggregation RPC
+--
+-- Why: `memoryStatus()` previously did a plain
+--   supabase.from('memory_items').select('project, source_type, category')
+-- which hits PostgREST's default 1000-row cap. On a store with 3,397 active
+-- rows the `by_project` / `by_source_type` / `by_category` histograms only
+-- summed to ~1000 even though `total_active` was correct. Pushing the GROUP
+-- BY server-side eliminates the cap and saves the round-trip of streaming
+-- every row to the client just to count them.
+--
+-- Safe to re-run — CREATE OR REPLACE.
+
+create or replace function memory_status_aggregation()
+returns table (
+  total_active   bigint,
+  sessions       bigint,
+  by_project     jsonb,
+  by_source_type jsonb,
+  by_category    jsonb
+)
+language sql
+stable
+as $$
+  select
+    (select count(*)::bigint from memory_items
+       where is_active = true and archived = false) as total_active,
+    (select count(*)::bigint from memory_sessions) as sessions,
+    coalesce(
+      (select jsonb_object_agg(project, c) from (
+         select project, count(*)::bigint as c
+         from memory_items
+         where is_active = true and archived = false
+         group by project
+       ) p),
+      '{}'::jsonb
+    ) as by_project,
+    coalesce(
+      (select jsonb_object_agg(source_type, c) from (
+         select source_type, count(*)::bigint as c
+         from memory_items
+         where is_active = true and archived = false
+         group by source_type
+       ) s),
+      '{}'::jsonb
+    ) as by_source_type,
+    coalesce(
+      (select jsonb_object_agg(coalesce(category, 'uncategorized'), c) from (
+         select category, count(*)::bigint as c
+         from memory_items
+         where is_active = true and archived = false
+         group by category
+       ) cat),
+      '{}'::jsonb
+    ) as by_category;
+$$;
+
+-- Ensure the service role (and anon, if you allow it) can call the RPC.
+grant execute on function memory_status_aggregation() to anon, authenticated, service_role;

package/packages/server/src/setup/index.js

@@ -0,0 +1,14 @@
+// Aggregate export for the `termdeck init` setup helpers.
+//
+// The init wizards live in packages/cli/src/ but all the heavy lifting
+// (prompting, reading/writing config, applying migrations) lives here so
+// the CLI files can stay short, linear, and easy to audit.
+
+module.exports = {
+  prompts: require('./prompts'),
+  dotenv: require('./dotenv-io'),
+  yaml: require('./yaml-io'),
+  supabaseUrl: require('./supabase-url'),
+  migrations: require('./migrations'),
+  pgRunner: require('./pg-runner')
+};

package/packages/server/src/setup/migrations.js

@@ -0,0 +1,80 @@
+// Discover the SQL migration files that ship bundled inside the TermDeck
+// package. Both init wizards call this — init-engram for the six Engram
+// migrations, init-rumen for the two Rumen migrations.
+//
+// The wizards intentionally do NOT fall back to a sibling `../../engram`
+// working copy. Resolution order:
+//
+//   1. Files bundled at `packages/server/src/setup/engram-migrations/*.sql`
+//      (this directory is covered by the root package.json `files` glob).
+//   2. Files at `node_modules/@jhizzard/engram/migrations/*.sql` if that
+//      package is installed alongside TermDeck (future-proof path — shipping
+//      `@jhizzard/engram` as an optional peer would let us drop the bundled
+//      copy).
+
+const fs = require('fs');
+const path = require('path');
+
+const SETUP_DIR = __dirname;
+
+function listBundled(subdir) {
+  const dir = path.join(SETUP_DIR, subdir);
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter((f) => f.toLowerCase().endsWith('.sql'))
+    .sort()
+    .map((f) => path.join(dir, f));
+}
+
+function tryNodeModules(packageName, migrationSubdir = 'migrations') {
+  try {
+    // Resolve the package's main file, then look for a migrations sibling dir.
+    const pkgJsonPath = require.resolve(`${packageName}/package.json`, {
+      paths: [process.cwd(), SETUP_DIR]
+    });
+    const pkgDir = path.dirname(pkgJsonPath);
+    const migrationDir = path.join(pkgDir, migrationSubdir);
+    if (!fs.existsSync(migrationDir)) return [];
+    return fs.readdirSync(migrationDir)
+      .filter((f) => f.toLowerCase().endsWith('.sql'))
+      .sort()
+      .map((f) => path.join(migrationDir, f));
+  } catch (_err) {
+    return [];
+  }
+}
+
+function listEngramMigrations() {
+  const fromNm = tryNodeModules('@jhizzard/engram');
+  if (fromNm.length > 0) return fromNm;
+  return listBundled('engram-migrations');
+}
+
+function listRumenMigrations() {
+  const fromNm = tryNodeModules('@jhizzard/rumen');
+  if (fromNm.length > 0) return fromNm;
+  return listBundled(path.join('rumen', 'migrations'));
+}
+
+function rumenFunctionDir() {
+  // Same resolution order.
+  try {
+    const pkgJsonPath = require.resolve('@jhizzard/rumen/package.json', {
+      paths: [process.cwd(), SETUP_DIR]
+    });
+    const candidate = path.join(path.dirname(pkgJsonPath), 'supabase', 'functions', 'rumen-tick');
+    if (fs.existsSync(candidate)) return candidate;
+  } catch (_err) { /* fallthrough */ }
+  return path.join(SETUP_DIR, 'rumen', 'functions', 'rumen-tick');
+}
+
+function readFile(filepath) {
+  return fs.readFileSync(filepath, 'utf-8');
+}
+
+module.exports = {
+  listEngramMigrations,
+  listRumenMigrations,
+  rumenFunctionDir,
+  readFile
+};

package/packages/server/src/setup/pg-runner.js

@@ -0,0 +1,113 @@
+// Thin wrapper around node-postgres for applying SQL migration files.
+//
+// `pg` is a runtime dep of the top-level package and must be present when the
+// init wizards run. If `require('pg')` fails we throw an actionable error
+// pointing the user at `npm install` — TermDeck's normal install path pulls
+// `pg` in automatically, so this only fires on partial / broken installs.
+
+const fs = require('fs');
+const path = require('path');
+
+function loadPg() {
+  try {
+    return require('pg');
+  } catch (err) {
+    const e = new Error(
+      "Could not load node-postgres ('pg'). TermDeck's init wizards need it to " +
+      'apply migrations. Run `npm install` (or reinstall TermDeck with `npm install -g @jhizzard/termdeck`) and try again.'
+    );
+    e.cause = err;
+    throw e;
+  }
+}
+
+// Connect to the given postgres URL and return a live Client. The caller is
+// responsible for `await client.end()`.
+async function connect(databaseUrl) {
+  const { Client } = loadPg();
+  const client = new Client({
+    connectionString: databaseUrl,
+    // Supabase pooler endpoints present a TLS cert that Node's default CA
+    // store accepts, but some users hit verification issues behind corporate
+    // proxies. Enable TLS with rejectUnauthorized=false so the wizard works
+    // in those environments — we're talking to a Supabase host we already
+    // trust by project ref, and the connection is still encrypted.
+    ssl: { rejectUnauthorized: false },
+    // Keep connection attempts bounded so a misconfigured URL fails fast.
+    connectionTimeoutMillis: 15000
+  });
+  try {
+    await client.connect();
+  } catch (err) {
+    const friendly = mapConnectError(err);
+    const e = new Error(`Could not connect to Postgres: ${friendly}`);
+    e.cause = err;
+    throw e;
+  }
+  return client;
+}
+
+function mapConnectError(err) {
+  const msg = err && err.message ? err.message : String(err);
+  if (/ENOTFOUND/.test(msg)) return 'host not found (check the project URL)';
+  if (/ECONNREFUSED/.test(msg)) return 'connection refused (wrong port?)';
+  if (/password authentication failed/i.test(msg)) return 'password authentication failed (check the service_role / db password)';
+  if (/SASL/i.test(msg)) return 'authentication failed (check username, it should be `postgres` or `postgres.<project-ref>`)';
+  if (/timeout/i.test(msg)) return 'connect timed out after 15s (network issue or wrong host)';
+  return msg;
+}
+
+// Read a migration file and execute it as a single batched query. Returns
+// `{ ok, file, elapsedMs, rowCount, skipped?, error? }`.
+//
+// A migration "skipped" means the file body included a marker the runner
+// detected as already-applied (reserved for future use — currently not wired).
+async function applyFile(client, filepath) {
+  const started = Date.now();
+  const sql = fs.readFileSync(filepath, 'utf-8');
+  const base = path.basename(filepath);
+  try {
+    const result = await client.query(sql);
+    const rowCount = Array.isArray(result)
+      ? result.reduce((sum, r) => sum + (r && r.rowCount ? r.rowCount : 0), 0)
+      : (result && result.rowCount) || 0;
+    return {
+      ok: true,
+      file: base,
+      elapsedMs: Date.now() - started,
+      rowCount
+    };
+  } catch (err) {
+    return {
+      ok: false,
+      file: base,
+      elapsedMs: Date.now() - started,
+      error: err && err.message ? err.message : String(err)
+    };
+  }
+}
+
+// Apply a list of migration files in order. Returns an array of per-file
+// results. Stops on first failure unless `continueOnError: true` is passed.
+async function applyAll(client, files, { continueOnError = false } = {}) {
+  const results = [];
+  for (const file of files) {
+    const result = await applyFile(client, file);
+    results.push(result);
+    if (!result.ok && !continueOnError) break;
+  }
+  return results;
+}
+
+// Run a single SQL string (not a file). Useful for parameterized checks like
+// `SELECT COUNT(*) FROM memory_items`.
+async function run(client, sql, params = []) {
+  return client.query(sql, params);
+}
+
+module.exports = {
+  connect,
+  applyFile,
+  applyAll,
+  run
+};