@jhizzard/termdeck 0.2.0 → 0.2.2

package/packages/server/src/setup/engram-migrations/004_engram_match_count_cap_and_explain.sql

@@ -0,0 +1,176 @@
+-- Engram v0.2 — match_count cap + EXPLAIN variant
+--
+-- Two changes to the search surface:
+--
+--   1. memory_hybrid_search gains a configurable cap on `match_count`.
+--      Default cap: 200. The original function was unbounded, which risks
+--      runaway queries at scale (10k+ rows pulled per call).
+--
+--      Override per-database:   ALTER DATABASE your_db SET engram.max_match_count = 500;
+--      Override per-session:    SET engram.max_match_count = 500;
+--      Leave unset:             cap defaults to 200.
+--
+--   2. A new function `memory_hybrid_search_explain` that returns
+--      EXPLAIN (ANALYZE, BUFFERS) output for an equivalent call. Used by
+--      `engram diagnose` to troubleshoot slow recall queries.
+--
+-- Rerun-safe: CREATE OR REPLACE on both.
+
+-- ── memory_hybrid_search ─────────────────────────────────────────────────
+
+create or replace function memory_hybrid_search (
+  query_text          text,
+  query_embedding     vector(1536),
+  match_count         int default 20,
+  full_text_weight    float default 1.0,
+  semantic_weight     float default 1.0,
+  rrf_k               int default 60,
+  filter_project      text default null,
+  filter_source_type  text default null
+)
+returns table (
+  id          uuid,
+  content     text,
+  source_type text,
+  category    text,
+  project     text,
+  metadata    jsonb,
+  score       float,
+  created_at  timestamptz
+)
+language sql stable
+as $$
+with candidates as (
+  select
+    m.id,
+    m.content,
+    m.source_type,
+    m.category,
+    m.project,
+    m.metadata,
+    m.created_at,
+    m.embedding,
+    ts_rank_cd(to_tsvector('english', m.content), plainto_tsquery('english', query_text))
+      as ft_rank,
+    1 - (m.embedding <=> query_embedding) as sem_rank,
+    extract(epoch from (now() - m.created_at))::float as age_seconds
+  from memory_items m
+  where m.is_active = true
+    and m.archived = false
+    and m.superseded_by is null
+    and m.embedding is not null
+    and (filter_project is null or m.project = filter_project)
+    and (filter_source_type is null or m.source_type = filter_source_type)
+),
+ft_ranked as (
+  select id, row_number() over (order by ft_rank desc nulls last) as rank
+  from candidates where ft_rank > 0
+),
+sem_ranked as (
+  select id, row_number() over (order by sem_rank desc nulls last) as rank
+  from candidates
+),
+fused as (
+  select
+    c.id,
+    c.content,
+    c.source_type,
+    c.category,
+    c.project,
+    c.metadata,
+    c.created_at,
+    c.age_seconds,
+    coalesce(full_text_weight / (rrf_k + ft.rank), 0.0) +
+    coalesce(semantic_weight  / (rrf_k + sr.rank), 0.0) as base_score
+  from candidates c
+  left join ft_ranked  ft on ft.id = c.id
+  left join sem_ranked sr on sr.id = c.id
+),
+scored as (
+  select
+    f.id,
+    f.content,
+    f.source_type,
+    f.category,
+    f.project,
+    f.metadata,
+    f.created_at,
+    f.base_score
+      * case f.source_type
+          when 'decision'        then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'architecture'    then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'preference'      then 1.0 / (1.0 + f.age_seconds / (365.0 * 86400.0))
+          when 'fact'            then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'convention'      then 1.0 / (1.0 + f.age_seconds / ( 90.0 * 86400.0))
+          when 'bug_fix'         then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'debugging'       then 1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+          when 'session_summary' then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'document_chunk'  then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          when 'code_context'    then 1.0 / (1.0 + f.age_seconds / ( 14.0 * 86400.0))
+          else                         1.0 / (1.0 + f.age_seconds / ( 30.0 * 86400.0))
+        end
+      * case f.source_type
+          when 'decision'       then 1.5
+          when 'architecture'   then 1.4
+          when 'bug_fix'        then 1.3
+          when 'preference'     then 1.2
+          when 'fact'           then 1.0
+          when 'document_chunk' then 0.6
+          else                       1.0
+        end
+      * case
+          when filter_project is null then 1.0
+          when f.project = filter_project then 1.5
+          when f.project = 'global' then 1.0
+          else 0.7
+        end
+      as score
+  from fused f
+)
+select
+  s.id,
+  s.content,
+  s.source_type,
+  s.category,
+  s.project,
+  s.metadata,
+  s.score,
+  s.created_at
+from scored s
+order by s.score desc
+limit least(
+  greatest(match_count, 1),
+  coalesce(nullif(current_setting('engram.max_match_count', true), '')::int, 200)
+);
+$$;
+
+-- ── memory_hybrid_search_explain ─────────────────────────────────────────
+
+create or replace function memory_hybrid_search_explain (
+  query_text          text,
+  query_embedding     vector(1536),
+  match_count         int default 20,
+  full_text_weight    float default 1.0,
+  semantic_weight     float default 1.0,
+  rrf_k               int default 60,
+  filter_project      text default null,
+  filter_source_type  text default null
+)
+returns setof text
+language plpgsql
+as $$
+begin
+  return query execute
+    'explain (analyze, buffers, format text) '
+    || 'select * from memory_hybrid_search($1, $2, $3, $4, $5, $6, $7, $8)'
+  using
+    query_text,
+    query_embedding,
+    match_count,
+    full_text_weight,
+    semantic_weight,
+    rrf_k,
+    filter_project,
+    filter_source_type;
+end;
+$$;

package/packages/server/src/setup/engram-migrations/005_v0_1_to_v0_2_upgrade.sql

@@ -0,0 +1,23 @@
+-- Engram v0.2 — minimal additive delta for stores provisioned against the
+-- original rag-system schema.
+--
+-- This migration is idempotent and non-destructive:
+--   • adds only the single column (`archived`) that Engram v0.2 reads/writes
+--     but which is absent from the original rag-system `memory_items` table;
+--   • re-creates the two partial indexes under v2 names so they cannot
+--     collide with any pre-existing same-named index that uses a different
+--     `where` predicate;
+--   • does NOT replace `memory_hybrid_search`, `match_memories`, or any
+--     other SQL function — those remain on their production versions.
+--
+-- Apply once, in the Supabase SQL editor, against any existing store that
+-- pre-dates Engram v0.2's `archived` soft-delete column.
+
+alter table memory_items
+  add column if not exists archived boolean not null default false;
+
+create index if not exists memory_items_project_idx_v2
+  on memory_items(project) where is_active = true and archived = false;
+
+create index if not exists memory_items_source_type_idx_v2
+  on memory_items(source_type) where is_active = true and archived = false;

package/packages/server/src/setup/engram-migrations/006_memory_status_rpc.sql

@@ -0,0 +1,58 @@
+-- Engram migration 006 — memory_status_aggregation RPC
+--
+-- Why: `memoryStatus()` previously did a plain
+--   supabase.from('memory_items').select('project, source_type, category')
+-- which hits PostgREST's default 1000-row cap. On a store with 3,397 active
+-- rows the `by_project` / `by_source_type` / `by_category` histograms only
+-- summed to ~1000 even though `total_active` was correct. Pushing the GROUP
+-- BY server-side eliminates the cap and saves the round-trip of streaming
+-- every row to the client just to count them.
+--
+-- Safe to re-run — CREATE OR REPLACE.
+
+create or replace function memory_status_aggregation()
+returns table (
+  total_active   bigint,
+  sessions       bigint,
+  by_project     jsonb,
+  by_source_type jsonb,
+  by_category    jsonb
+)
+language sql
+stable
+as $$
+  select
+    (select count(*)::bigint from memory_items
+       where is_active = true and archived = false) as total_active,
+    (select count(*)::bigint from memory_sessions) as sessions,
+    coalesce(
+      (select jsonb_object_agg(project, c) from (
+         select project, count(*)::bigint as c
+         from memory_items
+         where is_active = true and archived = false
+         group by project
+       ) p),
+      '{}'::jsonb
+    ) as by_project,
+    coalesce(
+      (select jsonb_object_agg(source_type, c) from (
+         select source_type, count(*)::bigint as c
+         from memory_items
+         where is_active = true and archived = false
+         group by source_type
+       ) s),
+      '{}'::jsonb
+    ) as by_source_type,
+    coalesce(
+      (select jsonb_object_agg(coalesce(category, 'uncategorized'), c) from (
+         select category, count(*)::bigint as c
+         from memory_items
+         where is_active = true and archived = false
+         group by category
+       ) cat),
+      '{}'::jsonb
+    ) as by_category;
+$$;
+
+-- Ensure the service role (and anon, if you allow it) can call the RPC.
+grant execute on function memory_status_aggregation() to anon, authenticated, service_role;

package/packages/server/src/setup/index.js

@@ -0,0 +1,14 @@
+// Aggregate export for the `termdeck init` setup helpers.
+//
+// The init wizards live in packages/cli/src/ but all the heavy lifting
+// (prompting, reading/writing config, applying migrations) lives here so
+// the CLI files can stay short, linear, and easy to audit.
+
+module.exports = {
+  prompts: require('./prompts'),
+  dotenv: require('./dotenv-io'),
+  yaml: require('./yaml-io'),
+  supabaseUrl: require('./supabase-url'),
+  migrations: require('./migrations'),
+  pgRunner: require('./pg-runner')
+};

package/packages/server/src/setup/migrations.js

@@ -0,0 +1,80 @@
+// Discover the SQL migration files that ship bundled inside the TermDeck
+// package. Both init wizards call this — init-engram for the six Engram
+// migrations, init-rumen for the two Rumen migrations.
+//
+// The wizards intentionally do NOT fall back to a sibling `../../engram`
+// working copy. Resolution order:
+//
+//   1. Files bundled at `packages/server/src/setup/engram-migrations/*.sql`
+//      (this directory is covered by the root package.json `files` glob).
+//   2. Files at `node_modules/@jhizzard/engram/migrations/*.sql` if that
+//      package is installed alongside TermDeck (future-proof path — shipping
+//      `@jhizzard/engram` as an optional peer would let us drop the bundled
+//      copy).
+
+const fs = require('fs');
+const path = require('path');
+
+const SETUP_DIR = __dirname;
+
+function listBundled(subdir) {
+  const dir = path.join(SETUP_DIR, subdir);
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter((f) => f.toLowerCase().endsWith('.sql'))
+    .sort()
+    .map((f) => path.join(dir, f));
+}
+
+function tryNodeModules(packageName, migrationSubdir = 'migrations') {
+  try {
+    // Resolve the package's main file, then look for a migrations sibling dir.
+    const pkgJsonPath = require.resolve(`${packageName}/package.json`, {
+      paths: [process.cwd(), SETUP_DIR]
+    });
+    const pkgDir = path.dirname(pkgJsonPath);
+    const migrationDir = path.join(pkgDir, migrationSubdir);
+    if (!fs.existsSync(migrationDir)) return [];
+    return fs.readdirSync(migrationDir)
+      .filter((f) => f.toLowerCase().endsWith('.sql'))
+      .sort()
+      .map((f) => path.join(migrationDir, f));
+  } catch (_err) {
+    return [];
+  }
+}
+
+function listEngramMigrations() {
+  const fromNm = tryNodeModules('@jhizzard/engram');
+  if (fromNm.length > 0) return fromNm;
+  return listBundled('engram-migrations');
+}
+
+function listRumenMigrations() {
+  const fromNm = tryNodeModules('@jhizzard/rumen');
+  if (fromNm.length > 0) return fromNm;
+  return listBundled(path.join('rumen', 'migrations'));
+}
+
+function rumenFunctionDir() {
+  // Same resolution order.
+  try {
+    const pkgJsonPath = require.resolve('@jhizzard/rumen/package.json', {
+      paths: [process.cwd(), SETUP_DIR]
+    });
+    const candidate = path.join(path.dirname(pkgJsonPath), 'supabase', 'functions', 'rumen-tick');
+    if (fs.existsSync(candidate)) return candidate;
+  } catch (_err) { /* fallthrough */ }
+  return path.join(SETUP_DIR, 'rumen', 'functions', 'rumen-tick');
+}
+
+function readFile(filepath) {
+  return fs.readFileSync(filepath, 'utf-8');
+}
+
+module.exports = {
+  listEngramMigrations,
+  listRumenMigrations,
+  rumenFunctionDir,
+  readFile
+};

package/packages/server/src/setup/pg-runner.js

@@ -0,0 +1,113 @@
+// Thin wrapper around node-postgres for applying SQL migration files.
+//
+// `pg` is a runtime dep of the top-level package and must be present when the
+// init wizards run. If `require('pg')` fails we throw an actionable error
+// pointing the user at `npm install` — TermDeck's normal install path pulls
+// `pg` in automatically, so this only fires on partial / broken installs.
+
+const fs = require('fs');
+const path = require('path');
+
+function loadPg() {
+  try {
+    return require('pg');
+  } catch (err) {
+    const e = new Error(
+      "Could not load node-postgres ('pg'). TermDeck's init wizards need it to " +
+      'apply migrations. Run `npm install` (or reinstall TermDeck with `npm install -g @jhizzard/termdeck`) and try again.'
+    );
+    e.cause = err;
+    throw e;
+  }
+}
+
+// Connect to the given postgres URL and return a live Client. The caller is
+// responsible for `await client.end()`.
+async function connect(databaseUrl) {
+  const { Client } = loadPg();
+  const client = new Client({
+    connectionString: databaseUrl,
+    // Supabase pooler endpoints present a TLS cert that Node's default CA
+    // store accepts, but some users hit verification issues behind corporate
+    // proxies. Enable TLS with rejectUnauthorized=false so the wizard works
+    // in those environments — we're talking to a Supabase host we already
+    // trust by project ref, and the connection is still encrypted.
+    ssl: { rejectUnauthorized: false },
+    // Keep connection attempts bounded so a misconfigured URL fails fast.
+    connectionTimeoutMillis: 15000
+  });
+  try {
+    await client.connect();
+  } catch (err) {
+    const friendly = mapConnectError(err);
+    const e = new Error(`Could not connect to Postgres: ${friendly}`);
+    e.cause = err;
+    throw e;
+  }
+  return client;
+}
+
+function mapConnectError(err) {
+  const msg = err && err.message ? err.message : String(err);
+  if (/ENOTFOUND/.test(msg)) return 'host not found (check the project URL)';
+  if (/ECONNREFUSED/.test(msg)) return 'connection refused (wrong port?)';
+  if (/password authentication failed/i.test(msg)) return 'password authentication failed (check the service_role / db password)';
+  if (/SASL/i.test(msg)) return 'authentication failed (check username, it should be `postgres` or `postgres.<project-ref>`)';
+  if (/timeout/i.test(msg)) return 'connect timed out after 15s (network issue or wrong host)';
+  return msg;
+}
+
+// Read a migration file and execute it as a single batched query. Returns
+// `{ ok, file, elapsedMs, rowCount, skipped?, error? }`.
+//
+// A migration "skipped" means the file body included a marker the runner
+// detected as already-applied (reserved for future use — currently not wired).
+async function applyFile(client, filepath) {
+  const started = Date.now();
+  const sql = fs.readFileSync(filepath, 'utf-8');
+  const base = path.basename(filepath);
+  try {
+    const result = await client.query(sql);
+    const rowCount = Array.isArray(result)
+      ? result.reduce((sum, r) => sum + (r && r.rowCount ? r.rowCount : 0), 0)
+      : (result && result.rowCount) || 0;
+    return {
+      ok: true,
+      file: base,
+      elapsedMs: Date.now() - started,
+      rowCount
+    };
+  } catch (err) {
+    return {
+      ok: false,
+      file: base,
+      elapsedMs: Date.now() - started,
+      error: err && err.message ? err.message : String(err)
+    };
+  }
+}
+
+// Apply a list of migration files in order. Returns an array of per-file
+// results. Stops on first failure unless `continueOnError: true` is passed.
+async function applyAll(client, files, { continueOnError = false } = {}) {
+  const results = [];
+  for (const file of files) {
+    const result = await applyFile(client, file);
+    results.push(result);
+    if (!result.ok && !continueOnError) break;
+  }
+  return results;
+}
+
+// Run a single SQL string (not a file). Useful for parameterized checks like
+// `SELECT COUNT(*) FROM memory_items`.
+async function run(client, sql, params = []) {
+  return client.query(sql, params);
+}
+
+module.exports = {
+  connect,
+  applyFile,
+  applyAll,
+  run
+};

package/packages/server/src/setup/prompts.js

@@ -0,0 +1,177 @@
+// Readline-based interactive prompts for the `termdeck init` wizards.
+//
+// No external deps. All prompts in a single wizard run share ONE readline
+// interface (via the module-scoped `rl` below). We consume lines via the
+// `line` event and an internal FIFO of pending resolvers instead of
+// `rl.question()` — the latter has broken behavior when stdin is a piped
+// stream in non-terminal mode, where the second `.question()` call silently
+// hangs. The event-driven path works for both TTY and piped input, which lets
+// us drive the wizard non-interactively in tests:
+//
+//   printf 'a\nb\nc\n' | termdeck init --engram --dry-run
+//
+// Secret prompts still mute stdout echo when TTY is attached; on non-TTY
+// stdin they fall back to visible input so piped test runs work.
+
+const readline = require('readline');
+
+let rl = null;
+const waiting = [];   // pending resolver functions
+const buffered = [];  // lines that arrived before anyone was waiting
+let sawEnd = false;
+
+function getRl() {
+  if (rl) return rl;
+  rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+    terminal: false
+  });
+  rl.on('line', (line) => {
+    if (waiting.length > 0) {
+      const resolver = waiting.shift();
+      resolver(line);
+    } else {
+      buffered.push(line);
+    }
+  });
+  rl.on('close', () => {
+    sawEnd = true;
+    while (waiting.length > 0) {
+      const resolver = waiting.shift();
+      resolver('');
+    }
+  });
+  return rl;
+}
+
+function readLine() {
+  return new Promise((resolve) => {
+    getRl();
+    if (buffered.length > 0) {
+      resolve(buffered.shift());
+      return;
+    }
+    if (sawEnd) { resolve(''); return; }
+    waiting.push(resolve);
+  });
+}
+
+// Basic non-secret prompt. Returns empty string if the user just hits enter.
+async function ask(question, { defaultValue } = {}) {
+  const suffix = defaultValue ? ` [${defaultValue}]` : '';
+  if (question) process.stdout.write(`${question}${suffix}: `);
+  const line = await readLine();
+  const trimmed = (line || '').trim();
+  return trimmed || defaultValue || '';
+}
+
+// Required prompt — re-asks up to `maxAttempts` times if the answer is empty
+// or fails `validate(value) → null | string`. A validator returning a string
+// message means "invalid, re-ask with this error". Throws if all attempts fail.
+async function askRequired(question, { validate, maxAttempts = 3 } = {}) {
+  for (let i = 0; i < maxAttempts; i++) {
+    const answer = await ask(question);
+    if (!answer) {
+      process.stdout.write('  (required)\n');
+      continue;
+    }
+    if (validate) {
+      const err = validate(answer);
+      if (err) {
+        process.stdout.write(`  ${err}\n`);
+        continue;
+      }
+    }
+    return answer;
+  }
+  throw new Error(`No valid answer after ${maxAttempts} attempts: ${question}`);
+}
+
+// Optional prompt. Empty → null. Still validates non-empty answers.
+async function askOptional(question, { validate } = {}) {
+  const answer = await ask(question);
+  if (!answer) return null;
+  if (validate) {
+    const err = validate(answer);
+    if (err) {
+      process.stdout.write(`  ${err} — ignoring.\n`);
+      return null;
+    }
+  }
+  return answer;
+}
+
+// Secret prompt. On TTY we mute echo; on non-TTY we fall back to a visible
+// line read. Callers typically pass an empty string for `question` and write
+// their own label first.
+async function askSecret(question) {
+  if (!process.stdin.isTTY) {
+    return ask(question);
+  }
+  if (question) process.stdout.write(`${question}: `);
+  // Raw-mode reader. Detach the shared readline for the duration so both
+  // consumers aren't racing on stdin 'data' events.
+  return new Promise((resolve) => {
+    if (rl) rl.pause();
+    const stdin = process.stdin;
+    stdin.setRawMode(true);
+    stdin.resume();
+    stdin.setEncoding('utf-8');
+
+    let buffer = '';
+    const onData = (chunk) => {
+      for (const ch of chunk) {
+        if (ch === '\n' || ch === '\r' || ch === '\u0004') {
+          stdin.setRawMode(false);
+          stdin.removeListener('data', onData);
+          process.stdout.write('\n');
+          if (rl) rl.resume();
+          resolve(buffer);
+          return;
+        }
+        if (ch === '\u0003') {
+          stdin.setRawMode(false);
+          stdin.removeListener('data', onData);
+          process.stdout.write('\n');
+          process.kill(process.pid, 'SIGINT');
+          return;
+        }
+        if (ch === '\u007f' || ch === '\b') {
+          if (buffer.length > 0) {
+            buffer = buffer.slice(0, -1);
+            process.stdout.write('\b \b');
+          }
+          continue;
+        }
+        buffer += ch;
+        process.stdout.write('*');
+      }
+    };
+    stdin.on('data', onData);
+  });
+}
+
+// Yes/no confirm. Returns boolean.
+async function confirm(question, { defaultYes = true } = {}) {
+  const suffix = defaultYes ? '[Y/n]' : '[y/N]';
+  const answer = (await ask(`${question} ${suffix}`)).toLowerCase();
+  if (!answer) return defaultYes;
+  return answer === 'y' || answer === 'yes';
+}
+
+function closeRl() {
+  if (rl) {
+    rl.close();
+    rl = null;
+  }
+}
+
+module.exports = {
+  ask,
+  askRequired,
+  askOptional,
+  askSecret,
+  confirm,
+  closeRl
+};