@jhizzard/termdeck 0.2.0 → 0.2.2

package/packages/server/src/setup/rumen/functions/rumen-tick/index.ts

@@ -0,0 +1,85 @@
+// Rumen v0.1 Supabase Edge Function entry point.
+//
+// IMPORTANT: This file targets the Deno runtime, NOT Node. It will not
+// compile under the root tsconfig.json — it is intentionally excluded.
+// A sibling tsconfig.json in this directory keeps the types sane for
+// editors, but the canonical build target is Deno's own type checker
+// (`deno check`) and Supabase's `supabase functions deploy`.
+//
+// Dependencies are pulled via `npm:` specifiers, which Supabase Edge
+// Functions support natively.
+//
+// Deployment:
+//   supabase functions deploy rumen-tick
+//   supabase secrets set DATABASE_URL="$DATABASE_URL"
+//
+// Triggered on a schedule by pg_cron — see migrations/002_pg_cron_schedule.sql.
+//
+// This function runs one Rumen job per invocation and returns a JSON summary.
+
+// @ts-ignore  Deno std import resolved at runtime.
+import { serve } from 'https://deno.land/std@0.224.0/http/server.ts';
+// @ts-ignore  npm specifier resolved at runtime.
+import { runRumenJob, createPoolFromUrl } from 'npm:@jhizzard/rumen@0.1.0';
+
+// @ts-ignore  Deno global available at runtime.
+declare const Deno: { env: { get: (k: string) => string | undefined } };
+
+serve(async (_req: Request) => {
+  const url = Deno.env.get('DATABASE_URL');
+  if (!url) {
+    console.error('[rumen] DATABASE_URL not set in Edge Function secrets');
+    return new Response(
+      JSON.stringify({
+        ok: false,
+        error: 'DATABASE_URL not set',
+      }),
+      {
+        status: 500,
+        headers: { 'Content-Type': 'application/json' },
+      },
+    );
+  }
+
+  const pool = createPoolFromUrl(url);
+
+  try {
+    console.log('[rumen] edge function tick starting');
+    const summary = await runRumenJob(pool, {
+      triggeredBy: 'schedule',
+    });
+    console.log(
+      '[rumen] edge function tick complete job_id=' +
+        summary.job_id +
+        ' status=' +
+        summary.status,
+    );
+
+    return new Response(
+      JSON.stringify({
+        ok: summary.status === 'done',
+        summary,
+      }),
+      {
+        status: summary.status === 'done' ? 200 : 500,
+        headers: { 'Content-Type': 'application/json' },
+      },
+    );
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error('[rumen] edge function tick threw:', err);
+    return new Response(
+      JSON.stringify({ ok: false, error: message }),
+      {
+        status: 500,
+        headers: { 'Content-Type': 'application/json' },
+      },
+    );
+  } finally {
+    try {
+      await pool.end();
+    } catch (err) {
+      console.error('[rumen] pool.end() failed:', err);
+    }
+  }
+});

package/packages/server/src/setup/rumen/functions/rumen-tick/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "lib": ["ES2022", "DOM"],
+    "strict": true,
+    "skipLibCheck": true,
+    "noEmit": true,
+    "allowImportingTsExtensions": false,
+    "types": []
+  },
+  "include": ["index.ts"]
+}

package/packages/server/src/setup/rumen/migrations/001_rumen_tables.sql

@@ -0,0 +1,91 @@
+-- Rumen v0.1 schema
+-- Non-destructive: creates three new tables under the rumen_ namespace.
+-- Does NOT modify or reference Engram's existing memory_items / memory_sessions tables.
+--
+-- Apply with:
+--   psql "$DIRECT_URL" -f migrations/001_rumen_tables.sql
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- rumen_jobs: one row per Rumen tick. Tracks what was processed.
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS rumen_jobs (
+  id                    UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  triggered_by          TEXT NOT NULL CHECK (triggered_by IN ('schedule', 'session_end', 'manual')),
+  status                TEXT NOT NULL CHECK (status IN ('pending', 'running', 'done', 'failed')),
+  sessions_processed    INTEGER NOT NULL DEFAULT 0,
+  insights_generated    INTEGER NOT NULL DEFAULT 0,
+  questions_generated   INTEGER NOT NULL DEFAULT 0,
+  error_message         TEXT,
+  source_session_ids    UUID[] NOT NULL DEFAULT '{}',
+  started_at            TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  completed_at          TIMESTAMPTZ
+);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_jobs_status
+  ON rumen_jobs (status);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_jobs_started_at
+  ON rumen_jobs (started_at DESC);
+
+-- GIN index so we can cheaply check "has this session already been processed?"
+CREATE INDEX IF NOT EXISTS idx_rumen_jobs_source_session_ids
+  ON rumen_jobs USING GIN (source_session_ids);
+
+-- ---------------------------------------------------------------------------
+-- rumen_insights: synthesized cross-project findings.
+-- v0.1 writes placeholder insight_text; v0.2 will add LLM synthesis.
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS rumen_insights (
+  id                    UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  job_id                UUID NOT NULL REFERENCES rumen_jobs(id) ON DELETE CASCADE,
+  source_memory_ids     UUID[] NOT NULL DEFAULT '{}',
+  projects              TEXT[] NOT NULL DEFAULT '{}',
+  insight_text          TEXT NOT NULL,
+  confidence            NUMERIC(4, 3) NOT NULL DEFAULT 0.000
+    CHECK (confidence >= 0.0 AND confidence <= 1.0),
+  acted_upon            BOOLEAN NOT NULL DEFAULT FALSE,
+  created_at            TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_insights_job_id
+  ON rumen_insights (job_id);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_insights_created_at
+  ON rumen_insights (created_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_insights_projects
+  ON rumen_insights USING GIN (projects);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_insights_source_memory_ids
+  ON rumen_insights USING GIN (source_memory_ids);
+
+-- ---------------------------------------------------------------------------
+-- rumen_questions: follow-up questions Rumen wants to ask the developer.
+-- Reserved for v0.3. The table is created in v0.1 so the schema is stable,
+-- but v0.1 never writes to it.
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS rumen_questions (
+  id                    UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  job_id                UUID NOT NULL REFERENCES rumen_jobs(id) ON DELETE CASCADE,
+  session_id            UUID,
+  question              TEXT NOT NULL,
+  context               TEXT,
+  asked_at              TIMESTAMPTZ,
+  answered_at           TIMESTAMPTZ,
+  answer                TEXT,
+  created_at            TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_questions_job_id
+  ON rumen_questions (job_id);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_questions_session_id
+  ON rumen_questions (session_id);
+
+CREATE INDEX IF NOT EXISTS idx_rumen_questions_unanswered
+  ON rumen_questions (created_at DESC)
+  WHERE answered_at IS NULL;
+
+COMMIT;

package/packages/server/src/setup/rumen/migrations/002_pg_cron_schedule.sql

@@ -0,0 +1,40 @@
+-- Rumen v0.1 schedule
+-- Schedules the rumen-tick Supabase Edge Function to run every 15 minutes
+-- via pg_cron + pg_net.
+--
+-- Before applying:
+--   1. Enable the pg_cron extension in the Supabase dashboard:
+--        Database -> Extensions -> pg_cron (toggle on)
+--   2. Enable the pg_net extension the same way (needed for net.http_post).
+--   3. Replace <project-ref> below with your actual Supabase project ref.
+--   4. Replace <service-role-jwt> below with a service-role key stored in
+--      Supabase Vault, NOT pasted inline. The SELECT below shows the Vault
+--      pattern; delete the inline-key variant before running.
+--
+-- Apply with:
+--   psql "$DIRECT_URL" -f migrations/002_pg_cron_schedule.sql
+
+-- Remove any prior schedule with the same name so re-running is idempotent.
+SELECT cron.unschedule('rumen-tick')
+  WHERE EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'rumen-tick');
+
+-- Schedule rumen-tick every 15 minutes.
+-- The body is empty JSON; the Edge Function reads DATABASE_URL from its
+-- own function secrets, not from the request body.
+SELECT cron.schedule(
+  'rumen-tick',
+  '*/15 * * * *',
+  $$
+    SELECT net.http_post(
+      url     := 'https://<project-ref>.supabase.co/functions/v1/rumen-tick',
+      headers := jsonb_build_object(
+        'Content-Type',  'application/json',
+        'Authorization', 'Bearer ' || (SELECT decrypted_secret FROM vault.decrypted_secrets WHERE name = 'rumen_service_role_key')
+      ),
+      body    := '{}'::jsonb
+    );
+  $$
+);
+
+-- Verify:
+-- SELECT jobname, schedule, active FROM cron.job WHERE jobname = 'rumen-tick';

package/packages/server/src/setup/supabase-url.js

@@ -0,0 +1,114 @@
+// Parse + validate Supabase URLs and derive what we can from them without the
+// database password. Useful for both init wizards:
+//
+//   - init-engram needs the project ref to show in status output and also
+//     needs a full DATABASE_URL to apply migrations; since the DB password
+//     cannot be derived from the project URL alone, the wizard prompts for
+//     the direct connection string separately.
+//
+//   - init-rumen needs the project ref to run `supabase link --project-ref`
+//     and to substitute into the pg_cron schedule SQL.
+
+// A Supabase project URL looks like:
+//   https://<project-ref>.supabase.co
+// The ref is 20 characters of lowercase alphanumerics, but we accept anything
+// that matches `[a-z0-9-]+` to avoid being stricter than Supabase itself.
+function parseProjectUrl(url) {
+  if (!url || typeof url !== 'string') {
+    return { ok: false, error: 'empty url' };
+  }
+  const trimmed = url.trim().replace(/\/+$/, '');
+  let u;
+  try {
+    u = new URL(trimmed);
+  } catch (_err) {
+    return { ok: false, error: 'not a valid URL' };
+  }
+  if (u.protocol !== 'https:') {
+    return { ok: false, error: 'must be https://' };
+  }
+  const m = u.hostname.match(/^([a-z0-9-]+)\.supabase\.(co|in)$/i);
+  if (!m) {
+    return { ok: false, error: 'hostname must end in .supabase.co' };
+  }
+  return {
+    ok: true,
+    url: `https://${u.hostname}`,
+    projectRef: m[1].toLowerCase(),
+    hostname: u.hostname
+  };
+}
+
+// Validate that a string has the shape of a Supabase service role key.
+// Supabase now ships two formats:
+//   (1) legacy JWT: `eyJ...` (header.payload.signature), usually ~200 chars
+//   (2) prefixed v2: `sb_secret_...` or `sb_publishable_...`
+// We accept both shapes and reject anything else with an explicit hint.
+function looksLikeServiceRole(key) {
+  if (!key || typeof key !== 'string') return 'empty';
+  const trimmed = key.trim();
+  if (trimmed.startsWith('sb_secret_')) return null;
+  if (trimmed.startsWith('sb_publishable_')) {
+    return 'that looks like a publishable (anon) key, not the service_role key';
+  }
+  if (trimmed.startsWith('eyJ')) {
+    // Rough JWT shape check — 3 dot-separated base64url chunks.
+    const parts = trimmed.split('.');
+    if (parts.length === 3) return null;
+    return 'JWT-looking string but not 3 segments';
+  }
+  return 'does not look like a Supabase service_role key (expected sb_secret_… or eyJ…)';
+}
+
+// Shape-check an OpenAI API key. Both classic `sk-` and project `sk-proj-`
+// formats are accepted.
+function looksLikeOpenAiKey(key) {
+  if (!key || typeof key !== 'string') return 'empty';
+  const trimmed = key.trim();
+  if (trimmed.startsWith('sk-') && trimmed.length >= 20) return null;
+  return 'OpenAI keys start with sk-';
+}
+
+// Shape-check an Anthropic API key.
+function looksLikeAnthropicKey(key) {
+  if (!key || typeof key !== 'string') return 'empty';
+  const trimmed = key.trim();
+  if (trimmed.startsWith('sk-ant-') && trimmed.length >= 30) return null;
+  return 'Anthropic keys start with sk-ant-';
+}
+
+// Shape-check a Postgres connection string. Accepts both Supabase pooler URLs
+// (`postgres://postgres.<ref>:...@aws-0-<region>.pooler.supabase.com:<port>/postgres`)
+// and direct connection URLs (`postgres://postgres:...@db.<ref>.supabase.co:5432/postgres`).
+function looksLikePostgresUrl(url) {
+  if (!url || typeof url !== 'string') return 'empty';
+  let u;
+  try {
+    u = new URL(url);
+  } catch (_err) {
+    return 'not a valid URL';
+  }
+  if (u.protocol !== 'postgres:' && u.protocol !== 'postgresql:') {
+    return 'must start with postgres:// or postgresql://';
+  }
+  if (!u.username || !u.password) {
+    return 'missing username or password — paste the full Connection String from Supabase';
+  }
+  return null;
+}
+
+// Mask all but the last 4 chars of a secret for logging.
+function maskSecret(value) {
+  if (!value || typeof value !== 'string') return '';
+  if (value.length <= 8) return '***';
+  return `${value.slice(0, 4)}…${value.slice(-4)}`;
+}
+
+module.exports = {
+  parseProjectUrl,
+  looksLikeServiceRole,
+  looksLikeOpenAiKey,
+  looksLikeAnthropicKey,
+  looksLikePostgresUrl,
+  maskSecret
+};

package/packages/server/src/setup/yaml-io.js

@@ -0,0 +1,99 @@
+// Targeted writer for ~/.termdeck/config.yaml that the `init --engram` wizard
+// uses to flip `rag.enabled: true` and point secret fields at `${VAR}` refs
+// instead of inline values.
+//
+// Uses the `yaml` package (already a dep) for a full parse + stringify round
+// trip. Comments WILL be lost on rewrite — yaml.stringify doesn't preserve
+// them. We back up the original to a timestamped `.bak` file first so the
+// user can recover any hand-written comments.
+//
+// If config.yaml doesn't exist yet, this helper lets the server's own
+// loadConfig() create the default template on next startup — it does NOT
+// write a fresh template itself.
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const CONFIG_DIR = path.join(os.homedir(), '.termdeck');
+const CONFIG_PATH = path.join(CONFIG_DIR, 'config.yaml');
+
+function loadYaml() {
+  const yaml = require('yaml');
+  if (!fs.existsSync(CONFIG_PATH)) return { parsed: {}, existed: false };
+  const raw = fs.readFileSync(CONFIG_PATH, 'utf-8');
+  try {
+    const parsed = yaml.parse(raw) || {};
+    return { parsed, existed: true };
+  } catch (err) {
+    throw new Error(`config.yaml exists but is not valid YAML — refusing to rewrite: ${err.message}`);
+  }
+}
+
+function backup() {
+  if (!fs.existsSync(CONFIG_PATH)) return null;
+  const ts = new Date().toISOString().replace(/[:.]/g, '-');
+  const bak = `${CONFIG_PATH}.${ts}.bak`;
+  fs.copyFileSync(CONFIG_PATH, bak);
+  return bak;
+}
+
+// Update the `rag.*` section of config.yaml. Pass only fields you want to
+// change — everything else (projects, themes, sessionLogs, etc.) is preserved.
+// Secret fields should be `${VAR}` references, not raw keys.
+function updateRagConfig(updates) {
+  const yaml = require('yaml');
+  if (!fs.existsSync(CONFIG_DIR)) fs.mkdirSync(CONFIG_DIR, { recursive: true });
+
+  const { parsed, existed } = loadYaml();
+  const rag = (parsed.rag && typeof parsed.rag === 'object') ? parsed.rag : {};
+
+  // Merge in a well-defined order so the output is readable.
+  const merged = {
+    enabled: updates.enabled != null ? updates.enabled : (rag.enabled != null ? rag.enabled : false),
+    supabaseUrl: updates.supabaseUrl || rag.supabaseUrl || '${SUPABASE_URL}',
+    supabaseKey: updates.supabaseKey || rag.supabaseKey || '${SUPABASE_SERVICE_ROLE_KEY}',
+    openaiApiKey: updates.openaiApiKey || rag.openaiApiKey || '${OPENAI_API_KEY}',
+    anthropicApiKey: updates.anthropicApiKey || rag.anthropicApiKey || '${ANTHROPIC_API_KEY}',
+    syncIntervalMs: rag.syncIntervalMs != null ? rag.syncIntervalMs : 10000,
+    engramMode: rag.engramMode || 'direct',
+    engramWebhookUrl: rag.engramWebhookUrl || 'http://localhost:37778/engram'
+  };
+
+  // Preserve any fields we didn't explicitly handle (e.g. tables, developerId).
+  for (const [k, v] of Object.entries(rag)) {
+    if (!(k in merged)) merged[k] = v;
+  }
+  parsed.rag = merged;
+
+  const bak = existed ? backup() : null;
+  fs.writeFileSync(CONFIG_PATH, yaml.stringify(parsed), 'utf-8');
+  return { path: CONFIG_PATH, backup: bak };
+}
+
+// Scan loaded parsed config for literal inline secrets in the rag section.
+// Returns an array of dot-paths that still look like raw values (not ${VAR}).
+function findInlineSecrets() {
+  const { parsed, existed } = loadYaml();
+  if (!existed) return [];
+  const rag = parsed.rag || {};
+  const hits = [];
+  const fields = ['supabaseKey', 'openaiApiKey', 'anthropicApiKey', 'supabaseUrl'];
+  for (const f of fields) {
+    const v = rag[f];
+    if (typeof v !== 'string') continue;
+    if (!v) continue;
+    const isEnvRef = /^\$\{[A-Z0-9_]+(?::-[^}]*)?\}$/i.test(v.trim());
+    if (!isEnvRef) hits.push(`rag.${f}`);
+  }
+  return hits;
+}
+
+module.exports = {
+  CONFIG_PATH,
+  CONFIG_DIR,
+  loadYaml,
+  updateRagConfig,
+  findInlineSecrets,
+  _backup: backup
+};