@useatlas/create 0.0.6 → 0.0.7

package/templates/docker/src/lib/db/internal.ts

@@ -4,19 +4,232 @@
  * Read-write Postgres connection for Atlas's own state (auth, audit, settings).
  * Completely separate from the analytics datasource in connection.ts.
  * Configured via DATABASE_URL.
+ *
+ * Effect migration (P11b):
+ * The pool singleton is replaced by an Effect-managed PgClient Layer.
+ * Pool lifecycle is automatic via Effect scope — closeInternalDB() is
+ * no longer needed (kept for backward compat but delegates to PgClient).
+ * The existing internalQuery/internalExecute API is unchanged.
  */
 
+import * as crypto from "crypto";
+import { Context, Effect, Layer, Schedule, Duration, Fiber } from "effect";
 import { createLogger } from "@atlas/api/lib/logger";
 
 const log = createLogger("internal-db");
 
+// ---------------------------------------------------------------------------
+// Connection URL encryption (AES-256-GCM)
+// ---------------------------------------------------------------------------
+
+const ENCRYPTION_ALGORITHM = "aes-256-gcm";
+const IV_LENGTH = 12; // 96-bit IV recommended for GCM
+const AUTH_TAG_LENGTH = 16;
+
+let _cachedKey: { raw: string; key: Buffer } | null = null;
+
+/**
+ * Returns the 32-byte encryption key derived via SHA-256 from
+ * ATLAS_ENCRYPTION_KEY (takes precedence) or BETTER_AUTH_SECRET.
+ * Returns null if neither is set. Result is cached.
+ */
+export function getEncryptionKey(): Buffer | null {
+  const raw = process.env.ATLAS_ENCRYPTION_KEY ?? process.env.BETTER_AUTH_SECRET;
+  if (!raw) return null;
+  if (_cachedKey && _cachedKey.raw === raw) return _cachedKey.key;
+  // Derive a fixed 32-byte key via SHA-256 so any-length secret works
+  const key = crypto.createHash("sha256").update(raw).digest();
+  _cachedKey = { raw, key };
+  return key;
+}
+
+/** @internal Reset cached encryption key — for testing only. */
+export function _resetEncryptionKeyCache(): void {
+  _cachedKey = null;
+}
+
+/**
+ * Encrypts a connection URL using AES-256-GCM.
+ * Returns `iv:authTag:ciphertext` (all base64). Returns the plaintext
+ * unchanged if no encryption key is available.
+ */
+export function encryptUrl(plaintext: string): string {
+  const key = getEncryptionKey();
+  if (!key) return plaintext;
+
+  const iv = crypto.randomBytes(IV_LENGTH);
+  const cipher = crypto.createCipheriv(ENCRYPTION_ALGORITHM, key, iv, { authTagLength: AUTH_TAG_LENGTH });
+  const encrypted = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
+  const authTag = cipher.getAuthTag();
+
+  // `:` is safe as delimiter — base64 alphabet is A-Za-z0-9+/= (no colon)
+  return `${iv.toString("base64")}:${authTag.toString("base64")}:${encrypted.toString("base64")}`;
+}
+
+/**
+ * Decrypts a connection URL encrypted by `encryptUrl()`.
+ * Plaintext detection (two checks):
+ *   1. Starts with a URL scheme (`postgresql://`, `mysql://`, etc.) → plaintext
+ *   2. Not exactly 3 colon-separated parts (`iv:authTag:ciphertext`) → plaintext
+ * Returns plaintext values as-is for backward compatibility with pre-encryption data.
+ */
+export function decryptUrl(stored: string): string {
+  if (isPlaintextUrl(stored)) return stored;
+
+  const key = getEncryptionKey();
+  if (!key) {
+    log.error("Encrypted connection URL found but no encryption key is available — set ATLAS_ENCRYPTION_KEY or BETTER_AUTH_SECRET");
+    throw new Error("Cannot decrypt connection URL: no encryption key available");
+  }
+
+  const parts = stored.split(":");
+  if (parts.length !== 3) {
+    log.error({ partCount: parts.length }, "Stored connection URL is not plaintext and does not match encrypted format (expected 3 colon-separated parts)");
+    throw new Error("Failed to decrypt connection URL: unrecognized format");
+  }
+
+  try {
+    const iv = Buffer.from(parts[0], "base64");
+    const authTag = Buffer.from(parts[1], "base64");
+    const ciphertext = Buffer.from(parts[2], "base64");
+
+    const decipher = crypto.createDecipheriv(ENCRYPTION_ALGORITHM, key, iv, { authTagLength: AUTH_TAG_LENGTH });
+    decipher.setAuthTag(authTag);
+    const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+    return decrypted.toString("utf8");
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err) },
+      "Failed to decrypt connection URL — data may be corrupted or key may have changed",
+    );
+    throw new Error("Failed to decrypt connection URL", { cause: err });
+  }
+}
+
+/** Returns true if the stored value looks like a plaintext URL (any URI scheme, not just database schemes). */
+export function isPlaintextUrl(value: string): boolean {
+  return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(value);
+}
+
 /** Typed interface for the internal pg.Pool — avoids importing pg at module level. */
+export interface InternalPoolClient {
+  query(sql: string, params?: unknown[]): Promise<{ rows: Record<string, unknown>[] }>;
+  release(): void;
+}
+
 export interface InternalPool {
   query(sql: string, params?: unknown[]): Promise<{ rows: Record<string, unknown>[] }>;
+  connect(): Promise<InternalPoolClient>;
   end(): Promise<void>;
   on(event: "error", listener: (err: Error) => void): void;
 }
 
+// ── Effect Service: InternalDB (P11b) ───────────────────────────────
+
+/**
+ * InternalDB Effect service — provides access to the internal Postgres pool.
+ *
+ * Effect-managed lifecycle: pool is created during Layer construction and
+ * closed automatically when the Layer scope ends. Replaces the manual
+ * _pool singleton + closeInternalDB() pattern.
+ */
+export interface InternalDBShape {
+  /** Execute a parameterized query returning typed rows. */
+  query<T extends Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
+  /**
+   * Fire-and-forget write (uses circuit breaker internally).
+   * Intentionally void (not Effect/Promise) — called from onFinish callbacks
+   * in the agent loop where back-pressure would block stream finalization.
+   */
+  execute(sql: string, params?: unknown[]): void;
+  /** Whether the internal DB is available. */
+  readonly available: boolean;
+  /** The underlying pool (for advanced usage like migrations). Null when DATABASE_URL is not set. */
+  readonly pool: InternalPool | null;
+}
+
+export class InternalDB extends Context.Tag("InternalDB")<
+  InternalDB,
+  InternalDBShape
+>() {}
+
+/**
+ * Create the Live Layer for InternalDB.
+ *
+ * Bridge layer: creates the pool via the existing getInternalDB() singleton
+ * to preserve production-tested pg.Pool configuration (sslmode normalization,
+ * max connections, idle timeout). Pool cleanup is managed by an Effect
+ * finalizer that delegates to closeInternalDB().
+ *
+ * Future: replace getInternalDB() with PgClient.make() for native @effect/sql.
+ */
+export function makeInternalDBLive(): Layer.Layer<InternalDB> {
+  return Layer.scoped(
+    InternalDB,
+    Effect.gen(function* () {
+      const databaseUrl = process.env.DATABASE_URL;
+      if (!databaseUrl) {
+        return {
+          query: async () => { throw new Error("DATABASE_URL is not set"); },
+          execute: () => { log.debug("internalExecute called but DATABASE_URL is not set — no-op"); },
+          available: false,
+          pool: null,
+        } satisfies InternalDBShape;
+      }
+
+      // Create pool via the existing getInternalDB() for now.
+      // This preserves the pg.Pool configuration (sslmode normalization,
+      // max connections, idle timeout) that has been production-tested.
+      // Future: replace with PgClient.make({ url: Redacted.make(databaseUrl) })
+      const pool = getInternalDB();
+
+      yield* Effect.addFinalizer(() =>
+        Effect.tryPromise({
+          try: () => closeInternalDB(),
+          catch: (err) => (err instanceof Error ? err.message : String(err)),
+        }).pipe(
+          Effect.catchAll((errMsg) => {
+            log.warn({ err: errMsg }, "Error closing internal DB pool via Effect finalizer");
+            return Effect.void;
+          }),
+        ),
+      );
+
+      return {
+        query: async <T extends Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]> => {
+          const result = await pool.query(sql, params);
+          return result.rows as T[];
+        },
+        execute: (sql: string, params?: unknown[]) => internalExecute(sql, params),
+        available: true,
+        pool,
+      } satisfies InternalDBShape;
+    }),
+  );
+}
+
+/** Create a test Layer for InternalDB. */
+export function createInternalDBTestLayer(
+  partial: Partial<InternalDBShape> = {},
+): Layer.Layer<InternalDB> {
+  const mockPool: InternalPool = {
+    query: async () => ({ rows: [] }),
+    async connect() {
+      return { query: async () => ({ rows: [] }), release() {} };
+    },
+    end: async () => {},
+    on: () => {},
+  };
+  return Layer.succeed(InternalDB, {
+    query: partial.query ?? (async () => []),
+    execute: partial.execute ?? (() => {}),
+    available: partial.available ?? true,
+    pool: partial.pool ?? mockPool,
+  });
+}
+
+// ── Legacy singleton (backward compat) ──────────────────────────────
+
 let _pool: InternalPool | null = null;
 
 /** Returns true if DATABASE_URL is configured. */
@@ -27,7 +240,8 @@ export function hasInternalDB(): boolean {
 /** Returns the singleton pg.Pool for the internal database. Throws if DATABASE_URL is not set. */
 export function getInternalDB(): InternalPool {
   if (!_pool) {
-    if (!process.env.DATABASE_URL) {
+    const databaseUrl = process.env.DATABASE_URL;
+    if (!databaseUrl) {
       throw new Error(
         "DATABASE_URL is not set. Atlas internal database requires a PostgreSQL connection string."
       );
@@ -35,7 +249,7 @@ export function getInternalDB(): InternalPool {
     // eslint-disable-next-line @typescript-eslint/no-require-imports
     const { Pool } = require("pg");
     // Normalize sslmode: pg v8 treats 'require' as 'verify-full' but warns.
-    const connString = process.env.DATABASE_URL!.replace(
+    const connString = databaseUrl.replace(
       /([?&])sslmode=require(?=&|$)/,
       "$1sslmode=verify-full",
     );
@@ -92,14 +306,81 @@ let _consecutiveFailures = 0;
 const MAX_CONSECUTIVE_FAILURES = 5;
 let _circuitOpen = false;
 let _droppedCount = 0;
+/** Recovery fiber — when set, a background fiber is attempting exponential backoff recovery. */
+let _recoveryFiber: Fiber.RuntimeFiber<void, never> | null = null;
+
+/**
+ * Exponential backoff recovery schedule for the circuit breaker.
+ * Starts at 30s, doubles each attempt, caps at 5 minutes.
+ * Retries up to 5 times with increasing delays (30s, 60s, 120s, 240s, 300s).
+ * If all retries fail, circuit remains open and recovery re-triggers on next write.
+ */
+const RECOVERY_SCHEDULE = Schedule.exponential(Duration.seconds(30)).pipe(
+  Schedule.union(Schedule.spaced(Duration.minutes(5))),
+  // Cap at 5 retries (30s → 60s → 120s → 240s → 300s)
+  Schedule.intersect(Schedule.recurs(5)),
+  Schedule.map(([duration]) => duration),
+);
+
+/**
+ * Start an exponential-backoff recovery probe. On success, re-opens the circuit.
+ * On exhaustion of retries, the circuit remains open and the recovery fiber clears
+ * itself so the next internalExecute call re-triggers recovery.
+ *
+ * After an initial 30s delay, makes the first probe attempt. On failure, retries
+ * up to 5 times with exponential backoff (30s, 60s, 120s, 240s, 300s).
+ * Worst-case recovery takes ~13 minutes from circuit trip to retry exhaustion.
+ */
+function _startRecovery(): void {
+  if (_recoveryFiber) return;
+
+  const probe = Effect.gen(function* () {
+    const pool = getInternalDB();
+    yield* Effect.tryPromise({
+      try: () => pool.query("SELECT 1"),
+      catch: (err) => (err instanceof Error ? err : new Error(String(err))),
+    });
+  });
+
+  const recovery = Effect.sleep(Duration.seconds(30)).pipe(
+    Effect.andThen(
+      probe.pipe(Effect.retry(RECOVERY_SCHEDULE)),
+    ),
+    Effect.andThen(
+      Effect.sync(() => {
+        const dropped = _droppedCount;
+        _circuitOpen = false;
+        _consecutiveFailures = 0;
+        _droppedCount = 0;
+        _recoveryFiber = null;
+        log.info({ droppedCount: dropped }, "Internal DB circuit breaker recovered — fire-and-forget writes resumed");
+      }),
+    ),
+    Effect.catchAll((err) => {
+      // All retries exhausted — keep circuit open, clear fiber so next write re-triggers recovery
+      _recoveryFiber = null;
+      log.error(
+        { err: err instanceof Error ? err.message : String(err), droppedCount: _droppedCount },
+        "Internal DB circuit breaker recovery exhausted — circuit remains open, will re-attempt on next write",
+      );
+      return Effect.void;
+    }),
+  );
+
+  _recoveryFiber = Effect.runFork(recovery);
+}
 
 /** Fire-and-forget query — async errors are logged, never thrown.
- * After 5 consecutive failures, a circuit breaker trips and silently
- * drops all calls for 60s before retrying. Throws synchronously if
- * DATABASE_URL is not set (callers should check hasInternalDB() first). */
+ * After 5 consecutive failures, a circuit breaker trips and drops
+ * all calls until recovery succeeds. Recovery uses exponential backoff
+ * (30s → 60s → 120s → 240s → 300s) via Effect.retry. Throws
+ * synchronously if DATABASE_URL is not set (callers should check
+ * hasInternalDB() first). */
 export function internalExecute(sql: string, params?: unknown[]): void {
   if (_circuitOpen) {
     _droppedCount++;
+    // Re-trigger recovery if previous attempt exhausted retries
+    if (!_recoveryFiber) _startRecovery();
     return;
   }
   const pool = getInternalDB();
@@ -110,14 +391,7 @@ export function internalExecute(sql: string, params?: unknown[]): void {
       if (_consecutiveFailures >= MAX_CONSECUTIVE_FAILURES && !_circuitOpen) {
         _circuitOpen = true;
         log.error("Internal DB circuit breaker open — fire-and-forget writes disabled until recovery");
-        // Try to recover every 60s
-        setTimeout(() => {
-          const dropped = _droppedCount;
-          _circuitOpen = false;
-          _consecutiveFailures = 0;
-          _droppedCount = 0;
-          log.info({ droppedCount: dropped }, "Internal DB circuit breaker recovered — fire-and-forget writes resumed");
-        }, 60_000).unref();
+        _startRecovery();
       }
       if (!_circuitOpen) {
         log.error(
@@ -137,142 +411,714 @@ export function _resetCircuitBreaker(): void {
   _consecutiveFailures = 0;
   _circuitOpen = false;
   _droppedCount = 0;
+  if (_recoveryFiber) {
+    Effect.runFork(Fiber.interrupt(_recoveryFiber));
+    _recoveryFiber = null;
+  }
+}
+
+/**
+ * Log a warning when DATABASE_URL and ATLAS_DATASOURCE_URL resolve to the
+ * same Postgres database. Internal tables (auth, audit, settings) will
+ * share the public schema with analytics data.
+ *
+ * This is intentional in single-DB deployments (e.g. Railway with one
+ * Postgres addon) but can confuse the seed script or the agent — call
+ * this once at migration time to surface the situation.
+ */
+function warnIfSharedDatabase(): void {
+  const databaseUrl = process.env.DATABASE_URL;
+  const datasourceUrl = process.env.ATLAS_DATASOURCE_URL;
+  if (!databaseUrl || !datasourceUrl) return;
+
+  try {
+    const internalParsed = new URL(databaseUrl);
+    const datasourceParsed = new URL(datasourceUrl);
+
+    // Compare host + port + pathname (database name) to detect shared DB
+    const sameHost = internalParsed.hostname === datasourceParsed.hostname;
+    const samePort = (internalParsed.port || "5432") === (datasourceParsed.port || "5432");
+    const sameDB = internalParsed.pathname === datasourceParsed.pathname;
+
+    if (sameHost && samePort && sameDB) {
+      log.warn(
+        "DATABASE_URL and ATLAS_DATASOURCE_URL point to the same database — " +
+        "Atlas internal tables will share the schema with analytics data. " +
+        "Consider using a separate database for ATLAS_DATASOURCE_URL to isolate analytics data.",
+      );
+    }
+  } catch {
+    // URL parsing failed — not critical, skip the warning
+    log.debug("Could not parse DATABASE_URL or ATLAS_DATASOURCE_URL for shared-DB detection");
+  }
 }
 
-/** Idempotent migration: creates audit_log, conversations, and messages tables. */
+/**
+ * Idempotent migration: runs versioned SQL migrations from `migrations/`
+ * directory, then applies data seeds.
+ *
+ * Replaces the old imperative DDL approach (152 individual pool.query calls)
+ * with a file-based migration runner tracked in `__atlas_migrations`. See #978.
+ */
 export async function migrateInternalDB(): Promise<void> {
+  // Warn when DATABASE_URL and ATLAS_DATASOURCE_URL resolve to the same
+  // database — internal tables will share the schema with analytics data.
+  // This is intentional in single-DB deployments but can surprise operators
+  // who expect isolation. (#962)
+  warnIfSharedDatabase();
+
   const pool = getInternalDB();
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS audit_log (
-      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-      timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
-      user_id TEXT,
-      user_label TEXT,
-      auth_mode TEXT NOT NULL,
-      sql TEXT NOT NULL,
-      duration_ms INTEGER NOT NULL,
-      row_count INTEGER,
-      success BOOLEAN NOT NULL,
-      error TEXT
-    );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_audit_log_timestamp ON audit_log(timestamp);`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_audit_log_user_id ON audit_log(user_id);`);
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS conversations (
-      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-      user_id TEXT,
-      title TEXT,
-      surface TEXT DEFAULT 'web',
-      connection_id TEXT,
-      created_at TIMESTAMPTZ DEFAULT now(),
-      updated_at TIMESTAMPTZ DEFAULT now()
+
+  const { runMigrations, runSeeds } = await import("@atlas/api/lib/db/migrate");
+  await runMigrations(pool);
+  await runSeeds(pool);
+
+  log.info("Internal DB migration complete");
+}
+
+// Old imperative DDL removed — see migrations/0000_baseline.sql (#978)
+
+// seedPromptLibrary moved to migrate.ts → runSeeds() (#978)
+
+/**
+ * Load admin-managed connections from the internal DB and register them
+ * in the ConnectionRegistry. Idempotent — safe to call at startup.
+ * Silently skips if no internal DB or the connections table doesn't exist yet.
+ */
+export async function loadSavedConnections(): Promise<number> {
+  if (!hasInternalDB()) return 0;
+
+  // Lazy-import to avoid circular dependency at module level
+  const { connections } = await import("@atlas/api/lib/db/connection");
+
+  try {
+    const rows = await internalQuery<{
+      id: string;
+      url: string;
+      type: string;
+      description: string | null;
+      schema_name: string | null;
+    }>("SELECT id, url, type, description, schema_name FROM connections");
+
+    let registered = 0;
+    for (const row of rows) {
+      try {
+        const url = decryptUrl(row.url);
+        connections.register(row.id, {
+          url,
+          description: row.description ?? undefined,
+          schema: row.schema_name ?? undefined,
+        });
+        registered++;
+      } catch (err) {
+        log.warn(
+          { connectionId: row.id, err: err instanceof Error ? err.message : String(err) },
+          "Failed to register saved connection — skipping",
+        );
+      }
+    }
+
+    if (registered > 0) {
+      log.info({ count: registered }, "Loaded saved connections from internal DB");
+    }
+    return registered;
+  } catch (err) {
+    // Table may not exist yet (pre-migration) — that's expected on first boot
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err) },
+      "Could not load saved connections (table may not exist yet)",
     );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id);`);
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS messages (
-      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-      conversation_id UUID REFERENCES conversations(id) ON DELETE CASCADE,
-      role TEXT NOT NULL,
-      content JSONB NOT NULL,
-      created_at TIMESTAMPTZ DEFAULT now()
+    return 0;
+  }
+}
+
+// ── Learned pattern helpers ─────────────────────────────────────────
+
+/**
+ * Find a learned pattern by exact normalized SQL match for the given org.
+ * Returns the pattern's id, confidence, and repetition count, or null if not found.
+ */
+export async function findPatternBySQL(
+  orgId: string | null | undefined,
+  patternSql: string,
+): Promise<{ id: string; confidence: number; repetitionCount: number } | null> {
+  const pool = getInternalDB();
+  const params: unknown[] = [patternSql];
+  let orgClause: string;
+  if (orgId) {
+    params.push(orgId);
+    orgClause = `org_id = $2`;
+  } else {
+    orgClause = `org_id IS NULL`;
+  }
+
+  const result = await pool.query(
+    `SELECT id, confidence, repetition_count FROM learned_patterns WHERE pattern_sql = $1 AND ${orgClause} LIMIT 1`,
+    params,
+  );
+
+  if (result.rows.length === 0) return null;
+  const row = result.rows[0];
+  return {
+    id: row.id as string,
+    confidence: row.confidence as number,
+    repetitionCount: row.repetition_count as number,
+  };
+}
+
+/**
+ * Insert a new learned pattern. Fire-and-forget — errors are logged, never thrown.
+ */
+export function insertLearnedPattern(pattern: {
+  orgId: string | null | undefined;
+  patternSql: string;
+  description: string;
+  sourceEntity: string;
+  sourceQueries: string[];
+  proposedBy: string;
+}): void {
+  internalExecute(
+    `INSERT INTO learned_patterns (org_id, pattern_sql, description, source_entity, source_queries, confidence, repetition_count, status, proposed_by)
+     VALUES ($1, $2, $3, $4, $5, 0.1, 1, 'pending', $6)`,
+    [
+      pattern.orgId ?? null,
+      pattern.patternSql,
+      pattern.description,
+      pattern.sourceEntity,
+      JSON.stringify(pattern.sourceQueries),
+      pattern.proposedBy,
+    ],
+  );
+}
+
+/**
+ * Increment repetition_count by 1 and increase confidence by 0.1 (capped at 1.0).
+ * When sourceFingerprint is provided, appends it to source_queries (capped at 100 entries).
+ * Fire-and-forget — errors are logged, never thrown.
+ */
+export function incrementPatternCount(id: string, sourceFingerprint?: string): void {
+  if (sourceFingerprint) {
+    const newEntry = JSON.stringify([sourceFingerprint]);
+    internalExecute(
+      `UPDATE learned_patterns SET
+        repetition_count = repetition_count + 1,
+        confidence = LEAST(1.0, confidence + 0.1),
+        source_queries = CASE
+          WHEN source_queries IS NULL THEN $2::jsonb
+          WHEN jsonb_array_length(source_queries) >= 100 THEN source_queries
+          ELSE source_queries || $2::jsonb
+        END,
+        updated_at = now()
+      WHERE id = $1`,
+      [id, newEntry],
     );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id);`);
-
-  // Slack integration tables
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS slack_installations (
-      team_id TEXT PRIMARY KEY,
-      bot_token TEXT NOT NULL,
-      installed_at TIMESTAMPTZ DEFAULT now()
+  } else {
+    internalExecute(
+      `UPDATE learned_patterns SET
+        repetition_count = repetition_count + 1,
+        confidence = LEAST(1.0, confidence + 0.1),
+        updated_at = now()
+      WHERE id = $1`,
+      [id],
     );
-  `);
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS slack_threads (
-      thread_ts TEXT NOT NULL,
-      channel_id TEXT NOT NULL,
-      conversation_id UUID NOT NULL,
-      PRIMARY KEY (thread_ts, channel_id)
+  }
+}
+
+/** Row shape returned by getApprovedPatterns. */
+export interface ApprovedPatternRow {
+  id: string;
+  org_id: string | null;
+  pattern_sql: string;
+  description: string | null;
+  source_entity: string | null;
+  /** Confidence score between 0.0 and 1.0. */
+  confidence: number;
+  [key: string]: unknown;
+}
+
+/** Row shape for query_suggestions table. */
+export interface QuerySuggestionRow {
+  id: string;
+  org_id: string | null;
+  description: string;
+  pattern_sql: string;
+  normalized_hash: string;
+  tables_involved: string; // JSONB string, parse to string[]
+  primary_table: string | null;
+  frequency: number;
+  clicked_count: number;
+  score: number;
+  last_seen_at: string;
+  created_at: string;
+  updated_at: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Fetch approved learned patterns, scoped to an org (or global when orgId is null).
+ * Ordered by confidence DESC, capped at 100 rows.
+ */
+export async function getApprovedPatterns(orgId: string | null): Promise<ApprovedPatternRow[]> {
+  if (!hasInternalDB()) return [];
+
+  return internalQuery<ApprovedPatternRow>(
+    orgId
+      ? `SELECT id, org_id, pattern_sql, description, source_entity, confidence
+         FROM learned_patterns
+         WHERE status = 'approved' AND (org_id = $1 OR org_id IS NULL)
+         ORDER BY confidence DESC
+         LIMIT 100`
+      : `SELECT id, org_id, pattern_sql, description, source_entity, confidence
+         FROM learned_patterns
+         WHERE status = 'approved' AND org_id IS NULL
+         ORDER BY confidence DESC
+         LIMIT 100`,
+    orgId ? [orgId] : [],
+  );
+}
+
+export async function upsertSuggestion(suggestion: {
+  orgId: string | null;
+  description: string;
+  patternSql: string;
+  normalizedHash: string;
+  tablesInvolved: string[];
+  primaryTable: string | null;
+  frequency: number;
+  score: number;
+  lastSeenAt: Date;
+}): Promise<"created" | "updated" | "skipped"> {
+  if (!hasInternalDB()) return "skipped";
+  try {
+    const rows = await internalQuery<{ id: string; created: boolean }>(
+      `INSERT INTO query_suggestions (org_id, description, pattern_sql, normalized_hash, tables_involved, primary_table, frequency, score, last_seen_at)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+       ON CONFLICT ON CONSTRAINT uq_query_suggestions_org_hash DO UPDATE SET
+         frequency = EXCLUDED.frequency,
+         score = EXCLUDED.score,
+         last_seen_at = EXCLUDED.last_seen_at,
+         updated_at = NOW()
+       RETURNING id, (xmax = 0) AS created`,
+      [
+        suggestion.orgId,
+        suggestion.description,
+        suggestion.patternSql,
+        suggestion.normalizedHash,
+        JSON.stringify(suggestion.tablesInvolved),
+        suggestion.primaryTable,
+        suggestion.frequency,
+        suggestion.score,
+        suggestion.lastSeenAt.toISOString(),
+      ]
     );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_slack_threads_conversation ON slack_threads(conversation_id);`);
-
-  // Action framework tables
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS action_log (
-      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-      requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-      resolved_at  TIMESTAMPTZ,
-      executed_at  TIMESTAMPTZ,
-      requested_by TEXT,
-      approved_by  TEXT,
-      auth_mode    TEXT NOT NULL,
-      action_type  TEXT NOT NULL,
-      target       TEXT NOT NULL,
-      summary      TEXT NOT NULL,
-      payload      JSONB NOT NULL,
-      status       TEXT NOT NULL DEFAULT 'pending',
-      result       JSONB,
-      error        TEXT,
-      rollback_info JSONB,
-      conversation_id UUID,
-      request_id   TEXT
+    return rows[0]?.created ? "created" : "updated";
+  } catch (err) {
+    log.warn({ err: err instanceof Error ? err.message : String(err) }, "Failed to upsert suggestion");
+    return "skipped";
+  }
+}
+
+export async function getSuggestionsByTables(
+  orgId: string | null,
+  tables: string[],
+  limit: number = 10
+): Promise<QuerySuggestionRow[]> {
+  if (!hasInternalDB()) return [];
+  try {
+    const orgClause = orgId != null ? "org_id = $1" : "org_id IS NULL";
+    const params: unknown[] = orgId != null ? [orgId] : [];
+    const nextIdx = params.length + 1;
+
+    let tableClause: string;
+    if (tables.length === 1) {
+      tableClause = `primary_table = $${nextIdx}`;
+      params.push(tables[0]);
+    } else {
+      tableClause = `tables_involved ?| $${nextIdx}::text[]`;
+      params.push(tables);
+    }
+
+    params.push(limit);
+    const limitIdx = params.length;
+
+    return await internalQuery<QuerySuggestionRow>(
+      `SELECT * FROM query_suggestions WHERE ${orgClause} AND ${tableClause} ORDER BY score DESC LIMIT $${limitIdx}`,
+      params
     );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_action_log_requested_by ON action_log(requested_by);`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_action_log_status ON action_log(status);`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_action_log_action_type ON action_log(action_type);`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_action_log_conversation ON action_log(conversation_id);`);
-
-  // Multi-database production hardening: add source tracking columns to audit_log
-  await pool.query(`ALTER TABLE audit_log ADD COLUMN IF NOT EXISTS source_id TEXT, ADD COLUMN IF NOT EXISTS source_type TEXT, ADD COLUMN IF NOT EXISTS target_host TEXT;`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_audit_log_source_id ON audit_log(source_id);`);
-
-  // Saved/starred conversations
-  await pool.query(`ALTER TABLE conversations ADD COLUMN IF NOT EXISTS starred BOOLEAN NOT NULL DEFAULT false;`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_conversations_starred ON conversations(user_id, starred) WHERE starred = true;`);
-
-  // Scheduled tasks
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS scheduled_tasks (
-      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-      owner_id TEXT NOT NULL,
-      name TEXT NOT NULL,
-      question TEXT NOT NULL,
-      cron_expression TEXT NOT NULL,
-      delivery_channel TEXT NOT NULL DEFAULT 'webhook',
-      recipients JSONB NOT NULL DEFAULT '[]',
-      connection_id TEXT,
-      approval_mode TEXT NOT NULL DEFAULT 'auto',
-      enabled BOOLEAN NOT NULL DEFAULT true,
-      last_run_at TIMESTAMPTZ,
-      next_run_at TIMESTAMPTZ,
-      created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-      updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+  } catch (err) {
+    log.warn({ err: err instanceof Error ? err.message : String(err) }, "Failed to get suggestions by tables");
+    return [];
+  }
+}
+
+export async function getPopularSuggestions(
+  orgId: string | null,
+  limit: number = 10
+): Promise<QuerySuggestionRow[]> {
+  if (!hasInternalDB()) return [];
+  try {
+    const orgClause = orgId != null ? "org_id = $1" : "org_id IS NULL";
+    const params: unknown[] = orgId != null ? [orgId, limit] : [limit];
+    const limitIdx = params.length;
+
+    return await internalQuery<QuerySuggestionRow>(
+      `SELECT * FROM query_suggestions WHERE ${orgClause} ORDER BY score DESC LIMIT $${limitIdx}`,
+      params
     );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_owner ON scheduled_tasks(owner_id);`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_enabled ON scheduled_tasks(enabled) WHERE enabled = true;`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_next_run ON scheduled_tasks(next_run_at) WHERE enabled = true;`);
-
-  await pool.query(`
-    CREATE TABLE IF NOT EXISTS scheduled_task_runs (
-      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-      task_id UUID NOT NULL REFERENCES scheduled_tasks(id) ON DELETE CASCADE,
-      started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-      completed_at TIMESTAMPTZ,
-      status TEXT NOT NULL DEFAULT 'running',
-      conversation_id UUID,
-      action_id UUID,
-      error TEXT,
-      tokens_used INTEGER,
-      created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+  } catch (err) {
+    log.warn({ err: err instanceof Error ? err.message : String(err) }, "Failed to get popular suggestions");
+    return [];
+  }
+}
+
+export function incrementSuggestionClick(
+  id: string,
+  orgId: string | null
+): void {
+  if (!hasInternalDB()) return;
+  const orgClause = orgId != null ? "org_id = $1" : "org_id IS NULL";
+  const params: unknown[] = orgId != null ? [orgId, id] : [id];
+  const idIdx = params.length;
+
+  internalExecute(
+    `UPDATE query_suggestions SET clicked_count = clicked_count + 1 WHERE ${orgClause} AND id = $${idIdx}`,
+    params
+  );
+}
+
+export async function deleteSuggestion(
+  id: string,
+  orgId: string | null
+): Promise<boolean> {
+  if (!hasInternalDB()) return false;
+  const orgClause = orgId != null ? "org_id = $1" : "org_id IS NULL";
+  const params: unknown[] = orgId != null ? [orgId, id] : [id];
+  const idIdx = params.length;
+
+  const rows = await internalQuery<{ id: string }>(
+    `DELETE FROM query_suggestions WHERE ${orgClause} AND id = $${idIdx} RETURNING id`,
+    params
+  );
+  return rows.length > 0;
+}
+
+export async function getAuditLogQueries(
+  orgId: string | null,
+  limit: number = 5000
+): Promise<Array<{ sql: string; tables_accessed: string | null; timestamp: string }>> {
+  if (!hasInternalDB()) return [];
+  try {
+    const orgClause = orgId != null ? "org_id = $1" : "org_id IS NULL";
+    const params: unknown[] = orgId != null ? [orgId, limit] : [limit];
+    const limitIdx = params.length;
+
+    return await internalQuery<{ sql: string; tables_accessed: string | null; timestamp: string }>(
+      `SELECT sql, tables_accessed, timestamp FROM audit_log WHERE ${orgClause} AND success = true AND sql IS NOT NULL ORDER BY timestamp DESC LIMIT $${limitIdx}`,
+      params
     );
-  `);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_task ON scheduled_task_runs(task_id);`);
-  await pool.query(`CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_status ON scheduled_task_runs(status);`);
+  } catch (err) {
+    log.warn({ err: err instanceof Error ? err.message : String(err) }, "Failed to get audit log queries");
+    return [];
+  }
+}
+
+// ── Workspace lifecycle helpers (0.9.0) ─────────────────────────────
+
+export type WorkspaceStatus = "active" | "suspended" | "deleted";
+export type PlanTier = "free" | "trial" | "team" | "enterprise";
+
+export interface WorkspaceRow {
+  id: string;
+  name: string;
+  slug: string;
+  workspace_status: WorkspaceStatus;
+  plan_tier: PlanTier;
+  byot: boolean;
+  stripe_customer_id: string | null;
+  trial_ends_at: string | null;
+  suspended_at: string | null;
+  deleted_at: string | null;
+  region: string | null;
+  region_assigned_at: string | null;
+  createdAt: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Get the workspace status for an organization.
+ * Returns null if the org doesn't exist or internal DB is unavailable.
+ * Throws on database errors — callers must handle failures explicitly.
+ */
+export async function getWorkspaceStatus(orgId: string): Promise<WorkspaceStatus | null> {
+  if (!hasInternalDB()) return null;
+  const rows = await internalQuery<{ workspace_status: WorkspaceStatus }>(
+    `SELECT workspace_status FROM organization WHERE id = $1`,
+    [orgId],
+  );
+  return rows[0]?.workspace_status ?? null;
+}
+
+/**
+ * Get full workspace details for an organization.
+ */
+export async function getWorkspaceDetails(orgId: string): Promise<WorkspaceRow | null> {
+  if (!hasInternalDB()) return null;
+  const rows = await internalQuery<WorkspaceRow>(
+    `SELECT id, name, slug, workspace_status, plan_tier, byot, stripe_customer_id, trial_ends_at, suspended_at, deleted_at, region, region_assigned_at, "createdAt"
+     FROM organization WHERE id = $1`,
+    [orgId],
+  );
+  return rows[0] ?? null;
+}
+
+/**
+ * Update workspace status. Returns true if the org was found and updated,
+ * false if no row matched the given orgId.
+ */
+export async function updateWorkspaceStatus(
+  orgId: string,
+  status: WorkspaceStatus,
+): Promise<boolean> {
+  const pool = getInternalDB();
+  const timestampCol = status === "suspended" ? "suspended_at" : status === "deleted" ? "deleted_at" : null;
+
+  let sql: string;
+  if (timestampCol) {
+    sql = `UPDATE organization SET workspace_status = $1, ${timestampCol} = now() WHERE id = $2 RETURNING id`;
+  } else {
+    // Activating: clear both timestamps
+    sql = `UPDATE organization SET workspace_status = $1, suspended_at = NULL, deleted_at = NULL WHERE id = $2 RETURNING id`;
+  }
+
+  const result = await pool.query(sql, [status, orgId]);
+  return result.rows.length > 0;
+}
+
+/**
+ * Update workspace plan tier. Returns true if the org was found and updated,
+ * false if no row matched the given orgId.
+ */
+export async function updateWorkspacePlanTier(
+  orgId: string,
+  planTier: PlanTier,
+): Promise<boolean> {
+  const pool = getInternalDB();
+  const result = await pool.query(
+    `UPDATE organization SET plan_tier = $1 WHERE id = $2 RETURNING id`,
+    [planTier, orgId],
+  );
+  return result.rows.length > 0;
+}
+
+/**
+ * Get the region assigned to a workspace. Returns null if no region is assigned
+ * or the workspace doesn't exist.
+ */
+export async function getWorkspaceRegion(orgId: string): Promise<string | null> {
+  if (!hasInternalDB()) return null;
+  const rows = await internalQuery<{ region: string | null }>(
+    `SELECT region FROM organization WHERE id = $1`,
+    [orgId],
+  );
+  return rows[0]?.region ?? null;
+}
+
+/**
+ * Assign a region to a workspace. Region is immutable — once set, returns
+ * `{ assigned: false, existing: <current region> }` without updating.
+ * On first assignment, returns `{ assigned: true }`. If the workspace
+ * does not exist, returns `{ assigned: false }` without an `existing` field.
+ */
+export async function setWorkspaceRegion(
+  orgId: string,
+  region: string,
+): Promise<{ assigned: boolean; existing?: string }> {
+  const pool = getInternalDB();
+  // Only assign if region is currently NULL (immutable after first assignment)
+  const result = await pool.query(
+    `UPDATE organization SET region = $1, region_assigned_at = now()
+     WHERE id = $2 AND region IS NULL RETURNING id`,
+    [region, orgId],
+  );
+  if (result.rows.length > 0) {
+    return { assigned: true };
+  }
+  // Check if workspace exists with a different region already set
+  const existing = await internalQuery<{ region: string | null }>(
+    `SELECT region FROM organization WHERE id = $1`,
+    [orgId],
+  );
+  if (existing.length === 0) {
+    return { assigned: false };
+  }
+  return { assigned: false, existing: existing[0].region ?? undefined };
+}
+
+/**
+ * Cascading soft-delete cleanup for a workspace (transactional):
+ * - Soft-deletes conversations (sets deleted_at)
+ * - Hard-deletes org-scoped semantic entities, learned patterns, and query suggestions
+ * - Hard-deletes org-scoped settings
+ * - Disables scheduled tasks
+ *
+ * All operations run inside a single transaction — either all succeed or
+ * none take effect, so retries are always safe.
+ */
+export async function cascadeWorkspaceDelete(orgId: string): Promise<{
+  conversations: number;
+  semanticEntities: number;
+  learnedPatterns: number;
+  suggestions: number;
+  scheduledTasks: number;
+  settings: number;
+}> {
+  const pool = getInternalDB();
+  const client = await pool.connect();
 
-  log.info("Internal DB migration complete (audit_log, conversations, messages, slack, action_log, scheduled_tasks)");
+  try {
+    await client.query("BEGIN");
+
+    const [convResult, seResult, lpResult, qsResult, stResult, settingsResult] = await Promise.all([
+      client.query(
+        `UPDATE conversations SET deleted_at = now(), updated_at = now() WHERE org_id = $1 AND deleted_at IS NULL RETURNING id`,
+        [orgId],
+      ),
+      client.query(
+        `DELETE FROM semantic_entities WHERE org_id = $1 RETURNING id`,
+        [orgId],
+      ),
+      client.query(
+        `DELETE FROM learned_patterns WHERE org_id = $1 RETURNING id`,
+        [orgId],
+      ),
+      client.query(
+        `DELETE FROM query_suggestions WHERE org_id = $1 RETURNING id`,
+        [orgId],
+      ),
+      client.query(
+        `UPDATE scheduled_tasks SET enabled = false, updated_at = now() WHERE org_id = $1 RETURNING id`,
+        [orgId],
+      ),
+      client.query(
+        `DELETE FROM settings WHERE org_id = $1 RETURNING key`,
+        [orgId],
+      ),
+    ]);
+
+    await client.query("COMMIT");
+
+    return {
+      conversations: convResult.rows.length,
+      semanticEntities: seResult.rows.length,
+      learnedPatterns: lpResult.rows.length,
+      suggestions: qsResult.rows.length,
+      scheduledTasks: stResult.rows.length,
+      settings: settingsResult.rows.length,
+    };
+  } catch (err) {
+    await client.query("ROLLBACK").catch(() => {
+      // intentionally ignored: ROLLBACK failure after a failed transaction is non-actionable
+    });
+    throw err;
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * Get a workspace health summary: member count, conversation count,
+ * query count (last 24h), connection count, and scheduled task count.
+ */
+export async function getWorkspaceHealthSummary(orgId: string): Promise<{
+  workspace: WorkspaceRow;
+  members: number;
+  conversations: number;
+  queriesLast24h: number;
+  connections: number;
+  scheduledTasks: number;
+} | null> {
+  if (!hasInternalDB()) return null;
+
+  const workspace = await getWorkspaceDetails(orgId);
+  if (!workspace) return null;
+
+  const [memberRows, convRows, queryRows, connRows, taskRows] = await Promise.all([
+    internalQuery<{ count: number }>(
+      `SELECT COUNT(*)::int as count FROM member WHERE "organizationId" = $1`,
+      [orgId],
+    ),
+    internalQuery<{ count: number }>(
+      `SELECT COUNT(*)::int as count FROM conversations WHERE org_id = $1`,
+      [orgId],
+    ),
+    internalQuery<{ count: number }>(
+      `SELECT COUNT(*)::int as count FROM audit_log WHERE org_id = $1 AND timestamp > now() - interval '24 hours'`,
+      [orgId],
+    ),
+    internalQuery<{ count: number }>(
+      `SELECT COUNT(*)::int as count FROM connections WHERE org_id = $1`,
+      [orgId],
+    ),
+    internalQuery<{ count: number }>(
+      `SELECT COUNT(*)::int as count FROM scheduled_tasks WHERE org_id = $1 AND enabled = true`,
+      [orgId],
+    ),
+  ]);
+
+  return {
+    workspace,
+    members: memberRows[0]?.count ?? 0,
+    conversations: convRows[0]?.count ?? 0,
+    queriesLast24h: queryRows[0]?.count ?? 0,
+    connections: connRows[0]?.count ?? 0,
+    scheduledTasks: taskRows[0]?.count ?? 0,
+  };
+}
+
+// ── Billing helpers (0.9.0 — Stripe billing) ────────────────────────
+
+/**
+ * Update the BYOT (Bring Your Own Token) flag for a workspace.
+ * Returns true if the org was found and updated.
+ */
+export async function updateWorkspaceByot(
+  orgId: string,
+  byot: boolean,
+): Promise<boolean> {
+  const pool = getInternalDB();
+  const result = await pool.query(
+    `UPDATE organization SET byot = $1 WHERE id = $2 RETURNING id`,
+    [byot, orgId],
+  );
+  return result.rows.length > 0;
+}
+
+/**
+ * Set the Stripe customer ID for a workspace.
+ */
+export async function setWorkspaceStripeCustomerId(
+  orgId: string,
+  stripeCustomerId: string,
+): Promise<boolean> {
+  const pool = getInternalDB();
+  const result = await pool.query(
+    `UPDATE organization SET stripe_customer_id = $1 WHERE id = $2 RETURNING id`,
+    [stripeCustomerId, orgId],
+  );
+  return result.rows.length > 0;
+}
+
+/**
+ * Set the trial end date for a workspace.
+ */
+export async function setWorkspaceTrialEndsAt(
+  orgId: string,
+  trialEndsAt: Date,
+): Promise<boolean> {
+  const pool = getInternalDB();
+  const result = await pool.query(
+    `UPDATE organization SET trial_ends_at = $1 WHERE id = $2 RETURNING id`,
+    [trialEndsAt.toISOString(), orgId],
+  );
+  return result.rows.length > 0;
 }