@data-club/ai-hub-server 0.0.1

package/src/bootstrap.ts

@@ -0,0 +1,254 @@
+/**
+ * Boot orchestration: env → open DB → migrate → seed → ready.
+ *
+ * The pieces that compose this (env loader, `openDb`, `runMigrations`,
+ * `runFirstBootSeed`, `createLogger`) are independently importable so tests
+ * and embedders can wire bespoke combinations. `bootServer` is the
+ * convenience entry point the `dev/server.ts` runner and the per-customer
+ * wrapper's Next.js mount call.
+ *
+ * @see SPECIFICATIONS.md §12.1 — env vars
+ * @see SPECIFICATIONS.md §6.7 — signing key bootstrap
+ * @see SPECIFICATIONS.md §6.10 — metrics token bootstrap
+ */
+
+import { join } from "node:path";
+
+import type Database from "better-sqlite3";
+
+import { parseEncryptionKey } from "./auth/encryption.js";
+import { openDb, type DbHandle, type DrizzleDB } from "./db/connection.js";
+import { runMigrations } from "./db/migrate.js";
+import { runFirstBootSeed, type SeedReport } from "./db/seed.js";
+import { createFetchHandler, type FetchHandler } from "./handlers/fetch.js";
+import type { LettaClient } from "./letta/client.js";
+import { createLogger, type Logger } from "./observability/logger.js";
+
+/* ────────────────────────────────────────────────────────────────────────── */
+/*  Config                                                                    */
+/* ────────────────────────────────────────────────────────────────────────── */
+
+/** Subset of `process.env` we care about — narrows for tests. */
+export type EnvLike = Record<string, string | undefined>;
+
+/**
+ * Parsed env config used by `bootServer`. Everything is optional except
+ * `encryptionKey`, which `parseEnvConfig` will raise on if missing.
+ */
+export interface EnvConfig {
+  /** Filesystem dir for SQLite + state. Default `./data`. */
+  dataDir: string;
+  /** SQLite connection string (file path or `:memory:`). */
+  databaseUrl: string;
+  /** Required AES-256 key parsed from `AI_HUB_ENCRYPTION_KEY`. */
+  encryptionKey: Buffer;
+  /** Bootstrap-only signing key (env value, decoded). */
+  signingKeyFromEnv: Buffer | undefined;
+  /** Bootstrap-only metrics scrape token (env value). */
+  metricsScrapeTokenFromEnv: string | undefined;
+  /** Letta API key from env. */
+  lettaApiKey: string | undefined;
+  /** Letta base URL from env. */
+  lettaBaseUrl: string | undefined;
+  /** Letta ADE URL override from env. */
+  lettaAdeUrl: string | undefined;
+  /** Pino log level. */
+  logLevel: string;
+  /** CORS origins (comma-separated). */
+  corsAllowedOrigins: string;
+}
+
+/**
+ * Parses `process.env` (or a test-supplied stand-in) into an `EnvConfig`.
+ * Throws on any required-but-missing variable so the operator gets a clear
+ * error on first boot instead of a confusing runtime failure.
+ *
+ * @param env - the env-var record to read from.
+ * @returns the parsed config.
+ */
+export function parseEnvConfig(env: EnvLike): EnvConfig {
+  const encRaw = env.AI_HUB_ENCRYPTION_KEY;
+  if (!encRaw) {
+    throw new Error(
+      "AI_HUB_ENCRYPTION_KEY is required (see docs/operations/env-vars.md).",
+    );
+  }
+  const encryptionKey = parseEncryptionKey(encRaw);
+
+  const dataDir = env.AI_HUB_DATA_DIR ?? "./data";
+  const databaseUrl = env.DATABASE_URL ?? `file:${join(dataDir, "ai_hub.db")}`;
+
+  return {
+    dataDir,
+    databaseUrl,
+    encryptionKey,
+    signingKeyFromEnv: parseSigningKey(env.SERVICE_TOKEN_SIGNING_KEY),
+    metricsScrapeTokenFromEnv: env.METRICS_SCRAPE_TOKEN || undefined,
+    lettaApiKey: env.LETTA_API_KEY || undefined,
+    // Letta Code's CLI reads `LETTA_BASE_URL` from process.env and
+    // appends `/v1/...` to it. Operators who set the env value
+    // matching the Letta Client SDK's expectation (`http://host/v1`)
+    // would otherwise get double-v1 URLs in the spawned CLI subprocess.
+    // Strip the trailing /v1 once at boot so both ai-hub-server's own
+    // direct API calls (already strip defensively in `letta/client.ts`)
+    // and Letta Code subprocess spawns see the bare host:port form.
+    // Mutating process.env here propagates to every later subprocess
+    // spawn — Letta Code SDK forwards `{...process.env}` verbatim.
+    lettaBaseUrl: normalizeLettaBaseUrl(env.LETTA_BASE_URL),
+    lettaAdeUrl: env.LETTA_ADE_URL || undefined,
+    logLevel: env.LOG_LEVEL ?? "info",
+    corsAllowedOrigins: env.CORS_ALLOWED_ORIGINS ?? "",
+  };
+}
+
+/**
+ * Decodes the bootstrap `SERVICE_TOKEN_SIGNING_KEY` env value.
+ *
+ * Accepts hex (any length, even number of chars) or base64 / base64url. If
+ * the input is neither, treats it as raw utf-8 bytes (operators who paste
+ * a passphrase get a deterministic key from it). Returns `undefined` when
+ * the env is unset.
+ *
+ * @param raw - the raw env value, if present.
+ * @returns the decoded key, or undefined.
+ */
+/**
+ * Strips a trailing `/v1` from `LETTA_BASE_URL` and writes the cleaned
+ * value back to `process.env` so subprocess spawns inherit the normalized
+ * form. Idempotent — bare-host inputs pass through unchanged. Returns
+ * the normalized value (or `undefined` when the env var is unset).
+ *
+ * @param raw - the raw `process.env.LETTA_BASE_URL`, possibly undefined.
+ * @returns the normalized base URL, or undefined when unset.
+ */
+function normalizeLettaBaseUrl(raw: string | undefined): string | undefined {
+  if (!raw) return undefined;
+  const stripped = raw.replace(/\/v1\/?$/i, "");
+  if (stripped !== raw) {
+    process.env.LETTA_BASE_URL = stripped;
+  }
+  return stripped;
+}
+
+function parseSigningKey(raw: string | undefined): Buffer | undefined {
+  if (!raw) return undefined;
+  const trimmed = raw.trim();
+  if (/^[0-9a-fA-F]+$/.test(trimmed) && trimmed.length % 2 === 0) {
+    return Buffer.from(trimmed, "hex");
+  }
+  // Try base64 / base64url.
+  const normalized = trimmed.replace(/-/g, "+").replace(/_/g, "/");
+  const padded = normalized + "===".slice((normalized.length + 3) % 4);
+  try {
+    const buf = Buffer.from(padded, "base64");
+    // Reject obviously broken decodes (base64 length != original).
+    if (buf.length > 0 && buf.toString("base64").replace(/=+$/, "") === normalized.replace(/=+$/, "")) {
+      return buf;
+    }
+  } catch {
+    /* fall through */
+  }
+  // Fallback: treat as raw utf-8 bytes.
+  return Buffer.from(trimmed, "utf8");
+}
+
+/* ────────────────────────────────────────────────────────────────────────── */
+/*  bootServer                                                                */
+/* ────────────────────────────────────────────────────────────────────────── */
+
+/** Returned by `bootServer` — everything the dev/Next.js mount needs. */
+export interface BootedServer {
+  /** The Fetch handler — `(Request) => Promise<Response>`. */
+  fetchHandler: FetchHandler;
+  /** Drizzle client (in case the embedder wants to read directly). */
+  db: DrizzleDB;
+  /** Raw better-sqlite3 handle. */
+  sqlite: Database.Database;
+  /** Logger configured per env. */
+  logger: Logger;
+  /** Parsed env config. */
+  config: EnvConfig;
+  /** What the seed actually wrote on this boot. */
+  seedReport: SeedReport;
+  /** Closes the DB handle. Call this on graceful shutdown. */
+  close: () => void;
+}
+
+/** Optional override: pass a pre-built logger or destination to capture. */
+export interface BootServerOptions {
+  /**
+   * Custom env source. Defaults to `process.env`. Tests pass a synthesized
+   * record so they don't have to mutate the actual environment.
+   */
+  env?: EnvLike;
+  /** Override the logger (e.g. test capture). */
+  logger?: Logger;
+  /** Override the connection — tests use `:memory:` here. */
+  databaseUrl?: string;
+  /**
+   * Override the Letta client (§6.4). Tests pass a mock so the server
+   * never reaches a real Letta. Production omits this — the default reads
+   * `admin_setting.letta_base_url` + decrypted API key on each use.
+   */
+  lettaClient?: LettaClient;
+}
+
+/**
+ * One-shot boot: parses env, opens the DB, runs migrations, runs the
+ * first-boot seed (idempotent), and returns the Fetch handler.
+ *
+ * Called by `dev/server.ts` and by per-customer wrappers' Next.js mount
+ * (typically wrapped in a module-level singleton so it runs once per
+ * Next.js process).
+ *
+ * @param opts - optional overrides for testing.
+ * @returns the booted server bundle.
+ */
+export async function bootServer(
+  opts: BootServerOptions = {},
+): Promise<BootedServer> {
+  const env = opts.env ?? (process.env as EnvLike);
+  const config = parseEnvConfig(env);
+  const logger = opts.logger ?? createLogger({ level: config.logLevel });
+
+  const databaseUrl = opts.databaseUrl ?? config.databaseUrl;
+  const handle: DbHandle = openDb(databaseUrl);
+
+  runMigrations(handle);
+  const seedReport = await runFirstBootSeed(
+    handle,
+    {
+      encryptionKey: config.encryptionKey,
+      signingKeyFromEnv: config.signingKeyFromEnv,
+      metricsScrapeTokenFromEnv: config.metricsScrapeTokenFromEnv,
+      lettaApiKey: config.lettaApiKey,
+      lettaBaseUrl: config.lettaBaseUrl,
+      lettaAdeUrl: config.lettaAdeUrl,
+    },
+    logger,
+  );
+
+  const fetchHandler = createFetchHandler({
+    db: handle.db,
+    sqlite: handle.raw,
+    logger,
+    encryptionKey: config.encryptionKey,
+    // Pass the booted env through as the runtime fallback source. Tests
+    // synthesize an env record; production uses process.env (the default
+    // inside createFetchHandler). Threading it explicitly here means
+    // bootTestServer's `env:` parameter actually reaches resolveApiKey
+    // in letta/client.ts.
+    envSource: env,
+    ...(opts.lettaClient !== undefined ? { lettaClient: opts.lettaClient } : {}),
+  });
+
+  return {
+    fetchHandler,
+    db: handle.db,
+    sqlite: handle.raw,
+    logger,
+    config,
+    seedReport,
+    close: () => handle.raw.close(),
+  };
+}

package/src/db/connection.ts

@@ -0,0 +1,75 @@
+/**
+ * better-sqlite3 connection + Drizzle wiring.
+ *
+ * On open we set the two PRAGMAs the spec requires (§6.8): `journal_mode=WAL`
+ * for concurrency under multiple readers + one writer, and `foreign_keys=ON`
+ * because SQLite ships them off by default.
+ *
+ * The connection is intentionally a per-server-instance singleton — Drizzle's
+ * client wraps better-sqlite3, and better-sqlite3 itself is fully synchronous
+ * (no connection pool needed; one handle is enough). Tests open their own
+ * handle against a `:memory:` or tmp-file path.
+ *
+ * @see SPECIFICATIONS.md §6.8 — persistence
+ * @see SPECIFICATIONS.md §11 — data model
+ */
+
+import { mkdirSync } from "node:fs";
+import { dirname, isAbsolute, resolve as pathResolve } from "node:path";
+
+import Database from "better-sqlite3";
+import { drizzle } from "drizzle-orm/better-sqlite3";
+
+import * as schema from "./schema.js";
+
+/** The Drizzle client type our handlers consume. */
+export type DrizzleDB = ReturnType<typeof drizzle<typeof schema>>;
+
+/** Bundles the raw better-sqlite3 handle alongside the Drizzle wrapper. */
+export interface DbHandle {
+  /** Drizzle client — typed against the schema. */
+  db: DrizzleDB;
+  /** Raw better-sqlite3 handle (used for PRAGMAs and `.close()`). */
+  raw: Database.Database;
+}
+
+/**
+ * Opens the SQLite database at `path` and applies the spec's PRAGMAs.
+ *
+ * `:memory:` is recognized verbatim and skipped from path-resolution +
+ * directory-creation logic (used by tests).
+ *
+ * @param path - filesystem path, a `file:`-prefixed URI, or `:memory:`.
+ * @returns Drizzle wrapper + raw better-sqlite3 handle.
+ */
+export function openDb(path: string): DbHandle {
+  const resolvedPath = resolveDbPath(path);
+  if (resolvedPath !== ":memory:") {
+    mkdirSync(dirname(resolvedPath), { recursive: true });
+  }
+  const raw = new Database(resolvedPath);
+  raw.pragma("journal_mode = WAL");
+  raw.pragma("foreign_keys = ON");
+  const db = drizzle(raw, { schema });
+  return { db, raw };
+}
+
+/**
+ * Resolves a path that may be `:memory:`, a `file:` URI from `DATABASE_URL`,
+ * or an ordinary filesystem path (relative or absolute).
+ *
+ * @param path - raw path string from env or caller.
+ * @returns absolute filesystem path or `:memory:`.
+ */
+function resolveDbPath(path: string): string {
+  if (path === ":memory:") return path;
+  let stripped = path;
+  if (stripped.startsWith("file:")) {
+    stripped = stripped.slice("file:".length);
+    if (stripped.startsWith("//")) stripped = stripped.slice(2);
+  }
+  if (stripped === ":memory:") return stripped;
+  return isAbsolute(stripped) ? stripped : pathResolve(stripped);
+}
+
+export { schema };

package/src/db/migrate.ts

@@ -0,0 +1,212 @@
+/**
+ * Boot-time migrations runner.
+ *
+ * Currently a single `0001_init` migration that builds the entire §11 schema
+ * idempotently via `CREATE TABLE IF NOT EXISTS`. Tracked in
+ * `_ai_hub_migrations` so future migrations can append additional steps and
+ * skip already-applied ones.
+ *
+ * **Why hand-rolled SQL instead of `drizzle-kit migrate`**: shipping
+ * `drizzle-kit` and a generated `drizzle/` SQL folder as runtime artifacts
+ * adds a build step that the per-customer wrapper doesn't want to think about.
+ * The schema is small and stable; CREATE-TABLE-IF-NOT-EXISTS is sufficient,
+ * and Drizzle's typed query builder already protects us from drift between
+ * the runtime SQL and `schema.ts`. (If migrations ever get complex enough
+ * that this breaks down, we move to drizzle-kit then — not pre-emptively.)
+ *
+ * @see SPECIFICATIONS.md §11 — data model the schema mirrors 1:1
+ */
+
+import type { DbHandle } from "./connection.js";
+
+/**
+ * Runs every not-yet-applied migration on `handle.raw`.
+ *
+ * Idempotent: running this on a fresh SQLite produces the v1 schema; running
+ * it on an already-migrated DB is a no-op. The migrations are wrapped in a
+ * single transaction so a partial failure can't leave the schema in a
+ * half-applied state.
+ *
+ * @param handle - the db handle returned from `openDb`.
+ */
+export function runMigrations(handle: DbHandle): void {
+  const { raw } = handle;
+
+  raw.exec(`
+    CREATE TABLE IF NOT EXISTS _ai_hub_migrations (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      name TEXT NOT NULL UNIQUE,
+      applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    )
+  `);
+
+  for (const m of MIGRATIONS) {
+    const already = raw
+      .prepare("SELECT 1 FROM _ai_hub_migrations WHERE name = ?")
+      .get(m.name) as { 1: number } | undefined;
+    if (already) continue;
+    const apply = raw.transaction(() => {
+      raw.exec(m.sql);
+      raw
+        .prepare("INSERT INTO _ai_hub_migrations (name) VALUES (?)")
+        .run(m.name);
+    });
+    apply();
+  }
+}
+
+/* ────────────────────────────────────────────────────────────────────────── */
+/*  Migrations table                                                          */
+/* ────────────────────────────────────────────────────────────────────────── */
+
+interface Migration {
+  /** Stable name used as the dedupe key — `NNNN_<slug>`. */
+  name: string;
+  /** Raw SQL applied verbatim inside a transaction. */
+  sql: string;
+}
+
+/**
+ * The canonical migration list. Append-only — never edit a past entry, only
+ * add a new one with a higher number. (Same convention as Drizzle / Prisma /
+ * Rails.)
+ */
+const MIGRATIONS: ReadonlyArray<Migration> = [
+  {
+    name: "0001_init",
+    sql: `
+      CREATE TABLE user (
+        id TEXT NOT NULL PRIMARY KEY,
+        username TEXT NOT NULL UNIQUE,
+        display_name TEXT,
+        password_hash TEXT,
+        identity_source TEXT NOT NULL DEFAULT 'local',
+        is_initial_password INTEGER NOT NULL DEFAULT 0,
+        role TEXT NOT NULL DEFAULT 'user',
+        theme_preference TEXT,
+        password_changed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+      CREATE INDEX user_role_idx ON user (role);
+
+      CREATE TABLE agent (
+        id TEXT NOT NULL PRIMARY KEY,
+        display_name TEXT NOT NULL,
+        letta_agent_id TEXT NOT NULL,
+        tpm_recovery_enabled INTEGER NOT NULL DEFAULT 0,
+        max_recovery_attempts INTEGER NOT NULL DEFAULT 2,
+        default_retry_after_seconds INTEGER NOT NULL DEFAULT 65,
+        stream_idle_timeout_seconds INTEGER NOT NULL DEFAULT 60,
+        sandbox_seed_name TEXT,
+        fork_source_conversation_id TEXT,
+        is_archived INTEGER NOT NULL DEFAULT 0,
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+
+      CREATE TABLE session (
+        id TEXT NOT NULL PRIMARY KEY,
+        user_id TEXT NOT NULL REFERENCES user(id) ON DELETE CASCADE,
+        agent_id TEXT NOT NULL REFERENCES agent(id) ON DELETE RESTRICT,
+        letta_conversation_id TEXT NOT NULL,
+        letta_agent_id TEXT NOT NULL,
+        title TEXT NOT NULL,
+        is_archived INTEGER NOT NULL DEFAULT 0,
+        reasoning_effort TEXT,
+        verbosity TEXT,
+        thinking_budget INTEGER,
+        sandbox_dir TEXT,
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+      CREATE INDEX session_user_agent_idx ON session (user_id, agent_id);
+
+      CREATE TABLE turn (
+        id TEXT NOT NULL PRIMARY KEY,
+        session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE,
+        status TEXT NOT NULL,
+        phase TEXT,
+        last_seq INTEGER NOT NULL DEFAULT 0,
+        idempotency_key TEXT,
+        started_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        finished_at TEXT
+      );
+
+      CREATE TABLE turn_event (
+        turn_id TEXT NOT NULL REFERENCES turn(id) ON DELETE CASCADE,
+        seq INTEGER NOT NULL,
+        event_name TEXT NOT NULL,
+        data TEXT NOT NULL,
+        state TEXT NOT NULL,
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        PRIMARY KEY (turn_id, seq)
+      );
+
+      CREATE TABLE admin_setting (
+        id INTEGER NOT NULL PRIMARY KEY,
+        letta_api_key_encrypted BLOB,
+        letta_base_url TEXT,
+        letta_ade_url TEXT,
+        feature_toggles TEXT NOT NULL DEFAULT '{}',
+        theme_switcher_enabled INTEGER NOT NULL DEFAULT 1,
+        theme_default TEXT NOT NULL DEFAULT 'light',
+        service_token_signing_key BLOB,
+        metrics_scrape_token TEXT,
+        metrics_scrape_token_rotated_at TEXT,
+        signing_key_rotated_at TEXT,
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+
+      CREATE TABLE message_override (
+        conversation_id TEXT NOT NULL,
+        ordinal_position INTEGER NOT NULL,
+        system_override TEXT,
+        user_override TEXT,
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        PRIMARY KEY (conversation_id, ordinal_position),
+        CHECK (system_override IS NOT NULL OR user_override IS NOT NULL)
+      );
+
+      CREATE TABLE message_usage (
+        letta_message_id TEXT NOT NULL PRIMARY KEY,
+        session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE,
+        prompt_tokens INTEGER,
+        completion_tokens INTEGER,
+        total_tokens INTEGER,
+        cached_input_tokens INTEGER,
+        cache_write_tokens INTEGER,
+        reasoning_tokens INTEGER,
+        context_tokens INTEGER,
+        run_id TEXT,
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+      CREATE INDEX message_usage_session_idx ON message_usage(session_id);
+    `,
+  },
+  {
+    name: "0002_rotating_message",
+    sql: `
+      CREATE TABLE rotating_message (
+        id TEXT NOT NULL PRIMARY KEY,
+        display_name TEXT NOT NULL,
+        prompt TEXT NOT NULL,
+        letta_agent_id TEXT NOT NULL,
+        schedule_kind TEXT NOT NULL,
+        interval_seconds INTEGER,
+        daily_time TEXT,
+        cron_expression TEXT,
+        current_conversation_id TEXT,
+        last_status TEXT NOT NULL DEFAULT 'pending',
+        last_error TEXT,
+        last_rotated_at TEXT,
+        next_rotation_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        is_archived INTEGER NOT NULL DEFAULT 0,
+        created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+      CREATE INDEX rotating_message_due_idx ON rotating_message (is_archived, next_rotation_at);
+    `,
+  },
+];