@data-club/ai-hub-server 0.0.1

package/src/letta/subprocessThrottle.ts

@@ -0,0 +1,251 @@
+/**
+ * Dynamic RAM-based concurrent-spawn throttle for Letta Code subprocesses.
+ *
+ * Every time we're about to spawn a Letta Code CLI subprocess (which happens
+ * once per chat turn — see `letta/client.ts` → `streamUserMessage`), we call
+ * `acquire()` first. If the host's *available* RAM has dropped below a
+ * configured threshold OR we've already got the configured hard-cap of live
+ * subprocesses, `acquire()` throws `ServerOverloadedError`. The chat route
+ * catches that and returns HTTP 503 with the `server_overloaded` SSE error
+ * event so the React client can surface a "server is busy" banner.
+ *
+ * **Why MemAvailable, not MemFree.** Linux uses unused RAM as page cache;
+ * `os.freemem()` / `/proc/meminfo`'s `MemFree` line reports memory that's
+ * *literally* unallocated, which on any healthy server is misleadingly low
+ * (often ~5% even when ~50% is actually allocatable without swapping).
+ * `/proc/meminfo`'s `MemAvailable` line is the kernel's own estimate of
+ * "memory that could be allocated without entering swap" — that's what we
+ * compare against the threshold.
+ *
+ * **Why two gates and not one.** The RAM gate is the primary throttle and
+ * adapts to whatever EC2 instance the deployment runs on. The hard-cap is a
+ * circuit breaker: if our `/proc/meminfo` math is ever wrong (kernel reports
+ * stale numbers under load, container cgroup limits trip earlier than host
+ * MemAvailable suggests, …), an unbounded subprocess pool would still be
+ * able to OOM-kill the box. The hard-cap is the floor on how bad it can get.
+ *
+ * @see ../../docs/operations/env-vars.md
+ */
+
+import { readFileSync } from "node:fs";
+import { totalmem } from "node:os";
+
+/* ────────────────────────────────────────────────────────────────────────── */
+/*  Config                                                                    */
+/* ────────────────────────────────────────────────────────────────────────── */
+
+/** Default minimum percentage of total RAM that must be available before we'll
+ *  spawn a new subprocess. Overridable via env. */
+export const DEFAULT_RAM_THRESHOLD_PCT = 20;
+/** Default hard cap on concurrent live subprocesses. Overridable via env. */
+export const DEFAULT_HARD_CAP = 100;
+
+/**
+ * Resolved throttle config. Pulled from env at construction time; tests pass
+ * an explicit one to bypass env reading.
+ */
+export interface ThrottleConfig {
+  /** Minimum % of total RAM that must remain available to accept a spawn. */
+  ramThresholdPct: number;
+  /** Maximum concurrent live subprocesses, regardless of RAM. */
+  hardCap: number;
+}
+
+/**
+ * Reads throttle config from the process environment, with defaults.
+ *
+ * Env vars:
+ * - `AI_HUB_SUBPROCESS_RAM_THRESHOLD_PCT` — integer 0..100. Defaults to 20.
+ *   A value <= 0 disables the RAM gate (every acquire passes the RAM check).
+ *   A value >= 100 means every acquire fails the RAM check (useful for
+ *   testing the overload path).
+ * - `AI_HUB_SUBPROCESS_HARD_CAP` — positive integer. Defaults to 100.
+ *
+ * @returns the parsed config bundle.
+ */
+export function readThrottleConfigFromEnv(): ThrottleConfig {
+  const rawPct = process.env.AI_HUB_SUBPROCESS_RAM_THRESHOLD_PCT;
+  const rawCap = process.env.AI_HUB_SUBPROCESS_HARD_CAP;
+  return {
+    ramThresholdPct: parseIntInRange(rawPct, 0, 100, DEFAULT_RAM_THRESHOLD_PCT),
+    hardCap: parseIntInRange(rawCap, 1, 10_000, DEFAULT_HARD_CAP),
+  };
+}
+
+function parseIntInRange(raw: string | undefined, min: number, max: number, fallback: number): number {
+  if (raw === undefined || raw === "") return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isFinite(parsed) || parsed < min || parsed > max) return fallback;
+  return parsed;
+}
+
+/* ────────────────────────────────────────────────────────────────────────── */
+/*  Memory reader                                                             */
+/* ────────────────────────────────────────────────────────────────────────── */
+
+/**
+ * Reader for the host's MemAvailable + MemTotal. Returns kilobytes (matches
+ * `/proc/meminfo` units). Tests inject a fake reader so they don't depend on
+ * the host's actual memory state.
+ */
+export interface MemReader {
+  /** @returns `{availableKb, totalKb}` snapshot at the moment of call. */
+  read(): { availableKb: number; totalKb: number };
+}
+
+/**
+ * Default reader — parses `/proc/meminfo` directly. Falls back to
+ * `os.totalmem()` for `totalKb` when /proc is unreadable (non-Linux), and
+ * returns `MemTotal` as `availableKb` in that case so the RAM gate is
+ * effectively disabled there (the hard-cap still applies).
+ *
+ * This file lives at a stable kernel-exposed path on every Linux host
+ * (including in Docker without any special mounts) so no privileges are
+ * needed.
+ *
+ * @see https://www.kernel.org/doc/Documentation/filesystems/proc.txt — the
+ *   meminfo section documents MemAvailable's semantics ("estimate of how
+ *   much memory is available for starting new applications, without
+ *   swapping").
+ */
+export const procMeminfoReader: MemReader = {
+  read() {
+    try {
+      const raw = readFileSync("/proc/meminfo", "utf8");
+      const total = raw.match(/^MemTotal:\s+(\d+)\s+kB/m);
+      const available = raw.match(/^MemAvailable:\s+(\d+)\s+kB/m);
+      if (total && available) {
+        return { totalKb: Number(total[1]), availableKb: Number(available[1]) };
+      }
+    } catch {
+      // Fall through to the os.totalmem fallback.
+    }
+    const totalKb = Math.round(totalmem() / 1024);
+    return { totalKb, availableKb: totalKb };
+  },
+};
+
+/* ────────────────────────────────────────────────────────────────────────── */
+/*  Throttle                                                                  */
+/* ────────────────────────────────────────────────────────────────────────── */
+
+/**
+ * Typed error thrown by `SubprocessThrottle.acquire()` when the host is too
+ * loaded to accept another subprocess. The chat route catches this and
+ * returns HTTP 503 with an `error {server_overloaded}` SSE event.
+ */
+export class ServerOverloadedError extends Error {
+  /** Stable error code for the SSE wire protocol's `error.code` field. */
+  readonly code = "server_overloaded" as const;
+  /** Which gate tripped — useful for logs + the eventual operator playbook. */
+  readonly gate: "ram" | "hard_cap";
+  /** Snapshot of the trigger at the moment we rejected. */
+  readonly detail: {
+    /** MemAvailable as a fraction of MemTotal at the time of rejection. */
+    availablePct: number;
+    /** Live subprocess count at the time of rejection. */
+    liveCount: number;
+    /** Configured RAM threshold (percent). */
+    ramThresholdPct: number;
+    /** Configured hard cap. */
+    hardCap: number;
+  };
+
+  constructor(gate: "ram" | "hard_cap", detail: ServerOverloadedError["detail"]) {
+    super(
+      gate === "ram"
+        ? `Server overloaded: only ${detail.availablePct.toFixed(1)}% RAM available (< ${detail.ramThresholdPct}% required).`
+        : `Server overloaded: ${detail.liveCount} live subprocesses (>= ${detail.hardCap} hard cap).`,
+    );
+    this.name = "ServerOverloadedError";
+    this.gate = gate;
+    this.detail = detail;
+  }
+}
+
+/**
+ * A leased slot in the throttle's live-subprocess counter. Returned by
+ * `acquire()`; callers must call `release()` exactly once (use try/finally).
+ */
+export interface ThrottleLease {
+  /** Releases the slot back to the throttle. Idempotent — calling twice
+   *  is a no-op. */
+  release(): void;
+}
+
+/**
+ * The throttle.
+ *
+ * Construct one per ai-hub-server process (it's stateful — the live-count
+ * counter is in-memory). Production wires this into `AIHubServerContext` via
+ * `createFetchHandler`; tests pass a fake `MemReader` to drive the RAM gate
+ * deterministically.
+ */
+export class SubprocessThrottle {
+  readonly config: ThrottleConfig;
+  private readonly memReader: MemReader;
+  private liveCount = 0;
+
+  /**
+   * @param config - resolved throttle config (use `readThrottleConfigFromEnv`).
+   * @param memReader - mem reader (defaults to /proc/meminfo).
+   */
+  constructor(config: ThrottleConfig, memReader: MemReader = procMeminfoReader) {
+    this.config = config;
+    this.memReader = memReader;
+  }
+
+  /**
+   * Attempts to acquire a slot. Throws `ServerOverloadedError` synchronously
+   * if either gate trips; otherwise increments the live counter and returns
+   * a lease the caller must release when the subprocess exits.
+   *
+   * The acquire-then-spawn dance has a small race: between the RAM check
+   * here and the actual spawn, another concurrent acquire could land. That's
+   * acceptable for v1 — the next acquire will see the updated MemAvailable
+   * and the hard cap is the absolute backstop.
+   *
+   * @returns a lease the caller must `.release()` in finally.
+   * @throws `ServerOverloadedError` if either gate trips.
+   */
+  acquire(): ThrottleLease {
+    // Hard cap check first — it's cheap and the more deterministic gate.
+    if (this.liveCount >= this.config.hardCap) {
+      throw new ServerOverloadedError("hard_cap", {
+        availablePct: 100, // unknown but irrelevant; the cap gate fired
+        liveCount: this.liveCount,
+        ramThresholdPct: this.config.ramThresholdPct,
+        hardCap: this.config.hardCap,
+      });
+    }
+
+    // RAM gate. ramThresholdPct=0 disables this gate entirely.
+    if (this.config.ramThresholdPct > 0) {
+      const { availableKb, totalKb } = this.memReader.read();
+      const availablePct = totalKb > 0 ? (availableKb / totalKb) * 100 : 100;
+      if (availablePct < this.config.ramThresholdPct) {
+        throw new ServerOverloadedError("ram", {
+          availablePct,
+          liveCount: this.liveCount,
+          ramThresholdPct: this.config.ramThresholdPct,
+          hardCap: this.config.hardCap,
+        });
+      }
+    }
+
+    this.liveCount++;
+    let released = false;
+    return {
+      release: () => {
+        if (released) return;
+        released = true;
+        this.liveCount--;
+      },
+    };
+  }
+
+  /** @returns the current live-subprocess count (for observability + tests). */
+  get currentLiveCount(): number {
+    return this.liveCount;
+  }
+}

package/src/letta/types.ts

@@ -0,0 +1,65 @@
+/**
+ * Project-shaped Letta types.
+ *
+ * Re-exports only the SDK surfaces ai-hub uses, so the rest of the server
+ * code never has to traverse `@letta-ai/letta-client`'s deep namespace
+ * structure. **No raw HTTP, ever** — these are aliases over the SDK's own
+ * types, not parallel inventions (§6.4).
+ *
+ * If the SDK rev removes or renames one of these, the typechecker flags it
+ * at upgrade time and `letta/client.ts` is the single place to adapt.
+ *
+ * @see SPECIFICATIONS.md §6.4 — SDK-only access pattern
+ * @see SPECIFICATIONS.md §6.4.1 — first-class surfaces we depend on
+ */
+
+import type Letta from "@letta-ai/letta-client";
+import type {
+  AssistantMessage,
+  HiddenReasoningMessage,
+  Message as LettaSdkMessage,
+  ReasoningMessage,
+  ToolCallMessage,
+  UserMessage,
+} from "@letta-ai/letta-client/resources/agents/messages.js";
+import type { LettaStreamingResponse } from "@letta-ai/letta-client/resources/agents/messages.js";
+import type { ToolReturnMessage } from "@letta-ai/letta-client/resources/tools.js";
+
+/**
+ * The underlying SDK client. We `import type` everywhere except in
+ * `letta/client.ts` (where the constructor actually runs) so the rest of the
+ * codebase stays SDK-version-agnostic in shape.
+ */
+export type LettaSdk = Letta;
+
+/**
+ * One streaming event from `client.conversations.messages.create({ streaming: true })`.
+ *
+ * The union spans every variant Letta emits in an SSE stream — assistant
+ * text, reasoning, tool calls, tool returns, errors, pings, stop reasons,
+ * usage stats. `letta/eventTranslator.ts` is the pure function from this
+ * union to ai-hub's wire-protocol events (§9).
+ *
+ * @see SPECIFICATIONS.md §9 — wire protocol
+ */
+export type LettaStreamEvent = LettaStreamingResponse;
+
+/**
+ * Alias for `AssistantMessage` — used by the translator + tests. See
+ * `letta/eventTranslator.ts` for the streamed-delta vs. cumulative-content
+ * note.
+ */
+export type LettaAssistantMessage = AssistantMessage;
+/** Alias for `ReasoningMessage` — see `eventTranslator.ts`. */
+export type LettaReasoningMessage = ReasoningMessage;
+/** Alias for `HiddenReasoningMessage` — see `eventTranslator.ts`. */
+export type LettaHiddenReasoningMessage = HiddenReasoningMessage;
+/** Alias for `ToolCallMessage` — see `eventTranslator.ts`. */
+export type LettaToolCallMessage = ToolCallMessage;
+/** Alias for `ToolReturnMessage` — see `eventTranslator.ts`. */
+export type LettaToolReturnMessage = ToolReturnMessage;
+/** Alias for `UserMessage` — only appears in history reads, not in streams. */
+export type LettaUserMessage = UserMessage;
+
+/** Full assembled Letta message (used by `messages.list`). */
+export type LettaMessage = LettaSdkMessage;

package/src/observability/logger.ts

@@ -0,0 +1,103 @@
+/**
+ * Structured logging via pino.
+ *
+ * Production: raw JSON to stdout, one line per record (§6.10 — twelve-factor
+ * pattern). Dev: piped through `pino-pretty` for human-readable output, gated
+ * on `NODE_ENV !== "production"` or `LOG_LEVEL=debug`.
+ *
+ * **Redaction is a load-bearing security control** (§13.3). pino's built-in
+ * `redact` option is the single point of truth — every log call goes through
+ * the configured logger, and a regression test asserts a known-secret value
+ * never appears in captured stdout.
+ *
+ * @see SPECIFICATIONS.md §6.10 — observability
+ * @see SPECIFICATIONS.md §13.3 — secrets
+ */
+
+import { pino, type Logger as PinoLogger, type LoggerOptions } from "pino";
+
+/** What the rest of the server consumes — keeps pino out of import surfaces. */
+export type Logger = PinoLogger;
+
+/**
+ * The redaction paths pino watches on every log record.
+ *
+ * The list explicitly covers the keys the spec calls out (`*_key`, `*_token`,
+ * `password`, `authorization`) plus their nested forms inside common places
+ * (`req.headers.authorization`, `body.password`, etc.). Adding a new
+ * sensitive field is a matter of appending a path here — there is no second
+ * redaction layer to keep in sync.
+ */
+const REDACT_PATHS: ReadonlyArray<string> = [
+  "password",
+  "current_password",
+  "new_password",
+  "passwordHash",
+  "password_hash",
+  "authorization",
+  "Authorization",
+  "headers.authorization",
+  "headers.Authorization",
+  "req.headers.authorization",
+  "req.headers.Authorization",
+  "request.headers.authorization",
+  "request.headers.Authorization",
+  "*.authorization",
+  "*.Authorization",
+  "service_token_signing_key",
+  "serviceTokenSigningKey",
+  "letta_api_key",
+  "lettaApiKey",
+  "metrics_scrape_token",
+  "metricsScrapeToken",
+  "encryption_key",
+  "encryptionKey",
+  "token",
+  "*.token",
+  "generated_password",
+  "*.generated_password",
+];
+
+/**
+ * Optional shape passed to `createLogger`. The dev-mode pretty-print stream
+ * is set up by the caller (the dev/server entrypoint) — `createLogger` itself
+ * stays format-agnostic so tests can pass a writable stream.
+ */
+export interface CreateLoggerOptions {
+  /** pino level (`fatal | error | warn | info | debug | trace`). Default `info`. */
+  level?: string;
+  /**
+   * Optional destination. Tests pass a memory stream so they can assert on
+   * the captured output. Production passes nothing — pino writes to stdout
+   * by default.
+   */
+  destination?: NodeJS.WritableStream;
+}
+
+/**
+ * Builds a pino logger with the project's redaction policy applied.
+ *
+ * @param opts - level + optional destination override.
+ * @returns the pino logger instance.
+ */
+export function createLogger(opts: CreateLoggerOptions = {}): Logger {
+  const config: LoggerOptions = {
+    level: opts.level ?? process.env.LOG_LEVEL ?? "info",
+    redact: {
+      paths: [...REDACT_PATHS],
+      censor: "[redacted]",
+      remove: false,
+    },
+  };
+  return opts.destination ? pino(config, opts.destination) : pino(config);
+}
+
+/**
+ * The redaction-path list — exposed for tests so a new path added in
+ * `REDACT_PATHS` is automatically covered without test churn.
+ *
+ * @returns a fresh array of the redaction paths used by `createLogger`.
+ */
+export function getRedactPaths(): string[] {
+  return [...REDACT_PATHS];
+}

package/src/observability/metrics.ts

@@ -0,0 +1,114 @@
+/**
+ * Process-local Prometheus-shaped metrics registry.
+ *
+ * v1 keeps this tiny — three counters + two gauges — to avoid pulling
+ * `prom-client` for what is a five-series surface. The names + types
+ * match §6.10's contract verbatim so an operator's existing dashboards
+ * keep working when a future version swaps in a heavier library.
+ *
+ * Counters monotonically increment (and persist across the process'
+ * lifetime); gauges hold a single most-recent value. Step 5 wires the
+ * chat-side series; Step 6 fills in the TPM-recovery counters.
+ *
+ * @see SPECIFICATIONS.md §6.10 — observability / metrics
+ */
+
+/** A monotonic counter. */
+export interface Counter {
+  /** Adds `n` (default 1) to the counter. */
+  inc(n?: number): void;
+  /** Reads the current value. */
+  value(): number;
+}
+
+/** A single-value gauge. */
+export interface Gauge {
+  /** Replaces the gauge's current value. */
+  set(value: number): void;
+  /** Reads the current value. */
+  value(): number;
+}
+
+/** Internal counter factory. */
+function counter(): Counter {
+  let v = 0;
+  return {
+    inc(n = 1) {
+      v += n;
+    },
+    value() {
+      return v;
+    },
+  };
+}
+
+/** Internal gauge factory. */
+function gauge(): Gauge {
+  let v = 0;
+  return {
+    set(value) {
+      v = value;
+    },
+    value() {
+      return v;
+    },
+  };
+}
+
+/** Total `/chat` requests received. Bumped on every accepted POST. */
+export const chatRequestsTotal: Counter = counter();
+/** TPM recovery attempts started — populated by Step 6. */
+export const tpmRecoveryAttemptsTotal: Counter = counter();
+/** TPM recoveries that exhausted their budget — populated by Step 6. */
+export const tpmExhaustedTotal: Counter = counter();
+/** Most-recent `/chat` turn duration in ms. */
+export const chatDurationMs: Gauge = gauge();
+/** Most-recent Letta SDK call duration in ms. */
+export const lettaRequestDurationMs: Gauge = gauge();
+
+/**
+ * Renders the registry as a Prometheus-text v0.0.4 exposition.
+ *
+ * The route handler in `routes/metrics.ts` calls this after the bearer
+ * token gate passes.
+ *
+ * @returns the full response body.
+ */
+export function renderMetrics(): string {
+  return (
+    [
+      "# HELP chat_requests_total Total /chat requests received.",
+      "# TYPE chat_requests_total counter",
+      `chat_requests_total ${chatRequestsTotal.value()}`,
+      "",
+      "# HELP tpm_recovery_attempts_total TPM recovery attempts started.",
+      "# TYPE tpm_recovery_attempts_total counter",
+      `tpm_recovery_attempts_total ${tpmRecoveryAttemptsTotal.value()}`,
+      "",
+      "# HELP tpm_exhausted_total TPM recoveries that exhausted their budget.",
+      "# TYPE tpm_exhausted_total counter",
+      `tpm_exhausted_total ${tpmExhaustedTotal.value()}`,
+      "",
+      "# HELP chat_duration_ms Most-recent /chat turn duration in ms.",
+      "# TYPE chat_duration_ms gauge",
+      `chat_duration_ms ${chatDurationMs.value()}`,
+      "",
+      "# HELP letta_request_duration_ms Most-recent Letta SDK call duration in ms.",
+      "# TYPE letta_request_duration_ms gauge",
+      `letta_request_duration_ms ${lettaRequestDurationMs.value()}`,
+      "",
+    ].join("\n")
+  );
+}
+
+/**
+ * Resets every series. **Test-only** — production never calls this.
+ * Without it, vitest's module-cache means counters carry across tests.
+ */
+export function _resetMetricsForTests(): void {
+  chatRequestsTotal.inc(-chatRequestsTotal.value());
+  tpmRecoveryAttemptsTotal.inc(-tpmRecoveryAttemptsTotal.value());
+  tpmExhaustedTotal.inc(-tpmExhaustedTotal.value());
+  chatDurationMs.set(0);
+  lettaRequestDurationMs.set(0);
+}