cartethyia 2.5.9 → 2.6.0

package/src/config/resources.ts

@@ -0,0 +1,137 @@
+/**
+ * Canonical resource policy: how much memory this process may use, and how
+ * much work it may hold at once.
+ *
+ * These two numbers were previously decided in three unrelated places — a
+ * hardcoded cap in `concurrency-guard.ts`, a host-wide `os.totalmem()` in the
+ * headroom guard, and a fallback constant in `runtime-metrics.ts` — which
+ * meant a container limited to 512 MiB admitted the same concurrency as a 4 GiB
+ * one and only started shedding once the *host* was nearly full. On a 1-core /
+ * 2 GiB VPS that is the difference between shedding early and being
+ * OOM-killed, and the OOM kills the dashboard too — the one surface that would
+ * have explained what happened.
+ *
+ * Everything here is derived from one limit and one body ceiling, so the
+ * numbers cannot disagree with each other:
+ *
+ *   concurrency = floor(limit × BUDGET_FRACTION / (TRANSIENT_COPIES × body))
+ *
+ * The transient-copies factor is why this is not simply "limit / body": a
+ * request body is held more than once between the socket read, the JSON parse,
+ * and the upstream re-serialization, so budgeting one copy per request
+ * understates the real hold by a factor of about two.
+ */
+import { CONFIG_SPEC, readOptionalInt } from "./spec";
+import { resolveMaxBodyBytes } from "./runtime";
+
+/** Used when no limit is discoverable at all (native dev, macOS, Windows). */
+export const FALLBACK_MEMORY_LIMIT_BYTES = 512 * 1024 * 1024;
+
+/**
+ * Ceiling on concurrent in-flight requests per process.
+ *
+ * A hard ceiling even when memory would allow more: this process is one core,
+ * and past this point queueing delay, not memory, is what degrades. It also
+ * bounds every per-request structure that is sized by concurrency.
+ */
+export const MAX_PROCESS_CONCURRENCY = 64;
+
+/**
+ * How many copies of a request body are resident at once.
+ *
+ * Measured, not assumed: the socket read buffer, the parsed object graph, and
+ * the upstream re-serialization each hold a body-sized allocation, and garbage
+ * collection lags rather than keeping them to one. Budgeting one copy halves
+ * the real peak and lets the process admit twice the concurrency it can hold.
+ */
+export const REQUEST_BODY_TRANSIENT_COPIES = 2;
+
+/** Share of the memory limit all in-flight request bodies together may hold. */
+export const REQUEST_BODY_BUDGET_FRACTION = 0.25;
+
+/** Share of the memory limit kept free for allocation spikes and GC slack. */
+export const MEMORY_HEADROOM_FRACTION = 0.2;
+
+/**
+ * Concurrency this process should admit, derived from the memory limit and the
+ * body ceiling rather than guessed.
+ *
+ * Worked examples, which are the point of having one formula:
+ * - 2 GiB limit, 4 MiB bodies → 64 (the ceiling; memory is not the binder)
+ * - 512 MiB limit, 4 MiB bodies → 16
+ * - 256 MiB limit, 16 MiB bodies → 2
+ *
+ * An explicit override is honoured, clamped to 1..MAX_PROCESS_CONCURRENCY. It
+ * is the operator's call and is never silently raised back to the derived
+ * value; exceeding the derived figure is a deliberate choice, so it is
+ * reported at the call site rather than corrected here.
+ */
+export function deriveProcessConcurrency(options: {
+  readonly memoryLimitBytes: number;
+  readonly maxBodyBytes: number;
+  readonly override?: number | undefined;
+}): number {
+  if (options.override !== undefined) {
+    if (!Number.isInteger(options.override)) return 1;
+    return Math.min(MAX_PROCESS_CONCURRENCY, Math.max(1, options.override));
+  }
+  const body = Math.max(1, options.maxBodyBytes);
+  const perRequest = REQUEST_BODY_TRANSIENT_COPIES * body;
+  const budget = Math.max(1, Math.floor(options.memoryLimitBytes * REQUEST_BODY_BUDGET_FRACTION));
+  const derived = Math.floor(budget / perRequest);
+  return Math.min(MAX_PROCESS_CONCURRENCY, Math.max(1, derived));
+}
+
+/**
+ * Bytes kept free below the memory limit.
+ *
+ * A guard that only reacts once RSS equals the limit has already lost: the
+ * allocation that breaches it happens between two samples, and the spike that
+ * matters — an upstream response landing all at once — arrives in far less
+ * than one sample interval.
+ */
+export function deriveMemoryHeadroomBytes(memoryLimitBytes: number): number {
+  return Math.ceil(memoryLimitBytes * MEMORY_HEADROOM_FRACTION);
+}
+
+/**
+ * The optional operator override, read through the config spec so "unset" and
+ * "out of range" are treated the way every other knob treats them.
+ */
+export function resolveProcessConcurrencyOverride(): number | undefined {
+  return readOptionalInt(
+    "CARTETHYIA_PROCESS_CONCURRENCY_LIMIT",
+    CONFIG_SPEC.CARTETHYIA_PROCESS_CONCURRENCY_LIMIT,
+  );
+}
+
+/**
+ * Slack added on top of every key's concurrency limit.
+ *
+ * Operator configuration, not something a key carries about itself: it is
+ * deliberate hidden allowance so a run that lands a few requests over the
+ * number an operator chose is not refused at that number. The effective
+ * ceiling is `limit + tolerance`, and only requests past it are refused.
+ */
+export function resolveConcurrencyTolerance(): number {
+  const value = readOptionalInt(
+    "CARTETHYIA_CONCURRENCY_TOLERANCE",
+    CONFIG_SPEC.CARTETHYIA_CONCURRENCY_TOLERANCE,
+  );
+  return value === undefined ? 0 : Math.max(0, value);
+}
+
+/**
+ * The effective concurrency for this process.
+ *
+ * Takes the limit as an argument rather than resolving it here: the limit
+ * owner is `resolveMemoryLimitBytes()` in `observability/runtime-metrics.ts`,
+ * and duplicating that resolution would let the two drift.
+ */
+export function resolveProcessConcurrency(memoryLimitBytes: number): number {
+  return deriveProcessConcurrency({
+    memoryLimitBytes,
+    maxBodyBytes: resolveMaxBodyBytes(),
+    override: resolveProcessConcurrencyOverride(),
+  });
+}

package/src/config/runtime.ts

@@ -1,76 +1,133 @@
-/**
- * HTTP listener, dashboard runtime, and graceful-shutdown drain resolvers.
- */
-
-import { CONFIG_SPEC, readInt, readOptionalText } from "./spec";
-import { resolvePackagePath } from "./package-paths";
-
-/** Resolves the HTTP listener port from the process environment. */
-export function resolvePort(): number {
-  return readInt("PORT", CONFIG_SPEC.PORT);
-}
-
-/** Resolves the dashboard static asset directory. */
-export function resolveDashboardDist(): string {
-  const raw = process.env.DASHBOARD_DIST?.trim();
-  if (!raw) return resolvePackagePath("dist", "dashboard");
-  return raw;
-}
-
-/**
- * Resolves the configured default gateway API key (`CARTETHYIA_API_KEY`), if
- * one is set. Consumers that need the raw secret — first-boot seeding and the
- * console's key recovery from existing installations — read it here instead of
- * probing `process.env` directly.
- */
-export function resolveDefaultGatewayApiKey(): string | undefined {
-  return readOptionalText("CARTETHYIA_API_KEY", CONFIG_SPEC.CARTETHYIA_API_KEY);
-}
-
-/** Resolves the request-body ceiling shared by the Bun listener and ingress middleware. */
-export function resolveMaxBodyBytes(): number {
-  return readInt("CARTETHYIA_SERVER_MAX_BODY_BYTES", CONFIG_SPEC.CARTETHYIA_SERVER_MAX_BODY_BYTES);
-}
-
-/** Resolves the listener idle-socket timeout (seconds). */
-export function resolveIdleTimeout(): number {
-  return readInt("CARTETHYIA_SERVER_IDLE_TIMEOUT", CONFIG_SPEC.CARTETHYIA_SERVER_IDLE_TIMEOUT);
-}
-
-/** Resolves the graceful-shutdown drain window: time for in-flight requests to finish before abort. */
-export function resolveShutdownDrainWindowMs(): number {
-  return readInt(
-    "CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS",
-    CONFIG_SPEC.CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS,
-  );
-}
-
-/** Resolves the operator drain token, or `undefined` when the drain route is disabled. */
-export function resolveDrainToken(): string | undefined {
-  return readOptionalText("CARTETHYIA_DRAIN_TOKEN", CONFIG_SPEC.CARTETHYIA_DRAIN_TOKEN);
-}
-
-/**
- * Whether Elysia precompiles route handlers at startup (AOT).
- *
- * Precompilation takes the eager validator branch, which calls TypeBox's
- * `SchemaCompile(schema)` and then reads `.buildResult.external` off the
- * result. In a source-run process that field is absent — the lazy
- * `require("typebox/schema")` resolves to a namespace that does not carry it —
- * and the process dies at the first schema-bearing route with
- * `undefined is not an object (evaluating 'this.tb.buildResult.external')`.
- * `scripts/build/aot.ts` avoids this by rewriting the call site and bundling
- * the namespaces statically, so only the built artifact can actually take the
- * eager path.
- *
- * So: precompilation is on for production *builds*, and off for a source run
- * even when `NODE_ENV=production`. A deployment that runs
- * `bun run src/main.ts` must not be pushed onto the eager branch.
- *
- * `CARTETHYIA_SOURCE_RUN` marks a source run explicitly. It is set by
- * `scripts/deploy/*.sh`, which are the only supported way to run from source.
- */
-export function resolveElysiaPrecompile(): boolean {
-  if (process.env.CARTETHYIA_SOURCE_RUN === "1") return false;
-  return process.env.NODE_ENV === "production";
-}
+/**
+ * HTTP listener, dashboard runtime, and graceful-shutdown drain resolvers.
+ */
+
+import { CONFIG_SPEC, readInt, readOptionalText } from "./spec";
+import { resolvePackagePath } from "./package-paths";
+
+/** Resolves the HTTP listener port from the process environment. */
+export function resolvePort(): number {
+  return readInt("PORT", CONFIG_SPEC.PORT);
+}
+
+/** Resolves the dashboard static asset directory. */
+export function resolveDashboardDist(): string {
+  const raw = process.env.DASHBOARD_DIST?.trim();
+  if (!raw) return resolvePackagePath("dist", "dashboard");
+  return raw;
+}
+
+/**
+ * Resolves the configured default gateway API key (`CARTETHYIA_API_KEY`), if
+ * one is set. Consumers that need the raw secret — first-boot seeding and the
+ * console's key recovery from existing installations — read it here instead of
+ * probing `process.env` directly.
+ */
+export function resolveDefaultGatewayApiKey(): string | undefined {
+  return readOptionalText("CARTETHYIA_API_KEY", CONFIG_SPEC.CARTETHYIA_API_KEY);
+}
+
+/** Resolves the request-body ceiling shared by the Bun listener and ingress middleware. */
+export function resolveMaxBodyBytes(): number {
+  return readInt("CARTETHYIA_SERVER_MAX_BODY_BYTES", CONFIG_SPEC.CARTETHYIA_SERVER_MAX_BODY_BYTES);
+}
+
+/** Resolves the listener idle-socket timeout (seconds). */
+export function resolveIdleTimeout(): number {
+  return readInt("CARTETHYIA_SERVER_IDLE_TIMEOUT", CONFIG_SPEC.CARTETHYIA_SERVER_IDLE_TIMEOUT);
+}
+
+/** Resolves the graceful-shutdown drain window: time for in-flight requests to finish before abort. */
+export function resolveShutdownDrainWindowMs(): number {
+  return readInt(
+    "CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS",
+    CONFIG_SPEC.CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS,
+  );
+}
+
+/** Resolves the operator drain token, or `undefined` when the drain route is disabled. */
+export function resolveDrainToken(): string | undefined {
+  return readOptionalText("CARTETHYIA_DRAIN_TOKEN", CONFIG_SPEC.CARTETHYIA_DRAIN_TOKEN);
+}
+
+/**
+ * Whether Elysia precompiles route handlers at startup (AOT).
+ *
+ * Precompilation takes the eager validator branch, which calls TypeBox's
+ * `SchemaCompile(schema)` and then reads `.buildResult.external` off the
+ * result. In a source-run process that field is absent — the lazy
+ * `require("typebox/schema")` resolves to a namespace that does not carry it —
+ * and the process dies at the first schema-bearing route with
+ * `undefined is not an object (evaluating 'this.tb.buildResult.external')`.
+ * `scripts/build/aot.ts` avoids this by rewriting the call site and bundling
+ * the namespaces statically, so only the built artifact can actually take the
+ * eager path.
+ *
+ * So: precompilation is on for production *builds*, and off for a source run
+ * even when `NODE_ENV=production`. A deployment that runs
+ * `bun run src/main.ts` must not be pushed onto the eager branch.
+ *
+ * `CARTETHYIA_SOURCE_RUN` marks a source run explicitly. It is set by
+ * `scripts/deploy/*.sh`, which are the only supported way to run from source.
+ */
+export function resolveElysiaPrecompile(): boolean {
+  if (process.env.CARTETHYIA_SOURCE_RUN === "1") return false;
+  return process.env.NODE_ENV === "production";
+}
+
+/**
+ * How many gateway processes this deployment runs, and which one this is.
+ *
+ * Multi-process is opt-in and deliberately narrow. Every process shares the
+ * port through `SO_REUSEPORT`, so the kernel load-balances across them — which
+ * means state that used to be process-local is now per-process unless it lives
+ * somewhere shared. A second process without that sharing is not "twice the
+ * capacity", it is two processes disagreeing:
+ *
+ * - **PostgreSQL, not SQLite.** Two processes on one SQLite file corrupt it;
+ *   there is no cross-process lock in the driver.
+ * - **Redis configured AND reachable.** Admission counters, quotas, and
+ *   cooldowns are per-process without it, so a per-key RPM limit silently
+ *   becomes N× the configured value.
+ * - **Linux.** `SO_REUSEPORT` is the mechanism; elsewhere it is a silent no-op
+ *   and the second process simply fails to bind.
+ *
+ * These are checked at resolution time rather than left to fail at bind time,
+ * because a half-configured multi-process deploy that starts anyway looks fine
+ * until a limit is enforced twice.
+ */
+export interface ProcessTopology {
+  readonly count: number;
+  readonly index: number;
+  /** True when more than one process serves this deployment. */
+  readonly multiProcess: boolean;
+}
+
+/**
+ * Resolves the topology from the environment.
+ *
+ * `count > 1` requires an explicit index: a process that does not know which
+ * one it is cannot own a shard of anything, and a guessed index would have two
+ * processes claiming the same one.
+ */
+export function resolveProcessTopology(): ProcessTopology {
+  const count = readInt("CARTETHYIA_PROCESS_COUNT", CONFIG_SPEC.CARTETHYIA_PROCESS_COUNT);
+  if (count === 1) return { count: 1, index: 0, multiProcess: false };
+  // Mandatory when the count is above one: a process that does not know which
+  // one it is cannot own a shard of anything, and a defaulted index would have
+  // two processes claiming the same number. The spec's default of 0 cannot
+  // express "absent", so the raw value is checked here before it is read.
+  const rawIndex = process.env.CARTETHYIA_PROCESS_INDEX?.trim();
+  if (rawIndex === undefined || rawIndex.length === 0) {
+    throw new Error(
+      `CARTETHYIA_PROCESS_INDEX is required when CARTETHYIA_PROCESS_COUNT is ${count} (0 – ${count - 1})`,
+    );
+  }
+  const index = readInt("CARTETHYIA_PROCESS_INDEX", CONFIG_SPEC.CARTETHYIA_PROCESS_INDEX);
+  if (index >= count) {
+    throw new Error(
+      `CARTETHYIA_PROCESS_INDEX must be an integer between 0 and ${count - 1} (got "${rawIndex}")`,
+    );
+  }
+  return { count, index, multiProcess: true };
+}

package/src/config/spec.ts

@@ -110,6 +110,28 @@ export const CONFIG_SPEC = {
     min: 0,
     max: 600_000,
   },
+  // Optional ceiling on in-flight requests per process. Left unset, the limit
+  // is derived from the memory limit and the body ceiling (see
+  // config/resources.ts): 2 GiB/4 MiB → 64, 512 MiB/4 MiB → 16, 256 MiB/16 MiB
+  // → 2. Set it to admit more than memory alone would allow — which is the
+  // operator's call, and is reported rather than corrected — or less, to keep
+  // a slow upstream from being handed more concurrency than it can serve.
+  // Optional-int, not int-with-a-zero-default: "unset" and "explicitly zero"
+  // must not be the same thing. Zero is not a valid ceiling, so a default of 0
+  // would need a second sentinel to mean "derive".
+  CARTETHYIA_PROCESS_CONCURRENCY_LIMIT: { kind: "optional-int", min: 1, max: 64 },
+  // Slack above each key's concurrency limit. Effective ceiling is
+  // limit + tolerance; only requests past it are refused.
+  CARTETHYIA_CONCURRENCY_TOLERANCE: { kind: "optional-int", min: 0, max: 1024 },
+  // How many gateway processes serve this deployment, and which one this is.
+  // Count > 1 requires Linux, CARTETHYIA_DB_MODE=full, and a REACHABLE Redis —
+  // checked at boot, because a half-configured multi-process deploy starts
+  // fine and then enforces every per-key limit N times over. Lite stays
+  // single-process: two processes on one SQLite file corrupt it.
+  CARTETHYIA_PROCESS_COUNT: { kind: "int", default: 1, min: 1, max: 16 },
+  // Which process this is (0 – count-1). Mandatory when count > 1: a process
+  // that does not know its index cannot own a shard of anything.
+  CARTETHYIA_PROCESS_INDEX: { kind: "int", default: 0, min: 0, max: 15 },
   // Optional secret for the operator drain endpoint (`POST /admin/drain`). When
   // set, a process can be stopped gracefully on platforms where a catchable
   // signal cannot be delivered to it (notably Windows, where `process.kill` of
@@ -130,9 +152,12 @@ export const CONFIG_SPEC = {
     min: 1,
     max: Number.MAX_SAFE_INTEGER,
   },
-  // Opt-in forced garbage collection. Off by default: see
-  // `observability/runtime-metrics.ts` — a forced collection is not
-  // reclamation, and the synchronous form stalls concurrent requests.
+  // Opt-in forced garbage collection under memory pressure. Off by default:
+  // see `observability/runtime-metrics.ts` — a forced collection is not
+  // reclamation, and a collection that runs while requests are in flight adds
+  // a latency spike. This gates the pressure path only; idle reclaim runs
+  // unconditionally, because reclaiming when nothing is in flight stalls
+  // nothing.
   CARTETHYIA_FORCED_GC: { kind: "flag" },
   // How often runtime gauges (memory + bounded-collection sizes) are sampled.
   CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS: {
@@ -151,37 +176,13 @@ export const CONFIG_SPEC = {
   /**
    * How long a day's telemetry is kept, in days.
    *
-   * The retention unit is the whole day: a day's event file and its index are
-   * deleted together, and the current day is never touched. Past this age an
-   * individual request no longer exists anywhere, so its detail is gone —
-   * totals, breakdowns and spend are unaffected, because the index keeps
-   * running sums rather than the rows.
+   * Whole raw event files past this age are removed. No summary or compacted
+   * form replaces the individual requests.
    *
    * This is the knob that decides how long the request list, per-request
    * filters, and the detail drawer stay complete.
    */
   CARTETHYIA_TELEMETRY_RETENTION_DAYS: { kind: "int", default: 30, min: 3, max: 365 },
-  /**
-   * Age at which a request's detail is described as no longer stored.
-   *
-   * Retention itself is `CARTETHYIA_TELEMETRY_RETENTION_DAYS`; this only words
-   * the operator-facing message shown when a request that old is asked for.
-   */
-  CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS: { kind: "int", default: 7, min: 1, max: 365 },
-  /**
-   * How often each day's index is brought up to date, in minutes.
-   *
-   * Bringing an index up to date means folding the bytes appended since the
-   * last pass — not re-reading the day. Reads fold before they answer too, so
-   * raising this does not make the numbers stale; it only leaves the on-disk
-   * index further behind, and the next read pays for the remainder.
-   */
-  CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES: {
-    kind: "int",
-    default: 60,
-    min: 5,
-    max: 1440,
-  },
   CARTETHYIA_TELEMETRY_PAYLOAD_MAX_BYTES: {
     kind: "int",
     default: 16 * 1024 * 1024,

package/src/config/telemetry.ts

@@ -20,28 +20,3 @@ export function resolveTelemetryPayloadMaxBytes(): number {
   );
 }
 
-/**
- * Age at which hourly telemetry files are compacted into a per-day summary.
- *
- * Clamped below the retention window: compacting at or past the delete age
- * would fold a file and throw it away in the same pass, spending the read for
- * totals that were about to be discarded.
- */
-export function resolveTelemetryCompactAfterDays(): number {
-  const compacted = readInt(
-    "CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS",
-    CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS,
-  );
-  return Math.min(compacted, resolveTelemetryRetentionDays());
-}
-
-/** Minutes between compaction passes. */
-export function resolveTelemetryCompactIntervalMs(): number {
-  const minutes = readInt(
-    "CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES",
-    CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES,
-  );
-  return minutes * 60_000;
-}
-
-

package/src/console/api-keys/bucket-spend.ts

@@ -2,7 +2,7 @@ import { eq } from "drizzle-orm";
 
 import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import { apiKeys } from "../../persistence/schema";
-import { queryIndexedBuckets } from "../../observability/event-day-index";
+import { sumTelemetryTokens } from "../../observability/event-query";
 import { utcDayStart, utcMonthStart } from "../shared/day";
 
 /**
@@ -27,27 +27,24 @@ export async function familyBucketSpend(
   keyId: string,
   now: Date,
 ): Promise<{ daily: number; monthly: number }> {
+  // The tenant comes from the key row itself: the caller passes only the key
+  // id, and the SQL sums are tenant-scoped so scoping must not be guessed.
+  const [parent] = await db
+    .select({ tenantId: apiKeys.tenantId })
+    .from(apiKeys)
+    .where(eq(apiKeys.id, keyId))
+    .limit(1);
+  const tenantId = parent?.tenantId;
+  if (tenantId === undefined) return { daily: 0, monthly: 0 };
   const children = await db
     .select({ id: apiKeys.id })
     .from(apiKeys)
     .where(eq(apiKeys.parentKeyId, keyId));
   const keyIds = [keyId, ...children.map((child) => child.id)];
-  const spentSince = async (start: Date): Promise<number> => {
-    // Folded first, so the index covers every event written so far including the
-    // current day's — then read from the index alone.
-    //
-    // Deliberately not index *plus* events: they hold the same requests, and
-    // adding them doubled every bucket. Folding first is what makes the index
-    // sufficient on its own; the watermark is the point up to which it is
-    // complete.
-    const allowed = new Set(keyIds);
-    const indexed = await queryIndexedBuckets({ since: start, foldFirst: true }).catch(
-      () => [],
-    );
-    return indexed
-      .filter((bucket) => bucket.apiKeyId !== null && allowed.has(bucket.apiKeyId))
-      .reduce((sum, bucket) => sum + bucket.inputTokens + bucket.outputTokens, 0);
-  };
+  // Two SQL sums instead of reading every event of the family: the bucket
+  // spend only needs a total, and this runs on the key-creation path.
+  const spentSince = async (start: Date): Promise<number> =>
+    sumTelemetryTokens(db, tenantId, { apiKeyIds: keyIds, since: start });
   const [daily, monthly] = await Promise.all([
     spentSince(utcDayStart(now)),
     spentSince(utcMonthStart(now)),