cartethyia 2.5.2 → 2.5.4

package/src/config/spec.ts

@@ -148,7 +148,62 @@ export const CONFIG_SPEC = {
    */
   CARTETHYIA_DISABLED_TASKS: { kind: "list" },
 
+  /**
+   * How long a day's telemetry is kept, in days.
+   *
+   * The retention unit is the whole day: a day's event file and its index are
+   * deleted together, and the current day is never touched. Past this age an
+   * individual request no longer exists anywhere, so its detail is gone —
+   * totals, breakdowns and spend are unaffected, because the index keeps
+   * running sums rather than the rows.
+   *
+   * This is the knob that decides how long the request list, per-request
+   * filters, and the detail drawer stay complete.
+   */
   CARTETHYIA_TELEMETRY_RETENTION_DAYS: { kind: "int", default: 30, min: 3, max: 365 },
+  /**
+   * Age at which a request's detail is described as no longer stored.
+   *
+   * Retention itself is `CARTETHYIA_TELEMETRY_RETENTION_DAYS`; this only words
+   * the message shown when a request that old is asked for. Kept because the
+   * message is operator-facing and was written against this knob.
+   */
+  CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS: { kind: "int", default: 7, min: 1, max: 365 },
+  /**
+   * How often each day's index is brought up to date, in minutes.
+   *
+   * Bringing an index up to date means folding the bytes appended since the
+   * last pass — not re-reading the day. Reads fold before they answer too, so
+   * raising this does not make the numbers stale; it only leaves the on-disk
+   * index further behind, and the next read pays for the remainder.
+   */
+  CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES: {
+    kind: "int",
+    default: 60,
+    min: 5,
+    max: 1440,
+  },
+  /**
+   * Inert. Accepted so existing deployments keep booting; nothing reads it.
+   *
+   * It sized the one-file-per-window scheme — events were written to a file
+   * per this many minutes, 96 a day at the default. Events now append to one
+   * file per UTC day, so there is no window left to size.
+   */
+  CARTETHYIA_TELEMETRY_WINDOW_MINUTES: {
+    kind: "int",
+    default: 15,
+    min: 1,
+    max: 1440,
+  },
+  /**
+   * Inert. Accepted so existing deployments keep booting; nothing reads it.
+   *
+   * It capped how many window files one pass could fold. Folding is driven by
+   * a byte offset into the day file now and never deletes the file, so there
+   * is no per-pass batch to bound.
+   */
+  CARTETHYIA_TELEMETRY_COMPACT_MAX_FILES: { kind: "int", default: 48, min: 1, max: 10_000 },
   CARTETHYIA_TELEMETRY_PAYLOAD_MAX_BYTES: {
     kind: "int",
     default: 16 * 1024 * 1024,
@@ -157,7 +212,14 @@ export const CONFIG_SPEC = {
   },
 
   // Upstream request timeout + retry backoff
-  CARTETHYIA_UPSTREAM_TIMEOUT_MS: { kind: "int", default: 120_000, min: 5_000, max: 600_000 },
+  //
+  // Strictly under 90s: a client whose own deadline is 90s must never be
+  // waiting on a gateway that is still holding a request open past it, and a
+  // provider that has not answered within 75s is not answering — the request
+  // is queued upstream or the egress is stuck, and the honest move is to fail
+  // over to another account rather than burn the caller's whole budget on one.
+  // 120s outlived the clients that call this, which cap lower.
+  CARTETHYIA_UPSTREAM_TIMEOUT_MS: { kind: "int", default: 75_000, min: 5_000, max: 600_000 },
   CARTETHYIA_STREAM_STALL_TIMEOUT_MS: {
     kind: "int",
     default: 360_000,
@@ -181,7 +243,24 @@ export const CONFIG_SPEC = {
   CARTETHYIA_ACCOUNT_QUOTA_COOLDOWN_MS: { kind: "int", default: 3_600_000, min: 0, max: 86_400_000 },
   CARTETHYIA_ACCOUNT_MODEL_CAPACITY_COOLDOWN_MS: { kind: "int", default: 120_000, min: 0, max: 86_400_000 },
   CARTETHYIA_ACCOUNT_TRANSIENT_COOLDOWN_MS: { kind: "int", default: 30_000, min: 0, max: 86_400_000 },
-  CARTETHYIA_ACCOUNT_UNCLASSIFIED_COOLDOWN_MS: { kind: "int", default: 60_000, min: 0, max: 86_400_000 },
+  /**
+   * An unmapped upstream failure cools the account down for 15 minutes.
+   *
+   * It used to be 60s, which put the account straight back into rotation and
+   * re-dialled an upstream that had just answered something this gateway does
+   * not understand — so the same unknown error was retried every minute, and
+   * an account that was actually broken contributed a steady stream of failed
+   * attempts instead of one. Fifteen minutes is long enough for a transient
+   * provider condition to clear on its own and short enough that no operator
+   * action is needed to recover it. It is still a cooldown with a deadline,
+   * never `disabled`, so nothing here can strand an account.
+   */
+  CARTETHYIA_ACCOUNT_UNCLASSIFIED_COOLDOWN_MS: {
+    kind: "int",
+    default: 900_000,
+    min: 0,
+    max: 86_400_000,
+  },
   CARTETHYIA_ACCOUNT_COOLDOWN_ESCALATION_MAX_MS: { kind: "int", default: 3_600_000, min: 0, max: 86_400_000 },
   CARTETHYIA_POOL_COOLDOWN_MS: { kind: "int", default: 120_000, min: 0, max: 86_400_000 },
   CARTETHYIA_ACCOUNT_AUTH_REJECTED_COOLDOWN_MS: { kind: "int", default: 300_000, min: 0, max: 86_400_000 },

package/src/config/telemetry.ts

@@ -19,3 +19,49 @@ export function resolveTelemetryPayloadMaxBytes(): number {
     CONFIG_SPEC.CARTETHYIA_TELEMETRY_PAYLOAD_MAX_BYTES,
   );
 }
+
+/**
+ * Age at which hourly telemetry files are compacted into a per-day summary.
+ *
+ * Clamped below the retention window: compacting at or past the delete age
+ * would fold a file and throw it away in the same pass, spending the read for
+ * totals that were about to be discarded.
+ */
+export function resolveTelemetryCompactAfterDays(): number {
+  const compacted = readInt(
+    "CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS",
+    CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS,
+  );
+  return Math.min(compacted, resolveTelemetryRetentionDays());
+}
+
+/** Minutes between compaction passes. */
+export function resolveTelemetryCompactIntervalMs(): number {
+  const minutes = readInt(
+    "CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES",
+    CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_INTERVAL_MINUTES,
+  );
+  return minutes * 60_000;
+}
+
+/**
+ * Minutes of events per source file, before it is folded into the day summary.
+ *
+ * Defaults to the sweep interval so exactly one window closes per pass: a
+ * window longer than the interval would leave files waiting several passes, and
+ * a shorter one would let several close between passes.
+ */
+export function resolveTelemetryWindowMinutes(): number {
+  return readInt(
+    "CARTETHYIA_TELEMETRY_WINDOW_MINUTES",
+    CONFIG_SPEC.CARTETHYIA_TELEMETRY_WINDOW_MINUTES,
+  );
+}
+
+/** Maximum hourly files folded in one compaction pass. */
+export function resolveTelemetryCompactMaxFiles(): number {
+  return readInt(
+    "CARTETHYIA_TELEMETRY_COMPACT_MAX_FILES",
+    CONFIG_SPEC.CARTETHYIA_TELEMETRY_COMPACT_MAX_FILES,
+  );
+}

package/src/console/api-keys/bucket-spend.ts

@@ -2,7 +2,7 @@ import { eq } from "drizzle-orm";
 
 import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import { apiKeys } from "../../persistence/schema";
-import { queryTelemetryEvents } from "../../observability/event-log";
+import { queryIndexedBuckets } from "../../observability/event-day-index";
 import { utcDayStart, utcMonthStart } from "../shared/day";
 
 /**
@@ -33,9 +33,20 @@ export async function familyBucketSpend(
     .where(eq(apiKeys.parentKeyId, keyId));
   const keyIds = [keyId, ...children.map((child) => child.id)];
   const spentSince = async (start: Date): Promise<number> => {
-    const events = await queryTelemetryEvents({ apiKeyIds: keyIds, since: start });
-    return events
-      .reduce((sum, event) => sum + (event.inputTokens ?? 0) + (event.outputTokens ?? 0), 0);
+    // Folded first, so the index covers every event written so far including the
+    // current day's — then read from the index alone.
+    //
+    // Deliberately not index *plus* events: they hold the same requests, and
+    // adding them doubled every bucket. Folding first is what makes the index
+    // sufficient on its own; the watermark is the point up to which it is
+    // complete.
+    const allowed = new Set(keyIds);
+    const indexed = await queryIndexedBuckets({ since: start, foldFirst: true }).catch(
+      () => [],
+    );
+    return indexed
+      .filter((bucket) => bucket.apiKeyId !== null && allowed.has(bucket.apiKeyId))
+      .reduce((sum, bucket) => sum + bucket.inputTokens + bucket.outputTokens, 0);
   };
   const [daily, monthly] = await Promise.all([
     spentSince(utcDayStart(now)),

package/src/console/backup/contracts.ts

@@ -20,6 +20,7 @@
  * as sensitive as the database.
  */
 import { getTableColumns, getTableName, type Column, type Table } from "drizzle-orm";
+import { resolveMaxBodyBytes } from "../../config/runtime";
 import {
   apiKeys,
   cliToolMappings,
@@ -45,8 +46,20 @@ import {
 export const BACKUP_APP = "cartethyia";
 export const BACKUP_VERSION = 1;
 
-/** Upper bound for restore payloads; the listener enforces the same ceiling. */
-export const MAX_BACKUP_BYTES = 8 * 1024 * 1024 * 1024;
+/**
+ * Upper bound for restore payloads.
+ *
+ * Deliberately not larger than the listener's own body cap. It used to claim
+ * 8 GiB while the server admitted 4 MiB, so the number described a limit that
+ * could never be reached: a file above the listener's cap is cut off before
+ * this check runs, and a large upload fails as a reset connection rather than
+ * the 413 this constant promises. The bound now states what the install
+ * actually admits, so the message an operator sees matches what happened.
+ *
+ * Raise `CARTETHYIA_SERVER_MAX_BODY_BYTES` to accept larger backups; this
+ * constant follows it rather than the two drifting apart again.
+ */
+export const MAX_BACKUP_BYTES = resolveMaxBodyBytes();
 
 /** Defense-in-depth row cap per table, after the byte bound. */
 export const MAX_ROWS_PER_TABLE = 500_000;

package/src/console/backup/routes.ts

@@ -78,6 +78,24 @@ const BACKUP_SCOPES = ["dashboard:write"] as const;
 const importBody = t.Object({
   password: t.String(),
   backup: t.Unknown(),
+  /**
+   * Table names to restore. Absent or empty restores every table in the file,
+   * which is what the dashboard's checklist defaults to and what the API did
+   * before selection existed.
+   */
+  tables: t.Optional(t.Array(t.String())),
+});
+
+/**
+ * The inspect body: a file, no password.
+ *
+ * Inspection writes nothing and reads no secret — it only reports which tables
+ * the file carries and which of them the live schema has — so it needs no
+ * re-authentication. Requiring the console password here would mean typing it
+ * twice for one restore.
+ */
+const inspectBody = t.Object({
+  backup: t.Unknown(),
 });
 
 const exportQuery = t.Object({
@@ -164,7 +182,12 @@ export function createBackupRoutes(config: BackupRoutesConfig): Elysia {
         // from nowhere else. A request body may not name the tenant it writes
         // into: that would let any holder of the scope import into, and
         // overwrite, a tenant they are not.
-        const result = await config.backupFor(request).restore(body.password, body.backup, access.tenantId);
+        const result = await config.backupFor(request).restore(
+          body.password,
+          body.backup,
+          access.tenantId,
+          body.tables === undefined ? undefined : { tables: body.tables },
+        );
         // Routing-visible write: the restore has committed, so the cached
         // snapshot must be rebuilt before the next `/v1/*` dispatch.
         await config.snapshotInvalidator?.invalidate();
@@ -214,5 +237,33 @@ export function createBackupRoutes(config: BackupRoutesConfig): Elysia {
       } catch (error) {
         return errorResponse(error, set, "Delete all failed");
       }
+    })
+    .post("/inspect", { body: inspectBody }, async ({ request, body, set }) => {
+      try {
+        const access = requireAnyScope(config.accessResolver(request), BACKUP_SCOPES);
+        if (access.tenantId === null) {
+          throw new ConsoleDomainError("tenant_required", 403, "Tenant isolation required");
+        }
+        // Same byte bound as the import, for the same reason: the file is
+        // parsed here too, and an unbounded one would be buffered before any
+        // validation could refuse it.
+        const declared = Number(request.headers.get("content-length") ?? "0");
+        const actualBytes = JSON.stringify(body ?? null).length;
+        if (
+          (Number.isFinite(declared) && declared > MAX_BACKUP_BYTES) ||
+          actualBytes > MAX_BACKUP_BYTES
+        ) {
+          set.status = 413;
+          return {
+            error: `backup exceeds the ${MAX_BACKUP_BYTES} byte limit`,
+            code: "request_too_large",
+          };
+        }
+        // Reads nothing and writes nothing: the checklist the restore wizard
+        // shows, so an operator can pick sections before typing a password.
+        return config.backupFor(request).inspect(body.backup);
+      } catch (error) {
+        return errorResponse(error, set, "Backup inspection failed");
+      }
     }) as unknown as Elysia;
 }

package/src/console/backup/service.ts

@@ -19,6 +19,7 @@
  * re-authentication.
  */
 import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import type { Table } from "drizzle-orm";
 import {
   models,
   providerAccounts,
@@ -29,7 +30,12 @@ import {
 } from "../../persistence/schema";
 import { ConsoleDomainError } from "../shared/errors";
 import { tablesForSection, tableName } from "./contracts";
-import type { BackupSection, RestoreWarning, ValidatedRestore } from "./contracts";
+import type {
+  BackupSection,
+  RestoreWarning,
+  ValidatedRestore,
+  ValidatedTable,
+} from "./contracts";
 import {
   applyRestore,
   decompressBackup,
@@ -90,6 +96,53 @@ export interface ImportResult {
   readonly format: "native" | "nine_router";
 }
 
+/**
+ * Which parts of a backup file a restore should actually write.
+ *
+ * Absent or empty means "everything", which is the historical behavior and
+ * what the dashboard's checklist defaults to. A table named here but absent
+ * from the file is simply not restored — it is not an error, because the
+ * checklist is built from an inspection of *this* file and a selection never
+ * invents a table.
+ */
+export interface BackupRestoreSelection {
+  /** Table names to restore; absent or empty restores every table present. */
+  readonly tables?: readonly string[];
+}
+
+/**
+ * Narrows a validated backup to the tables the operator selected.
+ *
+ * The tenant row is the one thing that is not independently selectable: it is
+ * not a table in either section (`BACKUP_SECTIONS` are config and telemetry),
+ * so it never appears on the checklist, and every tenant-scoped table
+ * cascades from it. Dropping it because it was not named would make a partial
+ * restore fail on a foreign key for a table the operator *did* ask for. So it
+ * rides along with any config selection and is written only when one is made.
+ *
+ * A table named in the selection but missing from the file is a no-op rather
+ * than an error: the checklist is built from an inspection of this file, but
+ * the selection is validated as names only, and refusing to restore anything
+ * over one stale name would be worse than ignoring it.
+ */
+function restrictToSelection(
+  validation: ValidatedRestore,
+  selection: BackupRestoreSelection | undefined,
+): ValidatedRestore {
+  const wanted = selection?.tables;
+  if (wanted === undefined || wanted.length === 0) return validation;
+  const selected = new Set(wanted);
+  const tables = new Map<Table, ValidatedTable>();
+  for (const [table, entry] of validation.tables) {
+    if (selected.has(tableName(table))) tables.set(table, entry);
+  }
+  return {
+    ...validation,
+    tables,
+    tenantRows: tables.size > 0 ? validation.tenantRows : [],
+  };
+}
+
 function sectionsOrDefault(sections: readonly BackupSection[] | undefined): readonly BackupSection[] {
   if (sections === undefined || sections.length === 0) return ["config"];
   for (const section of sections) {
@@ -246,6 +299,7 @@ export class BackupService {
     password: unknown,
     payload: unknown,
     tenantId: string,
+    selection?: BackupRestoreSelection,
   ): Promise<ImportResult> {
     if (typeof password !== "string" || !(await this.options.verifyPassword(password))) {
       throw new ConsoleDomainError("unauthorized", 401, "password is incorrect");
@@ -258,11 +312,11 @@ export class BackupService {
 
     if (detected.kind === "nine_router") {
       const { payload: converted, report } = convert9RouterBackup(detected.payload, tenantId);
-      const result = await this.apply(converted, tenantId);
+      const result = await this.apply(converted, tenantId, selection);
       return { ...result, report, format: "nine_router" };
     }
 
-    const result = await this.apply(detected.payload, tenantId);
+    const result = await this.apply(detected.payload, tenantId, selection);
     return { ...result, format: "native" };
   }
 
@@ -348,13 +402,15 @@ export class BackupService {
   private async apply(
     payload: unknown,
     tenantId: string,
+    selection?: BackupRestoreSelection,
   ): Promise<{ restored: Record<string, number>; skipped: Record<string, number>; warnings: readonly RestoreWarning[] }> {
     const validation = validateRestorePayload(payload, tenantId);
     if (!validation.ok) {
       throw new ConsoleDomainError("invalid_request", 400, validation.error);
     }
     try {
-      const compatible = await omitRowsForUnavailableProviders(this.options.db, validation.value, tenantId);
+      const scoped = restrictToSelection(validation.value, selection);
+      const compatible = await omitRowsForUnavailableProviders(this.options.db, scoped, tenantId);
       const applied = await applyRestore(this.options.db, compatible, restoreOrder(), tenantId);
       return { ...applied, warnings: compatible.warnings };
     } catch (error) {

package/src/console/console-router.ts

@@ -48,6 +48,13 @@ export interface ConsoleApiCompositionDeps {
   readonly routeSnapshotService: { invalidate(): Promise<number> };
   readonly poolSelector: NetworkPoolSelector;
   readonly telemetryBuffer: TelemetryBatchBuffer;
+  /**
+   * Queued writers the console can drain on demand (telemetry, usage, health
+   * events). Optional for the same reason the other reduced composition
+   * surface is: absent means the flush control is not offered rather than the
+   * console failing to build.
+   */
+  readonly flushBuffers?: { flush(): Promise<void> };
   /**
    * The request state store owns the live in-flight gauge, so the console reads
    * its snapshot/subscription from here rather than a free-standing module.
@@ -189,6 +196,7 @@ export function createConsoleRouter(deps: ConsoleApiCompositionDeps): Elysia {
     admissionService: deps.admissionService,
     ...(deps.modelStrikes ? { modelStrikes: deps.modelStrikes } : {}),
     readRoutingAccountInflight: deps.readRoutingAccountInflight,
+    ...(deps.flushBuffers ? { flushBuffers: deps.flushBuffers } : {}),
     credentialService,
     // The backup surface re-authenticates the operator, so it needs the current
     // user's hash. Read from the session on the request that asks for it, never

package/src/console/data-management/contracts.ts

@@ -0,0 +1,93 @@
+/**
+ * Disk housekeeping: what the telemetry and payload directories hold, and the
+ * reclaim actions an operator can run from the console.
+ *
+ * Everything here is files on disk, except one deliberate row-level action.
+ * Reclaiming a file is safe because the data it held is already past the point
+ * where the gateway would read it: payload frames expire 15 minutes after they
+ * are written, and telemetry files past the retention window are outside every
+ * query's scan bound. Deleting rows is a different decision with a different
+ * blast radius, and the only row data the console offers purging is the health
+ * & error log — purely an operator's diagnostic trail with no request-path
+ * reader, unlike everything else in this panel.
+ *
+ * Sizes are bytes as `number`. A directory is scanned on each read rather than
+ * cached, because the point of the panel is to show what is true now — a
+ * stale size would be worse than none.
+ */
+
+/** One measured category of on-disk data. */
+export interface DiskUsageBucket {
+  /** Stable id, so the dashboard can key rows without matching on copy. */
+  readonly id: "telemetry_hourly" | "telemetry_compacted" | "payload_frames";
+  readonly label: string;
+  readonly bytes: number;
+  readonly files: number;
+  /** Why this data exists, shown next to the size. */
+  readonly note: string;
+  /** Bytes reclaimable right now, when that is knowable without a dry run. */
+  readonly reclaimableBytes: number | null;
+  readonly reclaimableFiles: number | null;
+}
+
+export interface DiskUsageResponse {
+  readonly buckets: readonly DiskUsageBucket[];
+  readonly totalBytes: number;
+  /** Retention in days; telemetry older than this is outside every scan. */
+  readonly retentionDays: number;
+  /** Payload frames live this long before they are reclaimable. */
+  readonly payloadRetentionMs: number;
+  /**
+   * Stored health & error rows, for the purge row's label. Absent when the
+   * composition does not wire the health-event store (a reduced console),
+   * which is also when the purge control is not offered.
+   */
+  readonly healthEventCount?: number;
+}
+
+export type ReclaimAction = "payload_frames" | "telemetry_retention" | "telemetry_hourly";
+
+export interface ReclaimResponse {
+  readonly action: ReclaimAction;
+  readonly deletedFiles: number;
+  readonly reclaimedBytes: number;
+}
+
+/** Row-level deletion targets. The health & error log is the only one. */
+export type PurgeTarget = "health_events";
+
+export interface PurgeResponse {
+  readonly target: PurgeTarget;
+  /** Rows actually deleted — the number the operator's decision produced. */
+  readonly deleted: number;
+}
+
+export interface FlushResponse {
+  readonly flushed: boolean;
+}
+
+/**
+ * A reclaim the caller is not allowed to make.
+ *
+ * Kept as a named class so the route maps it to 403 rather than letting it
+ * surface as a 500 — an operator asking for something out of policy is a
+ * normal request, not a failure.
+ */
+export class ReclaimNotAllowedError extends Error {
+  readonly status = 403;
+
+  constructor(message: string) {
+    super(message);
+    this.name = "ReclaimNotAllowedError";
+  }
+}
+
+/** A purge whose backing store is not wired in this composition. */
+export class PurgeNotWiredError extends Error {
+  readonly status = 501;
+
+  constructor() {
+    super("Health & error log purging is not offered in this deployment's console composition.");
+    this.name = "PurgeNotWiredError";
+  }
+}