cartethyia 2.4.6 → 2.5.0

package/src/persistence/health-event-flush.ts

@@ -0,0 +1,137 @@
+// Deferred writes for the account/pool health audit log.
+//
+// `health_events` is an append-only operator log: nothing in the request path
+// reads it. It was written synchronously inside the same transaction as the
+// `provider_accounts` UPDATE that routing *does* read, so every failed attempt
+// paid an extra INSERT before the request could finish — on the embedded
+// backend, where writes queue behind a single writer, that was the difference
+// between one write and two.
+//
+// Only the audit row moves off the hot path. The status UPDATE stays
+// synchronous and transactional, because routing reads it: deferring that would
+// let a subsequent request pick an account that had just failed, which is worse
+// under exactly the failure wave this work targets.
+//
+// Same shape as `persistence/usage-flush.ts`: enqueue and return, a timer
+// flushes, shutdown flushes what remains. A crash loses the unflushed rows and
+// nothing else — the durable state routing depends on was never in here.
+import type { CartethyiaDatabase } from "./db-handle";
+import { healthEvents } from "./schema";
+
+/** One audit row waiting to be written. */
+export interface HealthEventRow {
+  readonly entityKind: "account" | "pool";
+  readonly accountId?: string | null;
+  readonly networkPoolId?: string | null;
+  readonly fromStatus: string | null;
+  readonly toStatus: string;
+  readonly reason: string;
+  readonly errorCategory: string | null;
+  readonly modelId?: string | null;
+  readonly createdAt: Date;
+}
+
+/**
+ * Rows dropped rather than accumulated without bound.
+ *
+ * A health log is worth less than a stable process: a failure wave that cannot
+ * be flushed must not grow this queue until the process dies. Beyond the cap
+ * the oldest rows are discarded and counted.
+ */
+const MAX_PENDING_ROWS = 2_000;
+
+/**
+ * Process-wide sink, injected by `account-health-service` and the pool-health
+ * path.
+ *
+ * A module-level optional rather than a constructor parameter on every call:
+ * those functions are reached from dozens of call sites with a plain database
+ * handle, and threading a buffer through all of them would be a far larger
+ * change than the write it defers. `undefined` means "write synchronously",
+ * which is today's behaviour and what tests that assert the row landed expect.
+ */
+let activeBuffer: HealthEventFlushBuffer | undefined;
+
+/** Installs the process-wide buffer. */
+export function setHealthEventBuffer(buffer: HealthEventFlushBuffer | undefined): void {
+  activeBuffer = buffer;
+}
+
+/** The installed buffer, or `undefined` when writes must stay synchronous. */
+export function healthEventBuffer(): HealthEventFlushBuffer | undefined {
+  return activeBuffer;
+}
+
+export class HealthEventFlushBuffer {
+  private readonly pending: HealthEventRow[] = [];
+  private timer: ReturnType<typeof setInterval> | undefined;
+  private flushing = false;
+  private dropped = 0;
+
+  constructor(
+    private readonly db: CartethyiaDatabase,
+    private readonly intervalMs = 1_000,
+  ) {
+    this.arm();
+  }
+
+  private arm(): void {
+    if (this.timer !== undefined) return;
+    // `unref` so a pending interval never holds the process open at shutdown;
+    // `stop()` drains explicitly instead.
+    this.timer = setInterval(() => void this.flush(), this.intervalMs);
+    this.timer.unref?.();
+  }
+
+  /** Records one audit row. Never touches the database; safe per attempt. */
+  enqueue(row: HealthEventRow): void {
+    if (this.pending.length >= MAX_PENDING_ROWS) {
+      this.pending.shift();
+      this.dropped += 1;
+    }
+    this.pending.push(row);
+  }
+
+  /** Rows discarded because the queue was full. Observability for the buffer. */
+  get droppedCount(): number {
+    return this.dropped;
+  }
+
+  /** Writes everything accumulated so far. Safe to call concurrently. */
+  async flush(): Promise<void> {
+    if (this.flushing || this.pending.length === 0) return;
+    this.flushing = true;
+    // Drained before the await: rows enqueued while the insert runs stay for
+    // the next pass rather than being lost if this one throws.
+    const batch = this.pending.splice(0, this.pending.length);
+    try {
+      await this.db.insert(healthEvents).values(
+        batch.map((row) => ({
+          entityKind: row.entityKind,
+          accountId: row.accountId ?? null,
+          networkPoolId: row.networkPoolId ?? null,
+          fromStatus: row.fromStatus as "active" | "cooldown" | "disabled" | null,
+          toStatus: row.toStatus as "active" | "cooldown" | "disabled",
+          reason: row.reason,
+          errorCategory: row.errorCategory as never,
+          modelId: row.modelId ?? null,
+          createdAt: row.createdAt,
+        })),
+      );
+    } catch {
+      // An audit write must never fail a request or retry a dispatch. The rows
+      // are dropped; the durable status change was written synchronously.
+    } finally {
+      this.flushing = false;
+    }
+  }
+
+  /** Stops the timer and flushes what remains. Called at shutdown. */
+  async stop(): Promise<void> {
+    if (this.timer !== undefined) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    await this.flush();
+  }
+}

package/src/persistence/migrate.ts

@@ -1,129 +1,181 @@
-/** Migration discovery + ledger helpers shared by every database backend. */
-import { sql } from "drizzle-orm";
-import { createHash } from "node:crypto";
-import { existsSync, readdirSync } from "node:fs";
-import { basename, resolve } from "node:path";
-import { resolvePackagePath } from "../config/package-paths";
-import type { CartethyiaDatabase } from "./db-handle";
-
-/**
- * SQL-only migration ledger. Migration files are the source of truth; this
- * table stores only which numbered files have already run.
- */
-export const MIGRATION_LEDGER_TABLE = "cartethyia_schema_migrations";
-
-export const LEDGER_DDL = `CREATE TABLE IF NOT EXISTS ${MIGRATION_LEDGER_TABLE} (
-  migration_id text PRIMARY KEY,
-  applied_at timestamptz NOT NULL DEFAULT now()
-)`;
-
-export function resolveMigrationsFolder(): string {
-  const folder = resolvePackagePath("migrations", "full-postgres");
-  if (!existsSync(folder)) {
-    throw new Error(`Migrations folder not found: ${folder}`);
-  }
-  return folder;
-}
-
-export function migrationFiles(folder: string): readonly string[] {
-  return readdirSync(folder)
-    .filter((file) => /^\d{4}_.+\.sql$/.test(file))
-    .sort()
-    .map((file) => resolve(folder, file));
-}
-
-export function migrationIdFor(file: string): string {
-  return basename(file);
-}
-
-/** The only migration file a fresh database builds from. */
-export const BASELINE_MIGRATION_ID = "0000_baseline.sql";
-const BASELINE_SCHEMA_MARKER_PREFIX = "baseline-schema:";
-
-/** Identifies the exact baseline schema source recorded by a fresh database. */
-export function baselineSchemaMarker(source: string): string {
-  const hash = createHash("sha256").update(source).digest("hex");
-  return `${BASELINE_SCHEMA_MARKER_PREFIX}${hash}`;
-}
-
-/**
- * Rejects databases whose schema was created by retired migration files or
- * which have tables but no record of the current baseline.
- */
-export function assertMigrationHistorySupported(
-  applied: ReadonlySet<string>,
-  expected: readonly string[],
-  hasTenants: boolean,
-  currentBaselineMarker?: string,
-): void {
-  const supported = new Set(expected);
-  const retired = [...applied].filter((migrationId) => !supported.has(migrationId));
-  if (retired.length > 0) {
-    throw new Error(
-      `Database contains retired migration history (${retired.join(", ")}). ` +
-      "Create a fresh database and restore from a backup; in-place upgrades are no longer supported.",
-    );
-  }
-  if (hasTenants && !applied.has(BASELINE_MIGRATION_ID)) {
-    throw new Error(
-      "Database contains an unversioned schema. Create a fresh database and restore from a backup; " +
-      "in-place upgrades are no longer supported.",
-    );
-  }
-  if (
-    hasTenants &&
-    currentBaselineMarker !== undefined &&
-    !applied.has(currentBaselineMarker)
-  ) {
-    throw new Error(
-      "Database was created from a different baseline schema. Create a fresh database and restore from a backup; " +
-      "in-place upgrades are no longer supported.",
-    );
-  }
-}
-
-/**
- * Splits a migration file on the drizzle `statement-breakpoint` markers the
- * files already carry. Only the embedded backend needs per-statement
- * application (single-statement failure tolerance); the pg path sends the
- * whole file at once exactly as before.
- */
-export function splitStatements(source: string): string[] {
-  return source
-    .split(/--> statement-breakpoint/g)
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-/**
- * Readiness view of the migration ledger: the numbered SQL files discovered on
- * disk compared against the ids recorded in {@link MIGRATION_LEDGER_TABLE}.
- */
-export interface MigrationLedgerStatus {
-  /** Migration file names discovered on disk, in apply order. */
-  readonly expected: readonly string[];
-  /** Expected migrations that have no ledger row. */
-  readonly pending: readonly string[];
-  /** True only when every expected migration is recorded in the ledger. */
-  readonly applied: boolean;
-}
-
-/**
- * Compares the migration files discovered on disk against the applied-migration
- * ledger. A reachable ledger alone is not readiness: an interrupted upgrade
- * leaves the table present while later migrations are still unapplied.
- *
- * Runs on the drizzle handle, so it is identical on every backend.
- */
-export async function readMigrationLedgerStatus(
-  db: CartethyiaDatabase,
-  folder: string = resolveMigrationsFolder(),
-): Promise<MigrationLedgerStatus> {
-  const expected = migrationFiles(folder).map((file) => migrationIdFor(file));
-  const result = await db.execute<{ migration_id: string }>(
-    sql.raw(`SELECT migration_id FROM ${MIGRATION_LEDGER_TABLE}`),
-  );
-  const applied = new Set(result.rows.map((row) => row.migration_id));
-  const pending = expected.filter((migrationId) => !applied.has(migrationId));
-  return { expected, pending, applied: pending.length === 0 };
-}
+/** Migration discovery + ledger helpers shared by every database backend. */
+import { sql } from "drizzle-orm";
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync } from "node:fs";
+import { basename, resolve } from "node:path";
+import { resolvePackagePath } from "../config/package-paths";
+import type { CartethyiaDatabase } from "./db-handle";
+
+/**
+ * SQL-only migration ledger. Migration files are the source of truth; this
+ * table stores only which numbered files have already run.
+ */
+export const MIGRATION_LEDGER_TABLE = "cartethyia_schema_migrations";
+
+export const LEDGER_DDL = `CREATE TABLE IF NOT EXISTS ${MIGRATION_LEDGER_TABLE} (
+  migration_id text PRIMARY KEY,
+  applied_at timestamptz NOT NULL DEFAULT now()
+)`;
+
+export function resolveMigrationsFolder(): string {
+  const folder = resolvePackagePath("migrations", "full-postgres");
+  if (!existsSync(folder)) {
+    throw new Error(`Migrations folder not found: ${folder}`);
+  }
+  return folder;
+}
+
+export function migrationFiles(folder: string): readonly string[] {
+  return readdirSync(folder)
+    .filter((file) => /^\d{4}_.+\.sql$/.test(file))
+    .sort()
+    .map((file) => resolve(folder, file));
+}
+
+export function migrationIdFor(file: string): string {
+  return basename(file);
+}
+
+/** The only migration file a fresh database builds from. */
+export const BASELINE_MIGRATION_ID = "0000_baseline.sql";
+const BASELINE_SCHEMA_MARKER_PREFIX = "baseline-schema:";
+
+/**
+ * Identifies the exact baseline schema recorded by a fresh database.
+ *
+ * The hash is taken over the *statements*, with comments stripped, because the
+ * raw file text is not the thing that matters: a comment edit changes the hash
+ * without changing the schema, and a changed hash makes every existing database
+ * refuse to boot ("different baseline schema"). That is not hypothetical — a
+ * documentation-only edit to the SQLite baseline header shipped in 2.4.6 and
+ * locked out every database created by 2.4.5.
+ *
+ * Stripping comments makes the marker track the schema a database was built
+ * from, which is what the check is actually asserting.
+ */
+export function baselineSchemaMarker(source: string): string {
+  return `${BASELINE_SCHEMA_MARKER_PREFIX}${hashStatements(source)}`;
+}
+
+/**
+ * Markers recorded by releases that hashed the raw baseline file text.
+ *
+ * Every 2.4.x release shipped the same SQLite baseline, so a database created by
+ * any of them carries this one marker. The *schema* is identical to the current
+ * baseline (verified: stripping comments from both yields the same hash), so
+ * these are accepted as proof of the same schema instead of telling every
+ * existing user to recreate their database.
+ *
+ * These are frozen history. Never add a marker for a schema that actually
+ * differs — that would let an incompatible database boot against a newer
+ * runtime, which is the failure this check exists to prevent.
+ */
+const LEGACY_BASELINE_SCHEMA_MARKERS: readonly string[] = [
+  // full-postgres/0000_baseline.sql — unchanged across v2.4.0-v2.4.6, so this is
+  // the raw-text hash every existing Full-mode database carries.
+  "baseline-schema:bf7d2b4f1f99b4470c81850a77cc4ebc989997fbe8c8b68ec62833164c73a6ad",
+  // lite-sqlite3/0000_baseline.sql, v2.4.0 – v2.4.5 (raw-text sha256).
+  "baseline-schema:438c5e014c5f86f483a84e8a106b0e62143d4eb0b0daad728a300bcc683d8c02",
+  // lite-sqlite3/0000_baseline.sql, v2.4.6 (same schema, different header).
+  "baseline-schema:405083572bb108bfe42e726b6f3d0b47c8d76c8e1de1a3a5d5fbbbe88a7ac4dd",
+];
+
+/** Accepted markers for the current baseline schema, current form first. */
+export function acceptedBaselineSchemaMarkers(source: string): readonly string[] {
+  return [baselineSchemaMarker(source), ...LEGACY_BASELINE_SCHEMA_MARKERS];
+}
+
+/** Hash of the baseline's statements, with comments and formatting removed. */
+function hashStatements(source: string): string {
+  const statements = source
+    // `--` to end of line, which is the only comment form these baselines use.
+    .replace(/--[^\n]*/g, "")
+    .split(/--> statement-breakpoint/)
+    .map((part) => part.replace(/\s+/g, " ").trim())
+    .filter((part) => part.length > 0)
+    .join("\n");
+  return createHash("sha256").update(statements).digest("hex");
+}
+
+/**
+ * Rejects databases whose schema was created by retired migration files or
+ * which have tables but no record of the current baseline.
+ */
+export function assertMigrationHistorySupported(
+  applied: ReadonlySet<string>,
+  expected: readonly string[],
+  hasTenants: boolean,
+  currentBaselineMarker?: string,
+): void {
+  const supported = new Set(expected);
+  const retired = [...applied].filter((migrationId) => !supported.has(migrationId));
+  if (retired.length > 0) {
+    throw new Error(
+      `Database contains retired migration history (${retired.join(", ")}). ` +
+      "Create a fresh database and restore from a backup; in-place upgrades are no longer supported.",
+    );
+  }
+  if (hasTenants && !applied.has(BASELINE_MIGRATION_ID)) {
+    throw new Error(
+      "Database contains an unversioned schema. Create a fresh database and restore from a backup; " +
+      "in-place upgrades are no longer supported.",
+    );
+  }
+  // Any marker in `expected` counts, not just the current one: a database built
+  // by an older release carries a frozen historical marker for the same schema.
+  const acceptedMarker =
+    currentBaselineMarker !== undefined &&
+    expected.some((id) => id.startsWith(BASELINE_SCHEMA_MARKER_PREFIX) && applied.has(id));
+  if (hasTenants && currentBaselineMarker !== undefined && !acceptedMarker) {
+    throw new Error(
+      "Database was created from a different baseline schema. Create a fresh database and restore from a backup; " +
+      "in-place upgrades are no longer supported.",
+    );
+  }
+}
+
+/**
+ * Splits a migration file on the drizzle `statement-breakpoint` markers the
+ * files already carry. Only the embedded backend needs per-statement
+ * application (single-statement failure tolerance); the pg path sends the
+ * whole file at once exactly as before.
+ */
+export function splitStatements(source: string): string[] {
+  return source
+    .split(/--> statement-breakpoint/g)
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0);
+}
+
+/**
+ * Readiness view of the migration ledger: the numbered SQL files discovered on
+ * disk compared against the ids recorded in {@link MIGRATION_LEDGER_TABLE}.
+ */
+export interface MigrationLedgerStatus {
+  /** Migration file names discovered on disk, in apply order. */
+  readonly expected: readonly string[];
+  /** Expected migrations that have no ledger row. */
+  readonly pending: readonly string[];
+  /** True only when every expected migration is recorded in the ledger. */
+  readonly applied: boolean;
+}
+
+/**
+ * Compares the migration files discovered on disk against the applied-migration
+ * ledger. A reachable ledger alone is not readiness: an interrupted upgrade
+ * leaves the table present while later migrations are still unapplied.
+ *
+ * Runs on the drizzle handle, so it is identical on every backend.
+ */
+export async function readMigrationLedgerStatus(
+  db: CartethyiaDatabase,
+  folder: string = resolveMigrationsFolder(),
+): Promise<MigrationLedgerStatus> {
+  const expected = migrationFiles(folder).map((file) => migrationIdFor(file));
+  const result = await db.execute<{ migration_id: string }>(
+    sql.raw(`SELECT migration_id FROM ${MIGRATION_LEDGER_TABLE}`),
+  );
+  const applied = new Set(result.rows.map((row) => row.migration_id));
+  const pending = expected.filter((migrationId) => !applied.has(migrationId));
+  return { expected, pending, applied: pending.length === 0 };
+}