@everystack/server 0.4.15 → 0.4.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/server",
-  "version": "0.4.15",
+  "version": "0.4.16",
   "description": "Server runtime primitives for Lambda — event adapters, routing, SSR, image processing",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -164,7 +164,7 @@
     "ts-jest": "29.4.9",
     "typescript": "5.9.3",
     "@everystack/auth": "0.4.0",
-    "@everystack/cli": "0.4.35"
+    "@everystack/cli": "0.4.46"
   },
   "scripts": {
     "test": "jest",

package/src/db-session.ts

@@ -0,0 +1,252 @@
+/**
+ * db:session — N statements, ONE connection, ONE transaction, N result sets.
+ *
+ * The problem it exists to solve: the ops Lambda holds one database connection per
+ * container and containers are ambient. `db:query` is one statement per invoke, so a
+ * caller that needs several statements to describe ONE moment (a catalog
+ * introspection, say) gets them answered by several containers on several
+ * connections. Any session state one statement sets is invisible to the next, and
+ * the assembled answer can be a collage of two renderings of one database. That is
+ * not a tuning problem; it is the absence of a session.
+ *
+ * So the session is the primitive. The caller sends its statements together, the
+ * Lambda runs them on one connection inside one transaction, and sends the result
+ * sets back in input order. The SQL stays in the caller (no version skew between a
+ * deployed server and the CLI that asks it questions), and the policy — read-only,
+ * isolation level, search_path — is the caller's too. This module hardcodes none of it.
+ *
+ * Three properties are load bearing:
+ *
+ * - **`searchPath` emits `SET LOCAL`, never a bare `SET`.** A bare `SET` on a pooled,
+ *   persistent connection outlives its command and pins whichever connection answered.
+ *   `SET LOCAL` reverts with the transaction, so reversion is structural rather than a
+ *   compensating statement that may not run. The pin consumes NO result slot — the
+ *   caller indexes results by statement, and a hidden extra slot would shift every one.
+ * - **Per-statement `allowFailure`** wraps a statement in a savepoint, so "this catalog
+ *   table may not exist yet" (`derived_provenance`, `backfill_log`, `schema_log`) costs
+ *   the one slot rather than the whole transaction. Same semantics as psql's
+ *   `ON_ERROR_ROLLBACK`. A statement NOT flagged is fatal: it aborts the transaction and
+ *   the whole session fails, because a partial answer to a consistency question is worse
+ *   than no answer. The savepoint is taken through the DRIVER's nested transaction, not
+ *   by issuing `SAVEPOINT` as SQL: postgres.js records any query error inside a
+ *   transaction scope and re-throws it after the callback returns, so a hand-rolled
+ *   `ROLLBACK TO SAVEPOINT` recovers the database and still loses the session. Only a
+ *   nested scope gets its own error bookkeeping. (Proven on a live PostgreSQL — the
+ *   hand-rolled version passed against a fake and failed against the driver.)
+ * - **The response is gzipped and size-checked.** Lambda's invoke response limit is 6 MB
+ *   and catalog JSON compresses 10-20x, so gzip alone carries any real schema. When it
+ *   does not, this HARD ERRORS with the size — it never truncates, and it never returns
+ *   a partial read that a fingerprint would silently be computed from.
+ */
+
+import { gzipSync } from 'node:zlib';
+
+/** One statement in a session. A bare string is shorthand for `{ sql, allowFailure: false }`. */
+export interface DbSessionStatement {
+  sql: string;
+  /** Survive a failure of THIS statement (savepoint), leaving the other slots intact. */
+  allowFailure?: boolean;
+}
+
+export interface DbSessionPayload {
+  statements?: Array<DbSessionStatement | string>;
+  /** `SET TRANSACTION READ ONLY` — the caller's policy, never assumed here. */
+  readOnly?: boolean;
+  /** Transaction isolation level, e.g. 'repeatable read' for a single-snapshot read. */
+  isolation?: string;
+  /** Pinned via `SET LOCAL search_path`; consumes no result slot. */
+  searchPath?: string;
+}
+
+/** A statement that failed under `allowFailure`. Occupies its slot; carries the cause. */
+export interface DbSessionStatementError {
+  everystackSessionError: true;
+  message: string;
+}
+
+/** One slot: the statement's rows, or the marker for an allowed failure. */
+export type DbSessionResult = any[] | DbSessionStatementError;
+
+/**
+ * Compressed-response ceiling. Lambda's synchronous invoke response limit is 6 MB;
+ * this leaves headroom for the JSON envelope and base64 expansion of the gzip payload.
+ */
+export const SESSION_MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
+
+/** The isolation levels PostgreSQL accepts. Anything else is refused, never coerced. */
+const ISOLATION_LEVELS = new Set([
+  'read uncommitted',
+  'read committed',
+  'repeatable read',
+  'serializable',
+]);
+
+/** A bare SQL identifier — the only shape a search_path element may take. */
+const PLAIN_IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/;
+
+/**
+ * Build the `SET LOCAL search_path` statement for a caller-supplied path.
+ *
+ * The path arrives as a string over an IAM-gated invoke and is interpolated into SQL,
+ * so it is validated as a comma-separated list of plain identifiers and rejected
+ * otherwise. `$user` is deliberately NOT accepted: this pin exists to make deparsed
+ * expressions canonical, and a role-dependent element would make the capture depend on
+ * who read it.
+ */
+export function buildSearchPathSql(searchPath: string): string {
+  const parts = searchPath.split(',').map((p) => p.trim()).filter((p) => p.length > 0);
+  if (parts.length === 0) {
+    throw new Error('searchPath must name at least one schema');
+  }
+  for (const part of parts) {
+    if (!PLAIN_IDENT.test(part)) {
+      throw new Error(
+        `searchPath element ${JSON.stringify(part)} is not a plain identifier — db:session pins only bare schema names`,
+      );
+    }
+  }
+  return `SET LOCAL search_path = ${parts.map((p) => `"${p}"`).join(', ')}`;
+}
+
+/** Normalize the payload's statements, or explain what is wrong with them. */
+export function normalizeStatements(
+  statements: Array<DbSessionStatement | string> | undefined,
+): DbSessionStatement[] {
+  if (!Array.isArray(statements) || statements.length === 0) {
+    throw new Error('db:session requires a non-empty { statements } array');
+  }
+  return statements.map((s, i) => {
+    const stmt = typeof s === 'string' ? { sql: s } : s;
+    if (!stmt || typeof stmt.sql !== 'string' || stmt.sql.trim() === '') {
+      throw new Error(`db:session statement ${i} has no sql`);
+    }
+    return { sql: stmt.sql, allowFailure: stmt.allowFailure === true };
+  });
+}
+
+/** The refusal when the compressed result set will not fit in an invoke response. */
+export function oversizeResponseError(bytes: number, statements: number): string {
+  const mb = (bytes / (1024 * 1024)).toFixed(1);
+  const capMb = (SESSION_MAX_RESPONSE_BYTES / (1024 * 1024)).toFixed(0);
+  return [
+    `db:session response too large: ${mb} MB compressed from ${statements} statements exceeds the ${capMb} MB invoke-response ceiling.`,
+    'Nothing was truncated and nothing partial was returned — a partial read is not a state anything may be computed from.',
+    'Split the read into more sessions, or narrow it (fewer schemas, fewer statements).',
+  ].join(' ');
+}
+
+/**
+ * The transaction handle: statements, plus a NESTED transaction — which drizzle maps to
+ * the driver's savepoint, the only mechanism that survives a failed statement.
+ */
+export interface SessionTx {
+  execute: (query: any) => Promise<any>;
+  transaction: <T>(fn: (sp: SessionTx) => Promise<T>) => Promise<T>;
+}
+
+/** The minimal database surface this needs: drizzle's `transaction`, nothing more. */
+export interface SessionCapableDb {
+  transaction: <T>(
+    fn: (tx: SessionTx) => Promise<T>,
+    config?: { isolationLevel?: any; accessMode?: any },
+  ) => Promise<T>;
+}
+
+const rowsOf = (result: any): any[] =>
+  Array.isArray(result) ? result : result?.rows ?? [];
+
+/**
+ * Run N statements as ONE session and return the result sets, in input order.
+ *
+ * The core, shared by the `db:session` action (which gzips this) and by the in-Lambda
+ * verbs that need a consistent multi-statement read on the operator connection —
+ * `db:apply`'s fingerprint endpoints and `db:reconcile`'s catalog read. One
+ * implementation, so the wire path and the in-process path cannot mean different things
+ * by "one session".
+ */
+export async function runSessionStatements(
+  db: SessionCapableDb,
+  statements: Array<DbSessionStatement | string> | undefined,
+  opts: Omit<DbSessionPayload, 'statements'> | undefined,
+  raw: (query: string) => any,
+): Promise<DbSessionResult[]> {
+  const stmts = normalizeStatements(statements);
+  const searchPathSql =
+    opts?.searchPath !== undefined ? buildSearchPathSql(String(opts.searchPath)) : undefined;
+
+  let isolationLevel: string | undefined;
+  if (opts?.isolation !== undefined) {
+    const level = String(opts.isolation).toLowerCase().trim();
+    if (!ISOLATION_LEVELS.has(level)) {
+      throw new Error(
+        `db:session isolation ${JSON.stringify(opts.isolation)} is not a PostgreSQL isolation level`,
+      );
+    }
+    isolationLevel = level;
+  }
+
+  // An EMPTY config is not the same as no config: drizzle treats any object as a request
+  // to configure the transaction and emits a chunk-less `SET TRANSACTION`, which is a
+  // syntax error. Declare nothing by passing nothing.
+  const txConfig = {
+    ...(isolationLevel ? { isolationLevel } : {}),
+    ...(opts?.readOnly ? { accessMode: 'read only' } : {}),
+  };
+  const hasConfig = Object.keys(txConfig).length > 0;
+
+  const results: DbSessionResult[] = [];
+  await db.transaction(
+    async (tx) => {
+      // The pin runs FIRST and its result is discarded — callers index by statement.
+      if (searchPathSql) await tx.execute(raw(searchPathSql));
+
+      for (const stmt of stmts) {
+        if (!stmt.allowFailure) {
+          // Fatal by default: a failure here aborts the transaction and the session.
+          results.push(rowsOf(await tx.execute(raw(stmt.sql))));
+          continue;
+        }
+        try {
+          // A nested transaction IS the savepoint, taken by the driver so its own error
+          // bookkeeping is scoped to it. Rolling back by hand would recover the database
+          // and still lose the session.
+          results.push(
+            await tx.transaction(async (sp) => rowsOf(await sp.execute(raw(stmt.sql)))),
+          );
+        } catch (err: any) {
+          results.push({
+            everystackSessionError: true,
+            message: err?.message || String(err),
+          });
+        }
+      }
+    },
+    hasConfig ? txConfig : undefined,
+  );
+  return results;
+}
+
+/**
+ * Run a session and return the gzipped result sets (or a typed error object).
+ *
+ * The return value is the action's response verbatim: `{ gzip }` on success,
+ * `{ error }` on any refusal or unrecovered failure. Never both.
+ */
+export async function runDbSession(
+  db: SessionCapableDb,
+  payload: DbSessionPayload | undefined,
+  raw: (query: string) => any,
+): Promise<{ gzip: string } | { error: string }> {
+  let results: DbSessionResult[];
+  try {
+    results = await runSessionStatements(db, payload?.statements, payload, raw);
+  } catch (err: any) {
+    return { error: err?.message || String(err) };
+  }
+
+  const gzip = gzipSync(Buffer.from(JSON.stringify(results), 'utf8')).toString('base64');
+  if (gzip.length > SESSION_MAX_RESPONSE_BYTES) {
+    return { error: oversizeResponseError(gzip.length, results.length) };
+  }
+  return { gzip };
+}

package/src/plugin.ts

@@ -706,6 +706,29 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       return { rows };
     };
 
+    // A SessionRunner over THIS container's operator connection, for the in-Lambda verbs
+    // (db:apply's fingerprint endpoints, db:reconcile's catalog read). Same core the
+    // db:session action serves over the wire, so both mean the same thing by "one session".
+    const sessionOverOpsDb = (sql: { raw: (q: string) => any }) =>
+      async (statements: any, opts?: any) => {
+        const { runSessionStatements } = await import('./db-session');
+        return runSessionStatements(opsDb as any, statements, opts, (q: string) => sql.raw(q));
+      };
+
+    // --- db:session — N statements, ONE connection, ONE transaction ---
+    // The primitive that makes a multi-statement remote read describe ONE moment.
+    // `db:query` is one statement per invoke, so a five-query introspection can be
+    // answered by five containers holding five connections, and any session state one
+    // statement sets is invisible to the next. This runs the caller's statements
+    // together on the single ops connection inside one transaction and returns the
+    // result sets in input order, gzipped. Policy (read-only, isolation, search_path)
+    // is the caller's; the SQL is the caller's. See db-session.ts.
+    actions['db:session'] = async (payload, _ctx) => {
+      const { sql } = await import('drizzle-orm');
+      const { runDbSession } = await import('./db-session');
+      return runDbSession(opsDb as any, payload as any, (query: string) => sql.raw(query));
+    };
+
     // --- db:doctor — "is my database actually secure?" ---
     // Probes the api connection (createDb / DATABASE_URL) and the operator connection
     // (opsDb / ADMIN_DATABASE_URL) from inside the VPC and reports whether the api path
@@ -852,6 +875,11 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
         const result: any = await opsDb.execute(sql.raw(query));
         return Array.isArray(result) ? result : result?.rows ?? [];
       };
+      // The fingerprint endpoints this apply gates on are multi-statement reads, so they
+      // run as ONE session on this container's operator connection. Server-side that was
+      // always one connection; saying so in the type is what stops a future caller from
+      // quietly reintroducing a per-statement read.
+      const session = sessionOverOpsDb(sql);
 
       const destructive = Number(plan.destructive) > 0;
       const snapStage = stage ?? ctx.environment ?? 'default';
@@ -860,7 +888,7 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       const SNAPSHOT_FRESHNESS_MS = 60 * 60_000;
       let snapshotId: string | undefined;
       try {
-        const result = await executeApplyPlan(runner, plan, {
+        const result = await executeApplyPlan(runner, session, plan, {
           actor: actor ?? null,
           gitRef: gitRef ?? null,
           // Descent/authority are decided by the CLI (checkout + caller identity)
@@ -930,7 +958,7 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
           const result: any = await opsDb.execute(sql.raw(query));
           return Array.isArray(result) ? result : result?.rows ?? [];
         };
-        const run = await executeReconcile(runner, {
+        const run = await executeReconcile(runner, sessionOverOpsDb(sql), {
           declared: p.declared,
           renamedTables: p.renamedTables,
           apply: p.apply,