@sixb/pg 0.1.14 → 0.1.16

package/src/pg-projection-run-storage.ts

@@ -19,6 +19,7 @@ import {
   immutableDatasetVersionConflict,
   mergeProjectionRunProgress,
   type PersistedProjectionRunRecord,
+  parseProjectionAttemptFailure,
   planProjectionRunFinish,
   planProjectionRunReclaim,
   projectionRunNotFound,
@@ -26,12 +27,14 @@ import {
   requireTelemetryProjectionRun,
   restoreProjectionRun,
   type StoredProjectionRunRecord,
+  SUPERSEDING_PROJECTION_RUN_STATUSES,
   staleProjectionRunExecution,
 } from "@sixb/core/internal/projection-run-storage-provider"
 import type {
   AdvanceProjectionTelemetryCheckpointInput,
   ExecutionStorage,
   FailProjectionRunEnqueueInput,
+  FindSupersedingProjectionRunInput,
   FinishProjectionRunInput,
   ListLatestProjectionRunsInput,
   ListLatestProjectionRunsResult,
@@ -44,6 +47,7 @@ import type {
   ProjectionRunStatus,
   ProjectionRunStorage,
   QueueProjectionRunInput,
+  RecordProjectionAttemptFailureInput,
   RecordProjectionMissingTargetInput,
   StartOrReclaimProjectionRunInput,
   TelemetryProjectionRunRecord,
@@ -159,8 +163,7 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
       const [updated] = await tx<DatabaseRow[]>`
         UPDATE projection_runs
         SET status = ${"running"}, started_at = COALESCE(started_at, ${input.startedAt ?? new Date()}),
-          finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken},
-          error = ${null}
+          finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
         WHERE project_id = ${input.projectId} AND id = ${input.id}
           AND (
             (status = ${"queued"} AND execution_token IS NULL)
@@ -229,6 +232,26 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
     })
   }
 
+  async recordAttemptFailure(
+    input: RecordProjectionAttemptFailureInput
+  ): Promise<ProjectionRunRecord> {
+    return runPgTransaction(this.sql, async (tx) => {
+      const existingRow = await requireMaterializationExecution(tx, input)
+      const error = parseProjectionAttemptFailure(restoreProjectionRunRow(existingRow), input.error)
+      const [updated] = await tx<DatabaseRow[]>`
+        UPDATE projection_runs
+        SET error = ${serializeSixbFailure(error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
+        WHERE project_id = ${input.projectId}
+          AND id = ${input.id}
+          AND status = ${"running"}
+          AND execution_token = ${input.executionToken}
+        RETURNING *
+      `
+      if (!updated) throw staleProjectionRunExecution(input.id)
+      return rowToProjectionRunRecord(updated)
+    })
+  }
+
   async finish(input: FinishProjectionRunInput): Promise<ProjectionRunRecord> {
     return runPgTransaction(this.sql, async (tx) => {
       const existingRow = await requireMaterializationExecution(tx, input)
@@ -411,6 +434,36 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
     return { runs, hasMore: offset + runs.length < total, total }
   }
 
+  async findSupersedingRun(
+    input: FindSupersedingProjectionRunInput
+  ): Promise<ProjectionRunRecord | null> {
+    const [run] = await this.sql<
+      Pick<
+        DatabaseRow,
+        "projection_id" | "materialization_protocol" | "dataset_version_created_at"
+      >[]
+    >`
+      SELECT projection_id, materialization_protocol, dataset_version_created_at
+      FROM projection_runs
+      WHERE project_id = ${input.projectId} AND id = ${input.id}
+    `
+    if (!run) throw projectionRunNotFound(input.projectId, input.id)
+    if (run.materialization_protocol !== "replacement") return null
+    // Canonical UTC ISO timestamps order as text, but only under a binary collation: a
+    // linguistic one may weigh their punctuation differently.
+    const [newer] = await this.sql<DatabaseRow[]>`
+      SELECT * FROM projection_runs
+      WHERE project_id = ${input.projectId}
+        AND projection_id = ${run.projection_id}
+        AND materialization_protocol = ${"replacement"}
+        AND status = ANY(${this.sql.array([...SUPERSEDING_PROJECTION_RUN_STATUSES])}::text[])
+        AND dataset_version_created_at COLLATE "C" > ${run.dataset_version_created_at} COLLATE "C"
+      ORDER BY dataset_version_created_at COLLATE "C" DESC, id DESC
+      LIMIT 1
+    `
+    return newer ? rowToProjectionRunRecord(newer) : null
+  }
+
   async listLatestByProjectionIds(
     input: ListLatestProjectionRunsInput
   ): Promise<ListLatestProjectionRunsResult> {
@@ -421,6 +474,7 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
       ownerIds: input.projectionIds,
       projectId: input.projectId,
       ownerIdFor: (row) => row.projection_id,
+      excludedStatus: "superseded",
     })
 
     return { runs: rows.map(rowToProjectionRunRecord) }

package/src/storage-errors.ts

@@ -1,7 +1,9 @@
+import { createSixbError } from "@sixb/core/internal/errors"
+
 // Map thrown errors to their PostgreSQL/porsager meaning by code rather than by matching
 // message text. Database errors arrive in the native Postgres format with a SQLSTATE on
-// `.code`; connection-layer failures arrive with porsager's own codes or, for Node socket
-// errors, as an `AggregateError` of `ECONNREFUSED`/`ETIMEDOUT`/… entries.
+// `.code` and the server's `severity`; connection-layer failures arrive with porsager's own
+// codes or the socket's system error code.
 
 /** PostgreSQL SQLSTATE for a unique-constraint violation. */
 const UNIQUE_VIOLATION = "23505"
@@ -17,20 +19,52 @@ const FOREIGN_KEY_VIOLATION = "23503"
  */
 const RETRYABLE_TRANSACTION_CONFLICT_CODES = new Set(["40001", "40P01"])
 
-const CONNECTION_ERROR_CODES = new Set([
-  // porsager-specific
-  "CONNECT_TIMEOUT",
+/**
+ * Codes that mean the connection itself is gone, not that a statement failed: porsager's own codes
+ * for a connection that closed under a query or that the pool destroyed, and the socket errors a
+ * live connection can die with. Connect-time failures (`ECONNREFUSED`, `CONNECT_TIMEOUT`, ...) are
+ * left out on purpose: they reject before any session exists, so there is nothing to give up.
+ */
+const CONNECTION_LOST_CODES = new Set([
   "CONNECTION_CLOSED",
   "CONNECTION_DESTROYED",
-  "CONNECTION_ENDED",
-  // Node socket errors
-  "ECONNREFUSED",
   "ECONNRESET",
+  "EPIPE",
   "ETIMEDOUT",
+])
+
+/**
+ * Codes of a connection that could not be opened: refused, unresolvable, unreachable, or not
+ * answered in time. A server that accepts the socket and then refuses the session (too many
+ * connections, starting up) answers with a `FATAL` error, which `isConnectionLost` covers.
+ */
+const CONNECT_FAILED_CODES = new Set([
+  "CONNECT_TIMEOUT",
+  "ECONNREFUSED",
   "ENOTFOUND",
   "EAI_AGAIN",
+  "EHOSTUNREACH",
+  "ENETUNREACH",
 ])
 
+/**
+ * The shared lock tables PostgreSQL can fill, keyed by the setting its hint names. Both raise
+ * SQLSTATE 53200 (`out_of_memory`) with the message "out of shared memory", which reads like the
+ * server ran out of RAM; the hint is the only part that says which table filled up.
+ */
+const LOCK_TABLES = [
+  {
+    setting: "max_pred_locks_per_transaction",
+    table: "predicate lock table",
+    note:
+      " A long serializable transaction also keeps the predicate locks of every serializable" +
+      " transaction that commits while it is open.",
+  },
+  { setting: "max_locks_per_transaction", table: "lock table", note: "" },
+] as const
+
+const OUT_OF_MEMORY = "53200"
+
 /** The SQLSTATE / system error code carried by a thrown database or connection error. */
 export function pgErrorCode(error: unknown): string | undefined {
   if (error instanceof Error) {
@@ -63,16 +97,68 @@ export function isRetryableTransactionConflict(error: unknown): boolean {
 }
 
 /**
- * True when the error originates from the connection layer (porsager connection codes, or a
- * Node socket failure surfaced as an `AggregateError`) rather than from a SQL statement.
+ * True when the error means the session is gone: PostgreSQL has already rolled back its open
+ * transaction and released its locks, and nothing more may be sent on that connection.
+ *
+ * PostgreSQL ends the session after every `FATAL` or `PANIC` error — `pg_terminate_backend` (57P01),
+ * a server shutdown (57P02), `idle_in_transaction_session_timeout` (25P03) — so the severity decides
+ * rather than a list of SQLSTATEs.
  */
-export function isConnectionError(error: unknown): boolean {
+export function isConnectionLost(error: unknown): boolean {
   const code = pgErrorCode(error)
-  if (code !== undefined && CONNECTION_ERROR_CODES.has(code)) {
+  if (code !== undefined && CONNECTION_LOST_CODES.has(code)) {
     return true
   }
-  if (error instanceof AggregateError) {
-    return error.errors.some(isConnectionError)
+  const severity =
+    error instanceof Error ? (error as { readonly severity?: unknown }).severity : null
+  return severity === "FATAL" || severity === "PANIC"
+}
+
+/**
+ * Replace an error that says something about the database rather than the statement: a lost or
+ * unopenable connection becomes the retryable `storage.unavailable`, a full lock table an error
+ * naming the table and the setting that sizes it. Every other error is returned unchanged.
+ */
+export function explainPgError(error: unknown): unknown {
+  if (isStorageUnavailable(error)) {
+    return storageUnavailable(error)
   }
-  return false
+  if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
+    return error
+  }
+  const hint = (error as { readonly hint?: unknown }).hint
+  const lockTable =
+    typeof hint === "string" ? LOCK_TABLES.find(({ setting }) => hint.includes(setting)) : undefined
+  if (!lockTable) {
+    return error
+  }
+
+  const { setting, table, note } = lockTable
+  return new Error(
+    `[SixbPg] PostgreSQL aborted the transaction because its shared ${table} is full ` +
+      `(SQLSTATE 53200 "${error.message}"). The table holds ${setting} × (max_connections + ` +
+      `max_prepared_transactions) entries shared by all sessions.${note} Raise ${setting} ` +
+      "(applied at server restart) or shorten the transactions that overlap.",
+    { cause: error }
+  )
+}
+
+function isStorageUnavailable(error: unknown): boolean {
+  const code = pgErrorCode(error)
+  return isConnectionLost(error) || (code !== undefined && CONNECT_FAILED_CODES.has(code))
+}
+
+/**
+ * Whether a lost connection's transaction committed is unknown when the loss interrupted its
+ * COMMIT, so the message does not say it rolled back. Retrying is still safe for Sixb's own
+ * writes: each is fenced or idempotent.
+ */
+function storageUnavailable(error: unknown): Error {
+  const code = pgErrorCode(error)
+  const reason = error instanceof Error ? error.message : String(error)
+  return createSixbError(
+    "storage.unavailable",
+    `[SixbPg] PostgreSQL is unavailable: the connection was lost or could not be opened (${code ?? "no code"}: ${reason}).`,
+    { cause: error, ...(code === undefined ? {} : { details: { code } }) }
+  )
 }

package/src/transactions.ts

@@ -1,5 +1,22 @@
 import { createHash } from "node:crypto"
-import type { SQL, SQLClient } from "./pg-client"
+import type { ReservedSQL, SQL, SQLClient } from "./pg-client"
+import { explainPgError, isConnectionLost } from "./storage-errors"
+
+// Why transactions are driven here instead of through porsager's `sql.begin`:
+//
+// postgres.js 3.4.9 writes to a connection without checking that its socket is still open. When
+// the server or the network drops a connection during a transaction, `sql.begin` answers the
+// failed statement by sending ROLLBACK on it; the write is deferred to `setImmediate` and throws
+// `null is not an object (evaluating 'socket.write')` there, outside any promise, which kills the
+// process. `reserve().release()` has the matching flaw: it hands the connection back to the pool
+// without looking at it, and the next query sent on it crashes the same way.
+//
+// Holding a reserved connection and sending BEGIN/COMMIT/ROLLBACK ourselves lets us act on the
+// loss first. PostgreSQL has already rolled the transaction back and ended the session, so nothing
+// more is sent on that connection and it is never released: the pool reconnects a closed
+// connection on its own. The upstream fix is porsager/postgres#1209; until a release carries it,
+// a connection that drops *between* two statements still crashes, because nothing reports the loss
+// before the next statement's deferred write.
 
 export type PgStoreClient = SQL | SQLClient
 
@@ -10,6 +27,16 @@ export interface RunPgTransactionOptions {
 
 type PgTransactionIsolation = "unverifiedDefault" | "repeatableRead" | "serializable"
 
+// The isolation level is folded into `BEGIN` rather than a separate `SET TRANSACTION`: one
+// round-trip instead of two, and the level takes effect before any statement of the transaction.
+const BEGIN_STATEMENTS: Readonly<Record<PgTransactionIsolation, string>> = {
+  unverifiedDefault: "BEGIN",
+  repeatableRead: "BEGIN ISOLATION LEVEL REPEATABLE READ",
+  serializable: "BEGIN ISOLATION LEVEL SERIALIZABLE",
+}
+
+/** Every client handed to a transaction callback; a reserved connection can never start one. */
+const pgTransactionClients = new WeakSet<PgStoreClient>()
 const activePgTransactions = new WeakMap<PgStoreClient, PgTransactionIsolation>()
 
 export async function runPgTransaction<T>(
@@ -21,30 +48,93 @@ export async function runPgTransaction<T>(
     return run(sql)
   }
 
-  // porsager passes the callback a transaction-scoped client (a TransactionSql, which is an
-  // ISql == SQLClient). The `as Promise<T>` only unwraps porsager's UnwrapPromiseArray return
-  // type (our callbacks never return the pipelined-array form), not a structural cast.
-  //
-  // The isolation level is folded into `BEGIN` (`BEGIN ISOLATION LEVEL SERIALIZABLE`) rather than
-  // a separate `SET TRANSACTION` statement: one round-trip instead of two, and the level is
-  // guaranteed to take effect before any data statement of the transaction runs.
   const isolation = options.isolation ?? "unverifiedDefault"
-  const trackedRun = async (tx: SQLClient): Promise<T> => {
-    activePgTransactions.set(tx, isolation)
-    try {
-      return await run(tx)
-    } finally {
-      activePgTransactions.delete(tx)
+  try {
+    return await withReservedPgConnection(sql, (tx) => {
+      pgTransactionClients.add(tx)
+      return runPgTransactionOn(
+        tx,
+        async () => {
+          activePgTransactions.set(tx, isolation)
+          try {
+            return await run(tx)
+          } finally {
+            activePgTransactions.delete(tx)
+          }
+        },
+        isolation
+      )
+    })
+  } catch (error) {
+    throw explainPgError(error)
+  }
+}
+
+/**
+ * Hold one pool connection for `run`, and hand it back to the pool only while it is still usable.
+ * A connection whose session is gone stays out of the pool, which replaces it on its own.
+ */
+export async function withReservedPgConnection<T>(
+  sql: SQL,
+  run: (connection: ReservedSQL) => Promise<T>
+): Promise<T> {
+  const connection = await sql.reserve()
+  let usable = true
+  try {
+    return await run(connection)
+  } catch (error) {
+    usable = !isConnectionLost(error)
+    throw error
+  } finally {
+    if (usable) {
+      connection.release()
     }
   }
+}
 
-  if (isolation === "serializable") {
-    return sql.begin("isolation level serializable", trackedRun) as Promise<T>
+/**
+ * Run `run` inside one transaction on a reserved connection.
+ *
+ * PostgreSQL answers COMMIT with a `ROLLBACK` tag, not an error, when a statement of the
+ * transaction failed and the callback caught that error and carried on. That outcome is surfaced
+ * as an error: returning normally would report writes that were discarded.
+ */
+export async function runPgTransactionOn<T>(
+  connection: ReservedSQL,
+  run: () => Promise<T>,
+  isolation: PgTransactionIsolation = "unverifiedDefault"
+): Promise<T> {
+  await connection.unsafe(BEGIN_STATEMENTS[isolation])
+  const result = await undoOnFailure(run, () => connection.unsafe("ROLLBACK"))
+  const commit = await connection.unsafe("COMMIT")
+  if (commit.command !== "COMMIT") {
+    throw new Error(
+      "[SixbPg] PostgreSQL rolled the transaction back instead of committing it: a statement in it" +
+        " failed and its error was caught without being rethrown. A failed statement aborts the" +
+        " whole transaction, so let its error propagate out of the transaction callback."
+    )
   }
-  if (isolation === "repeatableRead") {
-    return sql.begin("isolation level repeatable read", trackedRun) as Promise<T>
+  return result
+}
+
+/**
+ * Run `run`; when it throws, first undo the session state it left on its connection (an open
+ * transaction, a session lock), then rethrow. Nothing is undone when the connection itself was lost:
+ * the server has already discarded that state, and anything more sent on it would crash
+ * postgres.js (see the top of this file).
+ */
+export async function undoOnFailure<T>(
+  run: () => Promise<T>,
+  undo: () => Promise<unknown>
+): Promise<T> {
+  try {
+    return await run()
+  } catch (error) {
+    if (!isConnectionLost(error)) {
+      await undo()
+    }
+    throw error
   }
-  return sql.begin(trackedRun) as Promise<T>
 }
 
 /**
@@ -95,5 +185,8 @@ function advisoryLockParts(key: string): readonly [number, number] {
 }
 
 function canStartPgTransaction(sql: PgStoreClient): sql is SQL {
-  return typeof (sql as { readonly begin?: unknown }).begin === "function"
+  return (
+    !pgTransactionClients.has(sql) &&
+    typeof (sql as { readonly reserve?: unknown }).reserve === "function"
+  )
 }