@owlmeans/postgres 0.1.15

package/src/bootstrap.ts

@@ -0,0 +1,126 @@
+import {
+  assertSqlIdentifier, PostgresBootstrapError, quoteIdent, quoteLiteral
+} from '@owlmeans/postgres-resource'
+import { Pool } from 'pg'
+import type { PoolClient } from 'pg'
+
+import type { BootstrapOptions, BootstrapReport, PostgresService } from './types.js'
+import { prepareConfig } from './utils/config.js'
+
+const exists = async (client: PoolClient, text: string, value: string): Promise<boolean> => {
+  const result = await client.query(text, [value])
+
+  return result.rowCount != null && result.rowCount > 0
+}
+
+/**
+ * Provision a least privileged role, its database and its schema.
+ *
+ * This is the one copy of a script every OwlMeans deployment used to carry inline. It runs
+ * on every start and every rebuild, so each step probes before it acts — "already exists"
+ * is the expected outcome, not an error.
+ *
+ * Two connections are unavoidable. `CREATE DATABASE` cannot run from inside the database
+ * it creates, so roles and databases are made over the maintenance connection the admin
+ * config points at, and the grants are applied over a short lived connection to the target.
+ *
+ * @throws {PostgresBootstrapError}
+ */
+export const bootstrapDb = async (
+  service: PostgresService, configAlias: string, opts: BootstrapOptions
+): Promise<BootstrapReport> => {
+  const role = assertSqlIdentifier(opts.role, 'role')
+  const database = assertSqlIdentifier(opts.database ?? opts.role, 'database')
+  const schema = opts.schema != null ? assertSqlIdentifier(opts.schema, 'schema') : null
+  const leastPrivilege = opts.leastPrivilege ?? true
+  const rotate = opts.rotatePassword ?? true
+
+  if (opts.password === '') {
+    throw new PostgresBootstrapError(`empty-password:${role}`)
+  }
+
+  const report: BootstrapReport = {
+    roleCreated: false,
+    passwordRotated: false,
+    databaseCreated: false,
+    schemaCreated: false,
+    grantsApplied: false
+  }
+
+  const config = service.config(configAlias)
+  const admin = await service.client(configAlias)
+
+  /**
+   * The password is the only value here that can't be bound: `CREATE ROLE` takes no
+   * parameters, so it reaches the server inside the statement text. Identifiers have
+   * already been asserted safe; the literal is escaped.
+   */
+  const secret = quoteLiteral(opts.password)
+
+  const client = await admin.connect()
+  try {
+    if (await exists(client, 'SELECT 1 FROM pg_roles WHERE rolname = $1', role)) {
+      if (rotate) {
+        await client.query(`ALTER ROLE ${quoteIdent(role)} WITH LOGIN PASSWORD ${secret}`)
+        report.passwordRotated = true
+      }
+    } else {
+      await client.query(`CREATE ROLE ${quoteIdent(role)} WITH LOGIN PASSWORD ${secret}`)
+      report.roleCreated = true
+    }
+
+    if (!await exists(client, 'SELECT 1 FROM pg_database WHERE datname = $1', database)) {
+      /** Never inside a transaction — Postgres refuses `CREATE DATABASE` in one. */
+      await client.query(`CREATE DATABASE ${quoteIdent(database)} OWNER ${quoteIdent(role)}`)
+      report.databaseCreated = true
+    }
+  } catch (error) {
+    throw wrap(error, `${role}@${database}`)
+  } finally {
+    client.release()
+  }
+
+  const target = new Pool(prepareConfig(config, { database, max: 1 }))
+  try {
+    if (schema != null) {
+      /**
+       * The resource layer creates its own schema too, but only once an application with
+       * resources boots. Creating it here — owned by the role — is what lets that
+       * application connect as a role with no `CREATE` right on the database at all.
+       */
+      await target.query(
+        `CREATE SCHEMA IF NOT EXISTS ${quoteIdent(schema)} AUTHORIZATION ${quoteIdent(role)}`
+      )
+      await target.query(`GRANT ALL ON SCHEMA ${quoteIdent(schema)} TO ${quoteIdent(role)}`)
+      await target.query(`ALTER ROLE ${quoteIdent(role)} SET search_path = ${quoteIdent(schema)}`)
+      report.schemaCreated = true
+    }
+
+    if (leastPrivilege) {
+      await target.query(`REVOKE CONNECT ON DATABASE ${quoteIdent(database)} FROM PUBLIC`)
+      await target.query('REVOKE CREATE ON SCHEMA public FROM PUBLIC')
+      await target.query(`GRANT CONNECT ON DATABASE ${quoteIdent(database)} TO ${quoteIdent(role)}`)
+      report.grantsApplied = true
+    }
+  } catch (error) {
+    throw wrap(error, `${role}@${database}`)
+  } finally {
+    await target.end().catch(() => undefined)
+  }
+
+  console.log(
+    `@owlmeans/postgres: bootstrap ${role}@${database} —`
+    + ` role ${report.roleCreated ? 'created' : report.passwordRotated ? 'rotated' : 'present'},`
+    + ` database ${report.databaseCreated ? 'created' : 'present'}`
+    + (schema != null ? `, schema ${schema} ready` : '')
+  )
+
+  return report
+}
+
+const wrap = (error: unknown, subject: string): Error => {
+  const failure = new PostgresBootstrapError(`${subject}:${(error as Error)?.message ?? error}`)
+  failure.cause = error
+
+  return failure
+}

package/src/consts.ts

@@ -0,0 +1,36 @@
+import { DEFAULT_DB_ALIAS } from '@owlmeans/postgres-resource'
+
+export const DEFAULT_ALIAS = DEFAULT_DB_ALIAS
+
+/**
+ * Config alias conventionally holding the superuser connection the admin path needs.
+ * Never the same as {@link DEFAULT_ALIAS} — the application connects as a least
+ * privileged role and must not carry superuser credentials in the same config entry.
+ */
+export const DEF_ADMIN_ALIAS = 'pg-admin'
+
+/** The database `CREATE DATABASE` has to be issued from, since it can't create the one it's in. */
+export const DEF_MAINTENANCE_DB = 'postgres'
+
+export const DEF_PORT = 5432
+
+/**
+ * `SELECT 1` readiness probe, generalized from the boot loop every OwlMeans deployment
+ * hand-rolled: a Postgres sidecar routinely accepts TCP before it accepts queries.
+ */
+export const DEF_RETRIES = 30
+export const DEF_RETRY_DELAY = 2000
+
+/**
+ * Postgres caps connections cluster wide (`max_connections`, 100 by default), so the
+ * per-process pool stays deliberately small — unlike a Mongo driver pool, an oversized
+ * one here starves every other client of the same server.
+ */
+export const DEF_POOL_SIZE = 10
+
+/**
+ * Failures the readiness probe treats as final rather than as "not up yet":
+ * `28P01`/`28000` wrong credentials, `3D000` no such database, `42501` no rights.
+ * Retrying any of them just delays the same error by a minute.
+ */
+export const TERMINAL_CONNECT_CODES = ['28P01', '28000', '3D000', '42501']

package/src/index.ts

@@ -0,0 +1,7 @@
+export type * from './types.js'
+export * from './consts.js'
+export * from './bootstrap.js'
+export * from './middleware.js'
+export * from './service.js'
+export * from './utils/config.js'
+export * from './utils/connection.js'

package/src/middleware.ts

@@ -0,0 +1,26 @@
+import { MiddlewareStage, MiddlewareType } from '@owlmeans/context'
+import type { Middleware } from '@owlmeans/context'
+
+import { DEFAULT_ALIAS } from './consts.js'
+import type { PostgresService } from './types.js'
+
+/**
+ * Run the work resources held back until every one of them had initialized.
+ *
+ * Foreign keys are the reason this exists: a key points at a table another resource owns,
+ * and resource initialization order is registration order, not dependency order. The
+ * Loading stage of the Context middleware type runs immediately after the last resource's
+ * `init()` — the first moment at which every table is known to exist.
+ */
+export const drainMiddleware = (alias: string = DEFAULT_ALIAS): Middleware => ({
+  type: MiddlewareType.Context,
+  stage: MiddlewareStage.Loading,
+
+  apply: async context => {
+    if (!context.hasService(alias)) {
+      return
+    }
+
+    await context.service<PostgresService>(alias).drain()
+  }
+})

package/src/service.ts

@@ -0,0 +1,287 @@
+import { makeKeyPairModel } from '@owlmeans/basic-keys'
+import { assertContext, Layer } from '@owlmeans/context'
+import type { BasicContext } from '@owlmeans/context'
+import {
+  makeTx, pgErrorToResourceError, pgIdentifier, refOf, resolvePlaceholders
+} from '@owlmeans/postgres-resource'
+import type { PostgresDb, PostgresMeta } from '@owlmeans/postgres-resource'
+import { createDbService } from '@owlmeans/resource'
+import type { ServerConfig, ServerContext } from '@owlmeans/server-context'
+import { drizzle } from 'drizzle-orm/node-postgres'
+import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
+import { Pool } from 'pg'
+import type { QueryResultRow } from 'pg'
+
+import { DEFAULT_ALIAS } from './consts.js'
+import { bootstrapDb } from './bootstrap.js'
+import { drainMiddleware } from './middleware.js'
+import type { PostgresService } from './types.js'
+import { poolDatabase, prepareConfig } from './utils/config.js'
+import { ensureSchema, probe } from './utils/connection.js'
+
+type Config = ServerConfig
+interface Context<C extends Config = Config> extends ServerContext<C> { }
+
+export const makePostgresDbService = (alias: string = DEFAULT_ALIAS): PostgresService => {
+  const location = `postgres:${alias}`
+
+  /**
+   * Handles are cached per config alias rather than rebuilt per call: `db()` runs on every
+   * single resource operation, and both the Drizzle binding and the schema name are fixed
+   * for the lifetime of a service instance — a layer switch produces a *new* instance.
+   */
+  const handles: Map<string, PostgresDb> = new Map()
+  const deferred: Map<string, Array<() => Promise<void>>> = new Map()
+
+  const service: PostgresService = createDbService<PostgresDb, Pool, PostgresService>(alias, {
+    db: async configAlias => {
+      const pool = await service.client(configAlias)
+      configAlias = service.ensureConfigAlias(configAlias)
+
+      let handle = handles.get(configAlias)
+      if (handle == null) {
+        handle = {
+          drizzle: drizzle(pool) as NodePgDatabase<Record<string, never>>,
+          pool,
+          /**
+           * `name()` is the layer aware namespace — the Postgres SCHEMA here, where Mongo
+           * reads the same value as a database name. It can outgrow 63 bytes once a layer
+           * suffix is appended, which Postgres would truncate silently.
+           */
+          schema: pgIdentifier(service.name(configAlias)),
+          database: poolDatabase(pool)
+        }
+        handles.set(configAlias, handle)
+      }
+
+      return handle
+    },
+
+    initialize: async configAlias => {
+      configAlias = service.ensureConfigAlias(configAlias)
+      const config = service.config(configAlias)
+
+      if (service.clients[configAlias] != null) {
+        return
+      }
+
+      if (service.layers == null) {
+        service.layers = [Layer.Global]
+      }
+      /**
+       * Sensitivity *adds* a layer the service is willing to serve. The Mongo copy of this
+       * block tests `includes` where it means `!includes`, so it only ever appends a layer
+       * that is already there — a latent no-op, corrected here.
+       */
+      if (config.serviceSensitive === true && !service.layers.includes(Layer.Service)) {
+        service.layers.push(Layer.Service)
+      }
+      if (config.entitySensitive === true && !service.layers.includes(Layer.Entity)) {
+        service.layers.push(Layer.Entity)
+      }
+
+      const meta = (config.meta ?? {}) as PostgresMeta
+      const pool = new Pool(prepareConfig(config))
+
+      /**
+       * Not optional: node-postgres emits `error` on *idle* clients when the server closes
+       * a connection — a routine event behind a load balancer — and an unhandled `error`
+       * on an EventEmitter terminates the process.
+       */
+      pool.on('error', error => {
+        console.error(`${location}: idle client error — ${(error as Error).message}`)
+      })
+
+      try {
+        await probe(pool, meta, location)
+      } catch (error) {
+        await pool.end().catch(() => undefined)
+        throw error
+      }
+
+      process.on('SIGTERM', () => { void pool.end().catch(() => undefined) })
+
+      if (service.clients[configAlias] != null) {
+        await pool.end().catch(() => undefined)
+        throw new SyntaxError(`Cannot replace existing postgres client: ${configAlias} - ${service.alias}`)
+      }
+      service.clients[configAlias] = pool
+
+      /**
+       * Only when the config names one. Resources create their own schema anyway, and the
+       * admin connection points at the maintenance database — where inventing a namespace
+       * from the service alias would leave an empty `postgres` schema behind on every boot.
+       */
+      if (config.schema != null) {
+        await ensureSchema(pool, pgIdentifier(service.name(configAlias)))
+      }
+    },
+
+    /**
+     * `configAlias` is part of the contract for symmetry only — a resource's qualified
+     * name comes from the table spec it compiled at init, which already resolved against
+     * whichever config that resource was bound to.
+     */
+    qualify: resourceAlias => {
+      const context = assertContext<Config, Context>(service.ctx as Context, location)
+
+      return refOf(context as unknown as BasicContext<any>, null, resourceAlias)
+    },
+
+    query: async <Row extends QueryResultRow = QueryResultRow>(
+      text: string, params?: unknown[], configAlias?: string
+    ) => {
+      const db = await service.db(configAlias)
+      const context = assertContext<Config, Context>(service.ctx as Context, location)
+      try {
+        const result = await db.pool.query<Row>(
+          resolvePlaceholders(text, context as unknown as BasicContext<any>, null), params as never[]
+        )
+
+        return result.rows
+      } catch (error) {
+        throw pgErrorToResourceError(error)
+      }
+    },
+
+    transaction: async (fn, configAlias) => {
+      const db = await service.db(configAlias)
+      const context = assertContext<Config, Context>(service.ctx as Context, location)
+      const client = await db.pool.connect()
+      try {
+        await client.query('BEGIN')
+        const result = await fn(makeTx(
+          client,
+          text => resolvePlaceholders(text, context as unknown as BasicContext<any>, null),
+          resourceAlias => refOf(context as unknown as BasicContext<any>, null, resourceAlias)
+        ))
+        await client.query('COMMIT')
+
+        return result
+      } catch (error) {
+        await client.query('ROLLBACK').catch(() => undefined)
+        throw pgErrorToResourceError(error)
+      } finally {
+        client.release()
+      }
+    },
+
+    defer: (configAlias, task) => {
+      configAlias = service.ensureConfigAlias(configAlias)
+      const tasks = deferred.get(configAlias)
+      if (tasks == null) {
+        deferred.set(configAlias, [task])
+
+        return
+      }
+      tasks.push(task)
+    },
+
+    drain: async configAlias => {
+      const aliases = configAlias != null
+        ? [service.ensureConfigAlias(configAlias)]
+        : [...deferred.keys()]
+
+      for (const key of aliases) {
+        const tasks = deferred.get(key)
+        if (tasks == null) {
+          continue
+        }
+        /**
+         * Cleared before running, not after: a task that throws has still had its side
+         * effects, and a later drain replaying it would apply them twice.
+         */
+        deferred.delete(key)
+        for (const task of tasks) {
+          await task()
+        }
+      }
+    },
+
+    bootstrap: async (configAlias, opts) => bootstrapDb(service, configAlias, opts),
+
+    lock: async (configAlias, record, fields) => {
+      configAlias = service.ensureConfigAlias(configAlias)
+      const config = service.config(configAlias)
+      if (config.encryptionKey == null) {
+        throw new SyntaxError(`No encryption key for locking: ${configAlias}`)
+      }
+      if ((fields.length ?? 0) < 1) {
+        throw new SyntaxError(`No fields to lock: ${JSON.stringify(record)}`)
+      }
+
+      const key = makeKeyPairModel(config.encryptionKey)
+
+      return Object.fromEntries(
+        await Promise.all(Object.entries(record).map(
+          async ([field, value]) => [
+            field,
+            fields.includes(field) ? await key.encrypt(value) : value
+          ]
+        ))
+      )
+    },
+
+    unlock: async (configAlias, record, fields) => {
+      configAlias = service.ensureConfigAlias(configAlias)
+      const config = service.config(configAlias)
+      if (config.encryptionKey == null) {
+        throw new SyntaxError(`No encryption key for unlocking: ${configAlias}`)
+      }
+      if ((fields.length ?? 0) < 1) {
+        throw new SyntaxError(`No fields to unlock: ${JSON.stringify(record)}`)
+      }
+
+      const key = makeKeyPairModel(config.encryptionKey)
+
+      return Object.fromEntries(
+        await Promise.all(Object.entries(record).map(
+          async ([field, value]) => [
+            field,
+            /** A value that was never locked decrypts to itself rather than failing. */
+            fields.includes(field) ? await key.decrypt(value).catch(() => value) : value
+          ]
+        ))
+      )
+    },
+
+    reinitializeContext: <T>(context: BasicContext<ServerConfig>) => {
+      const _service = makePostgresDbService(alias)
+
+      _service.ctx = context
+      _service.layers = service.layers
+
+      /**
+       * Pools carry over instead of being reopened. A layer switch changes the Postgres
+       * *schema*, not the server, and `max_connections` is a cluster wide budget — a pool
+       * per tenant exhausts it long before the tenants run out. The handle cache is
+       * deliberately *not* carried: its schema name is what the switch just changed.
+       */
+      Object.assign(_service.clients, service.clients)
+
+      return _service as T
+    }
+  }, service => async () => {
+    const context = assertContext<Config, Context>(service.ctx as Context, location)
+
+    await context.cfg.dbs?.filter(dbConfig => dbConfig.service === alias).reduce(async (prev, dbConfig) => {
+      await prev
+      service.config(dbConfig.alias)
+    }, Promise.resolve())
+
+    service.initialized = true
+  })
+
+  return service
+}
+
+export const appendPostgres = <C extends Config, T extends Context<C> = Context<C>>(
+  context: T, alias: string = DEFAULT_ALIAS
+): T => {
+  const service = makePostgresDbService(alias)
+
+  context.registerService(service)
+  context.registerMiddleware(drainMiddleware(alias))
+
+  return context
+}

package/src/types.ts

@@ -0,0 +1,42 @@
+import type { PostgresDbService } from '@owlmeans/postgres-resource'
+
+export interface PostgresService extends PostgresDbService {
+  /**
+   * Provision a least privileged role, its database and its schema, using a superuser
+   * connection held under a *separate* config alias.
+   *
+   * Idempotent by design — every OwlMeans deployment calls it on each start and each
+   * rebuild, so it probes before it creates and never fails on "already exists".
+   *
+   * @throws {PostgresBootstrapError}
+   */
+  bootstrap: (configAlias: string, opts: BootstrapOptions) => Promise<BootstrapReport>
+}
+
+export interface BootstrapOptions {
+  /** Login role to create, or whose password to rotate. */
+  role: string
+  password: string
+  /** Database to create, owned by `role`. Defaults to the role name. */
+  database?: string
+  /** Schema to create inside it, owned by `role` and set as its `search_path`. */
+  schema?: string
+  /**
+   * Revoke `PUBLIC`'s rights on the database and on `public`, granting them to `role`
+   * alone. Defaults to `true` — the point of the admin path is least privilege.
+   */
+  leastPrivilege?: boolean
+  /**
+   * Reset the password of a role that already exists. Defaults to `true`, because the
+   * caller generated the password it just passed in and the role has to accept it.
+   */
+  rotatePassword?: boolean
+}
+
+export interface BootstrapReport {
+  roleCreated: boolean
+  passwordRotated: boolean
+  databaseCreated: boolean
+  schemaCreated: boolean
+  grantsApplied: boolean
+}

package/src/utils/config.ts

@@ -0,0 +1,109 @@
+import { PostgresConnectionError } from '@owlmeans/postgres-resource'
+import type { PostgresMeta } from '@owlmeans/postgres-resource'
+import type { DbConfig } from '@owlmeans/resource'
+import type { PoolConfig } from 'pg'
+
+import { DEF_POOL_SIZE, DEF_PORT } from '../consts.js'
+
+/**
+ * Split a `postgres://` URL into pool fields.
+ *
+ * node-postgres accepts a `connectionString` directly, but the parsed values *override*
+ * every sibling field — so `{ connectionString, database }` silently ignores the
+ * database. Parsing here instead keeps overriding possible, which the admin path needs
+ * to reach both the maintenance database and the target one over the same credentials.
+ *
+ * @throws {PostgresConnectionError}
+ */
+export const parseUrl = (url: string): PoolConfig => {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    throw new PostgresConnectionError('malformed-url')
+  }
+  if (!/^postgres(ql)?:$/.test(parsed.protocol)) {
+    throw new PostgresConnectionError(`unexpected-protocol:${parsed.protocol.replace(':', '')}`)
+  }
+
+  const database = decodeURIComponent(parsed.pathname.replace(/^\//, ''))
+  /** IPv6 literals arrive bracketed from `URL`; `pg` wants the bare address. */
+  const host = decodeURIComponent(parsed.hostname).replace(/^\[|\]$/g, '')
+  const mode = parsed.searchParams.get('sslmode')
+
+  const config: PoolConfig = {
+    host,
+    port: parsed.port !== '' ? parseInt(parsed.port, 10) : DEF_PORT
+  }
+  if (parsed.username !== '') {
+    config.user = decodeURIComponent(parsed.username)
+  }
+  if (parsed.password !== '') {
+    config.password = decodeURIComponent(parsed.password)
+  }
+  if (database !== '') {
+    config.database = database
+  }
+  if (mode != null && mode !== 'disable') {
+    /**
+     * `no-verify` is what a self signed in-cluster certificate needs, and is the only
+     * mode that may skip verification — anything else keeps the chain checked.
+     */
+    config.ssl = mode === 'no-verify' ? { rejectUnauthorized: false } : true
+  }
+
+  return config
+}
+
+/**
+ * Turn a {@link DbConfig} into a `pg` pool configuration.
+ *
+ * Values that start with `/` have already been read from disk by the `fileConfigReader`
+ * middleware, so a file mounted secret and a literal one are indistinguishable here —
+ * which is what lets a deployment move from env vars to mounted files without a code
+ * change.
+ */
+export const prepareConfig = (config: DbConfig, overrides?: PoolConfig): PoolConfig => {
+  const meta = (config.meta ?? {}) as PostgresMeta
+
+  /**
+   * `DbConfig.host` is an array for Mongo's replica sets. Postgres has no client side
+   * equivalent — multi-host failover is the pooler's job — so only the first is used.
+   */
+  const host = Array.isArray(config.host) ? config.host[0] : config.host
+
+  const base: PoolConfig = meta.url != null
+    ? parseUrl(meta.url)
+    : {
+      host,
+      port: config.port ?? DEF_PORT,
+      ...(config.user != null ? { user: config.user } : {}),
+      ...(config.secret != null ? { password: config.secret } : {}),
+      /** Absent, libpq falls back to the connection user's name. */
+      ...(meta.database != null ? { database: meta.database } : {})
+    }
+
+  const prepared: PoolConfig = {
+    ...base,
+    max: meta.max ?? DEF_POOL_SIZE,
+    ...(meta.ssl != null ? { ssl: meta.ssl as PoolConfig['ssl'] } : {}),
+    ...(meta.idleTimeoutMillis != null ? { idleTimeoutMillis: meta.idleTimeoutMillis } : {}),
+    ...(meta.connectionTimeoutMillis != null
+      ? { connectionTimeoutMillis: meta.connectionTimeoutMillis }
+      : {}),
+    /**
+     * A server side cap, so a query that outlives its caller stops holding a connection
+     * and its locks — the failure mode that turns one slow statement into a stuck pool.
+     */
+    ...(meta.statementTimeoutMillis != null
+      ? { statement_timeout: meta.statementTimeoutMillis }
+      : {}),
+    ...overrides
+  }
+
+  return prepared
+}
+
+/** Effective database of an open pool, as `pg` resolved it. */
+export const poolDatabase = (pool: { options?: PoolConfig }): string =>
+  pool.options?.database ?? pool.options?.user ?? ''