@nextsparkjs/core 0.1.0-beta.189 → 0.1.0-beta.190

package/scripts/db/cluster-changes.mjs

@@ -0,0 +1,205 @@
+// What a theme verification run changes outside the database it was given.
+//
+// A migration run is not contained in its database: 001 creates the
+// `authenticated`, `anon` and `service_role` roles, and 022 creates
+// `nextspark_app`, sets it to INHERIT and grants it `authenticated` plus
+// membership to the connecting user. Roles are cluster-wide, so those
+// statements reach every other database on the same Postgres server, including
+// ones the run was never pointed at.
+//
+// The database being empty says nothing about that, which is why the checks
+// here look at the cluster: the roles the migrations touch, and whether any
+// other database is using this server.
+
+/**
+ * Roles the core migrations create or alter, and what they do to each.
+ * `altered` separates the one whose attributes and memberships the migrations
+ * change even when it is already there from the ones a Supabase project
+ * already has and 001 only fills in.
+ */
+export const GLOBAL_OBJECTS = [
+  { role: 'authenticated', change: 'created if missing', altered: false },
+  { role: 'anon', change: 'created if missing', altered: false },
+  { role: 'service_role', change: 'created if missing', altered: false },
+  {
+    role: 'nextspark_app',
+    change: 'created if missing, set to INHERIT, granted authenticated',
+    altered: true,
+  },
+  {
+    role: 'current_user',
+    change: 'granted nextspark_app, so whoever runs this can SET ROLE to it',
+    altered: true,
+    connecting: true,
+  },
+]
+
+// `current_user` is whoever connects, so it is announced but never looked for
+// among the roles the server already has.
+const ALTERED_ROLES = new Set(
+  GLOBAL_OBJECTS.filter(object => object.altered && !object.connecting).map(object => object.role)
+)
+
+/**
+ * Databases a Postgres server creates for itself, or a managed one adds. Their
+ * presence says nothing about the server being in use — `postgres` is asked
+ * about separately, since a hosted project can keep the app's own schema there.
+ *
+ * Only `template0` and `template1` among templates: any database can be marked
+ * `datistemplate`, and one a user marked that way shares the cluster's roles
+ * like any other.
+ */
+const HOUSEKEEPING_DATABASES = new Set([
+  'postgres', 'template0', 'template1', 'rdsadmin', 'azure_maintenance', 'azure_sys', 'cloudsqladmin',
+])
+
+/** The database a client connects to when it only needs the server. */
+export const MAINTENANCE_DATABASE = 'postgres'
+
+export const ROLES_SQL = `
+  SELECT r.rolname AS name,
+         r.rolcanlogin AS can_login,
+         COALESCE(array_agg(g.rolname) FILTER (WHERE g.rolname IS NOT NULL), '{}'::text[]) AS member_of
+    FROM pg_roles r
+    LEFT JOIN pg_auth_members m ON m.member = r.oid
+    LEFT JOIN pg_roles g ON g.oid = m.roleid
+   WHERE r.rolname IN ('authenticated', 'anon', 'service_role')
+      OR r.rolname LIKE 'nextspark\\_%'
+   GROUP BY r.rolname, r.rolcanlogin
+   ORDER BY r.rolname
+`
+
+// A database that refuses connections (`datallowconn = false`) or is marked as
+// a template is still a database on this server, sharing the roles this run
+// alters, so it counts. The server's own templates are left out by name, in
+// HOUSEKEEPING_DATABASES.
+export const DATABASES_SQL = `
+  SELECT datname AS name
+    FROM pg_database
+   WHERE datname <> current_database()
+   ORDER BY datname
+`
+
+/** An array of role names, whether the driver hands over an array or `{a,b}`. */
+function asList(value) {
+  if (Array.isArray(value)) return value
+  if (typeof value !== 'string') return []
+  return value.replace(/^\{|\}$/g, '').split(',').filter(Boolean)
+}
+
+/**
+ * What the run would change beyond its own database, and whether the cluster
+ * shows signs of being used by anything else.
+ *
+ * A role the migrations touch that already exists is the stronger signal: the
+ * run alters it for whoever else is using it. Another database on the server is
+ * the weaker one, and enough on its own — a role created now lands in a cluster
+ * that is not this run's to change.
+ *
+ * `maintenance` is what looking at the `postgres` database answered:
+ * `{ objects }` when it could be read, `{ unreachable: true, reason }` when it
+ * could not. Not being able to look is itself a reason to refuse — a server
+ * that denies CONNECT to its own maintenance database is a managed one with
+ * someone else's rules on it, not a container this run may rearrange.
+ */
+export function inspectCluster({ roles = [], databases = [], maintenance = {} } = {}) {
+  const maintenanceObjects = maintenance.objects ?? []
+  const existingRoles = roles.map(role => ({
+    name: role.name,
+    canLogin: role.can_login ?? role.canLogin ?? false,
+    memberOf: asList(role.member_of ?? role.memberOf),
+  }))
+  const otherDatabases = databases
+    .map(database => database.name ?? database)
+    .filter(name => !HOUSEKEEPING_DATABASES.has(name))
+
+  const describe = role => (role.memberOf.length ? `${role.name} (member of ${role.memberOf.join(', ')})` : role.name)
+  const altered = existingRoles.filter(role => ALTERED_ROLES.has(role.name) || role.name.startsWith('nextspark_'))
+  const shared = existingRoles.filter(role => !altered.includes(role))
+
+  const reasons = []
+  if (altered.length > 0) {
+    reasons.push(`this run alters roles that are already here: ${altered.map(describe).join(', ')}`)
+  }
+  if (shared.length > 0) {
+    reasons.push(
+      `roles these migrations create already exist, so something else set this server up: ${shared.map(describe).join(', ')}`
+    )
+  }
+  if (otherDatabases.length > 0) {
+    const shown = otherDatabases.slice(0, 5).join(', ')
+    const rest = otherDatabases.length > 5 ? `, and ${otherDatabases.length - 5} more` : ''
+    reasons.push(`other databases share this server: ${shown}${rest}`)
+  }
+  if (maintenanceObjects.length > 0) {
+    const examples = maintenanceObjects.slice(0, 3).map(object => object.name ?? object).join(', ')
+    reasons.push(`the ${MAINTENANCE_DATABASE} database holds objects of its own, for example ${examples}`)
+  }
+  if (maintenance.unreachable) {
+    const detail = maintenance.reason ? `: ${maintenance.reason}` : ''
+    reasons.push(
+      `the ${MAINTENANCE_DATABASE} database could not be read${detail}, so what else is on this server is unknown`
+    )
+  }
+
+  return { disposable: reasons.length === 0, reasons, existingRoles, otherDatabases }
+}
+
+/**
+ * Words that can stand where a role name would, in the statements above:
+ * GRANT's own syntax, the privileges it grants, and the objects it grants them
+ * on. None of them is a role.
+ */
+const GRANT_KEYWORDS = new Set([
+  'ALL', 'PRIVILEGES', 'SELECT', 'INSERT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER',
+  'CREATE', 'CONNECT', 'TEMPORARY', 'TEMP', 'EXECUTE', 'USAGE', 'SET', 'ALTER', 'MAINTAIN',
+  'ON', 'IN', 'SCHEMA', 'TABLE', 'TABLES', 'SEQUENCE', 'SEQUENCES', 'FUNCTION', 'FUNCTIONS',
+  'ROUTINE', 'ROUTINES', 'DATABASE', 'DOMAIN', 'TYPE', 'LANGUAGE', 'PUBLIC', 'SYSTEM',
+  'FOREIGN', 'DATA', 'WRAPPER', 'SERVER', 'LARGE', 'OBJECT', 'TABLESPACE', 'PARAMETER',
+  'OPTION', 'ADMIN', 'INHERIT', 'GRANTED', 'BY', 'FOR', 'ROLE', 'COLUMN',
+])
+
+/**
+ * Every role a migration names, whichever statement names it.
+ *
+ * Both sides of a grant count: `GRANT authenticated TO reporting` alters
+ * `reporting` as much as it uses `authenticated`, and a grant of privileges —
+ * `GRANT SELECT ON ... TO reporting` — names only the grantee. A grant can list
+ * several of either, so each list is split on commas.
+ */
+export function rolesNamedIn(sql) {
+  const named = new Set()
+  // Comments and RAISE messages are prose: a GRANT quoted inside either is
+  // being talked about, not run. A GRANT inside format() is run, so string
+  // literals in general stay.
+  const statements = sql
+    .replace(/\/\*[\s\S]*?\*\//g, ' ')
+    .replace(/--[^\n]*/g, ' ')
+    .replace(/\bRAISE\s+(?:DEBUG|LOG|INFO|NOTICE|WARNING|EXCEPTION)\b[^;]*;/gi, ' ')
+
+  const add = list => {
+    for (const name of list.split(',')) {
+      // A grantee list can end inside a format() call, so the string and the
+      // call close on the last name: `format('GRANT x TO %I', current_user)`.
+      const role = name.trim().replace(/[)'"]+$/, '').replace(/^["']+/, '')
+      // GRANT's own keywords, and the object a privilege is granted on, are
+      // not roles; `format('... %I', current_user)` names the connecting user.
+      if (/^[a-z_]\w*$/i.test(role) && !GRANT_KEYWORDS.has(role.toUpperCase())) named.add(role)
+    }
+  }
+
+  for (const [, list] of statements.matchAll(/\b(?:CREATE|ALTER|DROP)\s+(?:ROLE|USER|GROUP)\s+(?:IF\s+(?:NOT\s+)?EXISTS\s+)?("?\w+"?)/gi)) {
+    add(list)
+  }
+  // The grantee list runs from the last TO/FROM to the end of the statement,
+  // past any WITH ADMIN OPTION / GRANTED BY tail.
+  for (const [, grantees] of statements.matchAll(/\b(?:GRANT|REVOKE)\b[^;]*?\s(?:TO|FROM)\s+([^;]+)/gi)) {
+    add(grantees.split(/\bWITH\b|\bGRANTED\s+BY\b|\bCASCADE\b|\bRESTRICT\b/i)[0])
+  }
+  // What a grant of role membership hands over, which is a role too.
+  for (const [, roles] of statements.matchAll(/\b(?:GRANT|REVOKE)\s+((?:"?\w+"?\s*,\s*)*"?\w+"?)\s+(?:TO|FROM)\s/gi)) {
+    add(roles)
+  }
+
+  return named
+}

package/scripts/db/connection-time-limits.mjs

@@ -0,0 +1,53 @@
+// Time limits on a pg connection that hold whatever the connection string says.
+//
+// pg reads a connection string's parameters over the options passed next to it,
+// so a URL ending in `?statement_timeout=0&query_timeout=` would switch off both
+// the server's limit and the client's. The client is therefore built exactly as
+// pg builds one for the string, with no limits among the options, and the two
+// limits are then set on the connection parameters pg resolved for it. Which of
+// the string's parameters a client acts on, and which it ignores, is decided by
+// the installed version of pg in the same way it would be without the limits;
+// only the two limits differ. pg reads them from those parameters when it
+// connects (statement_timeout, sent in the startup message) and each time it
+// sends a query (query_timeout).
+//
+// Nothing else a connection brings along lifts the server's limit: the
+// statement_timeout sent when connecting outranks one in
+// `options=-c statement_timeout=…`, in PGOPTIONS, or set on the database or the
+// role.
+
+import { scriptClient } from './ssl-config.mjs';
+
+/** The connection-string parameters pg reads over the time limits given next to them. */
+export const TIME_LIMIT_PARAMETERS = ['statement_timeout', 'query_timeout'];
+
+/** How pg builds a client for the string, with the shared application SSL policy. */
+function clientFor(connectionString, options = {}) {
+  return scriptClient(connectionString, options);
+}
+
+/**
+ * The time-limit parameters the connection string sets, as pg reads them: the
+ * ones whose value stays the same whatever limit is passed next to the string.
+ */
+export function timeLimitParametersIn(connectionString) {
+  const withLimits = limit => clientFor(connectionString, { statement_timeout: limit, query_timeout: limit }).connectionParameters;
+  const one = withLimits(1);
+  const two = withLimits(2);
+  return TIME_LIMIT_PARAMETERS.filter(name => one[name] === two[name]);
+}
+
+/**
+ * A client whose limits are the ones given: `connectMs` to connect,
+ * `statementMs` for the server to cancel a statement, `queryMs` for the client
+ * to give up on an answer. `database` connects to another database on the same
+ * server, as whoever the connection string names.
+ */
+export function timeLimitedClient(connectionString, { connectMs, statementMs, queryMs }, { database } = {}) {
+  const client = clientFor(connectionString, { connectionTimeoutMillis: connectMs });
+  const parameters = client.connectionParameters;
+  parameters.statement_timeout = statementMs;
+  parameters.query_timeout = queryMs;
+  if (database !== undefined) client.database = parameters.database = database;
+  return client;
+}

package/scripts/db/inspect-server.mjs

@@ -0,0 +1,108 @@
+// What db:verify-theme asks a Postgres server before it migrates anything.
+//
+// Every question has a time limit, not only the connection. A query that waits
+// on a lock someone else holds on a catalog, or a server that stops answering
+// once the connection is up, would otherwise leave the command waiting until
+// something outside kills it, when what the command owes is a refusal. The
+// limits hold whatever time limits the database URL sets (see
+// connection-time-limits.mjs).
+
+import { ROLES_SQL, DATABASES_SQL, MAINTENANCE_DATABASE } from './cluster-changes.mjs';
+import { timeLimitedClient } from './connection-time-limits.mjs';
+
+/**
+ * How long each step may take. The server cancels a statement past
+ * `statementMs`; `queryMs`, a little longer, is the client giving up on a
+ * server that does not even answer that cancellation.
+ */
+export const TIMEOUTS = { connectMs: 10000, statementMs: 15000, queryMs: 20000 };
+
+// Everything that can be created in a non-system schema: relations (tables,
+// views, sequences, indexes), functions and standalone types. Objects that
+// belong to an extension don't count, since the server's template database
+// can bring extensions along. Functions and types count because the
+// migrations' CREATE OR REPLACE would replace them without an error.
+export const EXISTING_OBJECTS_SQL = `
+  SELECT kind, name
+    FROM (
+      SELECT 'relation' AS kind, n.nspname || '.' || c.relname AS name, n.nspname,
+             'pg_class'::regclass AS classid, c.oid AS objid
+        FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+      UNION ALL
+      SELECT 'function', n.nspname || '.' || p.proname, n.nspname,
+             'pg_proc'::regclass, p.oid
+        FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
+      UNION ALL
+      SELECT 'type', n.nspname || '.' || t.typname, n.nspname,
+             'pg_type'::regclass, t.oid
+        FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+       WHERE t.typrelid = 0 AND t.typcategory <> 'A'
+    ) objects
+   WHERE nspname NOT IN ('pg_catalog', 'information_schema')
+     AND nspname !~ '^pg_(toast|temp)'
+     AND NOT EXISTS (
+       SELECT 1 FROM pg_depend d
+        WHERE d.classid = objects.classid AND d.objid = objects.objid AND d.deptype = 'e'
+     )
+   ORDER BY kind, name
+   LIMIT 5
+`;
+
+export function connect(connectionString, timeouts = TIMEOUTS, target = {}) {
+  return timeLimitedClient(connectionString, timeouts, target);
+}
+
+// `end()` drops the socket when a query is still active, so a connection whose
+// query timed out closes instead of waiting for a server that is not answering.
+async function withClient(connectionString, timeouts, work, target = {}) {
+  const client = connect(connectionString, timeouts, target);
+  await client.connect();
+  try {
+    return await work(client);
+  } finally {
+    await client.end();
+  }
+}
+
+/** What the target database holds, and what the cluster around it shows. */
+export function inspectTarget(connectionString, timeouts = TIMEOUTS) {
+  return withClient(connectionString, timeouts, async client => {
+    const [objects, roles, databases] = await Promise.all([
+      client.query(EXISTING_OBJECTS_SQL),
+      client.query(ROLES_SQL),
+      client.query(DATABASES_SQL),
+    ]);
+    return { existingObjects: objects.rows, roles: roles.rows, databases: databases.rows };
+  });
+}
+
+/**
+ * What the maintenance database holds. A hosted Postgres keeps `postgres` for
+ * connections that only need the server, and a project whose own schema lives
+ * there looks, from the outside, like a server with nothing on it.
+ *
+ * The connection goes to the server and user the database URL names, as pg
+ * reads them, with only the database changed. The target database is compared
+ * the way pg resolves it too, so a URL with no database whose user is
+ * `postgres` is already the maintenance database.
+ *
+ * A server that refuses the connection, or the query, or does not answer it in
+ * time, is reported as unreachable rather than as empty: not being able to look
+ * is what a managed cluster looks like, and reading it as "nothing there" is
+ * how a run ends up creating cluster-wide roles on a server it was never
+ * allowed to inspect.
+ */
+export async function inspectMaintenanceDatabase(databaseUrl, timeouts = TIMEOUTS) {
+  if (connect(databaseUrl, timeouts).database === MAINTENANCE_DATABASE) return { objects: [] };
+  try {
+    const result = await withClient(
+      databaseUrl,
+      timeouts,
+      client => client.query(EXISTING_OBJECTS_SQL),
+      { database: MAINTENANCE_DATABASE }
+    );
+    return { objects: result.rows };
+  } catch (error) {
+    return { unreachable: true, reason: error.message };
+  }
+}