@starci/hfs 4.10.1 → 4.11.8

package/runtime/scripts/hfs/db-baseline-verb.mjs

@@ -10,10 +10,12 @@ import { baselineMarkers, baselineSourceFindings } from './rules/database-baseli
 import { databaseQueueDeclarations } from './rules/database-queues.mjs';
 import { MIGRATIONS_DIR, MIGRATIONS_ARCHIVE_DIR } from './rules/database-constants.mjs';
 import { dbPlanMain } from './db-plan-verb.mjs';
+import { DbVerbError } from './db-errors.mjs';
+import { squashBaseline } from './db-squash.mjs';
 
 await pg.loadModule();
 
-export class DbVerbError extends Error {}
+export { DbVerbError };
 
 /** Strict shared options; unknown flags never reach the database CLI. */
 export function dbOptions(argv, allowed) {
@@ -22,7 +24,7 @@ export function dbOptions(argv, allowed) {
     const [flag, inline] = argv[index].split(/=(.*)/s);
     const name = flag.slice(2);
     if (!flag.startsWith('--') || !allowed.includes(name) || Object.hasOwn(opts, name)) throw new DbVerbError('unknown or duplicate database option');
-    if (['apply', 'json'].includes(name)) {
+    if (['apply', 'json', 'squash'].includes(name)) {
       if (inline !== undefined && !['true', 'false'].includes(inline)) throw new DbVerbError(`${flag} expects true or false`);
       opts[name] = inline !== 'false';
     } else {
@@ -42,19 +44,38 @@ export function dbAppRoot(cwd, opts) {
   return root;
 }
 
-/** Immediate SQL files only, never secrets or credential files. */
+/** A refusal that lists every offending file with the rule it broke, one distinct `<file>: <reason>` line each. */
+export const offendersError = (heading, offenders) => new DbVerbError(`${heading}:\n${[...new Set(offenders)].map((line) => `  ${line}`).join('\n')}`);
+
+/** One rule finding as an offender line: its file, its message and its code. */
+const findingLine = (item) => `${item.message.startsWith(item.path) ? item.message : `${item.path}: ${item.message}`} (${item.code})`;
+
+/**
+ * Why one SQL filename cannot join its chain, or null. The active chain holds R213 (`<ts14>_<kebab>.sql`, a valid UTC
+ * stamp; a stamp after now is no breach). The archive is frozen history (R257): it needs only the leading stamp that
+ * orders it, so snake_case names, impossible times and shared stamps of a retired chain stay as they were committed.
+ */
+function chainProblem(directory, name) {
+  const match = /^(\d{14})_/.exec(name);
+  if (directory === MIGRATIONS_ARCHIVE_DIR) return match ? null : 'an archived name starts with its 14-digit stamp and an underscore (R257)';
+  if (!isMigrationName(name)) return 'name is not <ts14>_<kebab>.sql (R213)';
+  if (migrationStamp(match[1]) === null) return `stamp ${match[1]} is no valid UTC time (R213)`;
+  return null;
+}
+
+/** Immediate SQL files only, never secrets or credential files, sorted by full name; every offender is named before the refusal. */
 export function dbSqlFiles(root, directory = MIGRATIONS_DIR) {
   const dir = path.join(root, directory);
   if (!fs.existsSync(dir)) return [];
-  return fs.readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isFile() && entry.name.endsWith('.sql')).map((entry) => {
-    const match = /^(\d{14})_/.exec(entry.name);
-    if (!isMigrationName(entry.name) || !match || migrationStamp(match[1]) === null) throw new DbVerbError('migration filenames must carry valid UTC stamps');
-    return { file: `${directory}/${entry.name}`, name: entry.name, stamp: match[1] };
-  }).sort((left, right) => left.name < right.name ? -1 : Number(left.name > right.name));
+  const sqlNames = fs.readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isFile() && entry.name.endsWith('.sql')).map((entry) => entry.name);
+  const offenders = sqlNames.flatMap((name) => { const problem = chainProblem(directory, name); return problem ? [`${directory}/${name}: ${problem}`] : []; });
+  if (offenders.length) throw offendersError('migration filenames break their chain rules', offenders.sort());
+  return sqlNames.map((name) => ({ file: `${directory}/${name}`, name, stamp: name.slice(0, 14) }))
+    .sort((left, right) => left.name < right.name ? -1 : Number(left.name > right.name));
 }
 
 /** The database of `--url-env` (default DATABASE_URL): the URL without its password and the password for PGPASSWORD. */
-function connectionOf(env, urlEnv) {
+export function connectionOf(env, urlEnv) {
   if (!/^[A-Z][A-Z0-9_]*$/.test(urlEnv)) throw new DbVerbError('--url-env must name an UPPER_CASE environment variable');
   let url;
   try { url = new URL(String(env[urlEnv] ?? '')); } catch { throw new DbVerbError(`${urlEnv} must hold a postgres connection URL`); }
@@ -68,7 +89,7 @@ const HISTORY = 'supabase_migrations.schema_migrations';
 const sqlLiteral = (value) => `'${String(value).replaceAll("'", "''")}'`;
 
 /** The psql and pg_dump invocation of one operation: a JSON rows query, a schema-only dump, or one history row applied or reverted. */
-function commandOf(op, url) {
+export function commandOf(op, url) {
   const psql = ['-X', '-q', '-v', 'ON_ERROR_STOP=1', '-d', url];
   if (op.kind === 'query') return ['psql', [...psql, '-A', '-t', '-c', `select coalesce(json_agg(row_to_json(q)), '[]'::json) from (${op.sql.replace(/;\s*$/, '')}) q`]];
   if (op.kind === 'dump') return ['pg_dump', ['--schema-only', '--schema', 'public', '--dbname', url]];
@@ -103,12 +124,17 @@ function baselinePlan(root, opts, now) {
   const active = dbSqlFiles(root);
   if (!active.length) throw new DbVerbError('baseline requires an existing active migration chain');
   const archived = dbSqlFiles(root, MIGRATIONS_ARCHIVE_DIR);
-  const oldStamps = [...active, ...archived].map((entry) => entry.stamp);
-  const last = oldStamps.toSorted((left, right) => left.localeCompare(right)).at(-1);
+  const archivedNames = new Set(archived.map((entry) => entry.name));
+  const twice = active.filter((entry) => archivedNames.has(entry.name)).map((entry) => `${entry.file}: also archived as ${MIGRATIONS_ARCHIVE_DIR}/${entry.name}; a migration is active or archived, never both (R257)`);
+  if (twice.length) throw offendersError('baseline would overwrite an existing migration', twice);
+  const chain = [...active, ...archived].sort((left, right) => left.stamp < right.stamp ? -1 : Number(left.stamp > right.stamp) || (left.name < right.name ? -1 : Number(left.name > right.name)));
+  // The new version sorts after every file; the header names the last valid stamp, the boundary R257 judges.
+  const boundary = chain.at(-1);
+  const last = chain.filter((entry) => migrationStamp(entry.stamp) !== null).at(-1)?.stamp ?? boundary.stamp;
   const stamp = opts.stamp ?? now().toISOString().replace(/\D/g, '').slice(0, 14);
-  if (!/^\d{14}$/.test(stamp) || migrationStamp(stamp) === null || stamp <= last) throw new DbVerbError('--stamp must be a valid UTC version after the archive boundary');
+  if (!/^\d{14}$/.test(stamp) || migrationStamp(stamp) === null || stamp <= boundary.stamp) throw new DbVerbError(`--stamp must be a valid UTC version after the archive boundary ${boundary.stamp} (${boundary.file})`);
   const file = `${MIGRATIONS_DIR}/${stamp}_baseline.sql`;
-  if (fs.existsSync(path.join(root, file)) || active.some((entry) => fs.existsSync(path.join(root, MIGRATIONS_ARCHIVE_DIR, entry.name)))) throw new DbVerbError('baseline would overwrite an existing migration');
+  if (fs.existsSync(path.join(root, file))) throw new DbVerbError(`baseline would overwrite an existing migration: ${file}`);
   return { active, last, stamp, file };
 }
 
@@ -116,7 +142,7 @@ const EXCLUDED_SCHEMAS = new Set(['information_schema', '_analytics', '_realtime
 const ROLE_OPTIONS = new Set(['canlogin', 'inherit', 'superuser', 'createdb', 'createrole', 'isreplication', 'bypassrls', 'connectionlimit']);
 const identifier = (name) => `"${name.replaceAll('"', '""')}"`;
 const literal = (value) => `'${value.replaceAll("'", "''")}'`;
-const excludedSchema = (name) => EXCLUDED_SCHEMAS.has(name) || /^(?:pg_|timescaledb_|_timescaledb_)/.test(name ?? '');
+export const excludedSchema = (name) => EXCLUDED_SCHEMAS.has(name) || /^(?:pg_|timescaledb_|_timescaledb_)/.test(name ?? '');
 
 /** Inventory source paths only; no credential, generated or vendor tree is traversed. */
 function baselineFiles(root) {
@@ -255,12 +281,19 @@ function baselineStatement(state, stmt, sql) {
 
 function baselineMetadata(root, plan, files) {
   const queues = databaseQueueDeclarations({ repoRoot: root, files, all: files });
-  if (queues.findings.length) throw new DbVerbError('baseline requires valid migration-owned queues, grants and cron declarations');
+  if (queues.findings.length) throw offendersError('baseline requires valid migration-owned queues, grants and cron declarations', queues.findings.map(findingLine));
   const state = { roles: new Map(), extensions: new Map() };
+  const offenders = [];
   for (const migration of plan.active) {
     const text = fs.readFileSync(path.join(root, migration.file), 'utf8');
-    for (const entry of parsed(text).stmts ?? []) baselineStatement(state, entry.stmt, statementSql(text, entry));
+    try {
+      for (const entry of parsed(text).stmts ?? []) baselineStatement(state, entry.stmt, statementSql(text, entry));
+    } catch (error) {
+      if (!(error instanceof DbVerbError)) throw error;
+      offenders.push(`${migration.file}: ${error.message}`);
+    }
   }
+  if (offenders.length) throw offendersError('baseline cannot rebuild the active chain from a schema-only dump', offenders);
   return { ...queues, roles: [...state.roles.values()].flat().join(''), extensions: [...state.extensions.values()].join('') };
 }
 
@@ -271,17 +304,57 @@ function metadataSql(metadata) {
   return `${queueSql}${grants}${cron}`;
 }
 
+/**
+ * The squash source holds the whole active chain: its history records every active stamp and nothing else. Only the history is read; the
+ * queues and cron jobs a plan also compares are what the squash itself drops or carries from the database.
+ */
+function sourceHistory(root, plan, ports, urlEnv) {
+  const rows = (sql) => JSON.parse(runDbPostgres(root, { kind: 'query', sql }, ports, urlEnv));
+  if (!rows("select pg_catalog.to_regclass('supabase_migrations.schema_migrations') is not null as present")[0]?.present) throw new DbVerbError(`the source records no migration history (${HISTORY}); apply its chain with starci app db migrate`);
+  const recorded = new Set(rows(`select version from ${HISTORY}`).map((row) => String(row.version)));
+  const active = new Set(plan.active.map((entry) => entry.stamp));
+  const offenders = [...[...active].filter((stamp) => !recorded.has(stamp)).map((stamp) => `${stamp}: active, not applied to the source`), ...[...recorded].filter((stamp) => !active.has(stamp)).map((stamp) => `${stamp}: applied to the source, not in the active chain`)];
+  if (offenders.length) throw offendersError('baseline --squash reads a source with exactly the active chain applied', offenders.sort());
+}
+
+/**
+ * A baseline reads a database whose history, queues and cron jobs equal the repository's. The plan's own refusal (already free of
+ * database output) is passed through as the reason; a drift names each inventory that differs, by its counts.
+ */
+function cleanPlan(root, urlEnv, ports) {
+  let out = '';
+  let err = '';
+  const code = dbPlanMain(['--json', '--url-env', urlEnv], { ...ports, cwd: root, stdout: (text) => { out += text; }, stderr: (text) => { err += text; } });
+  if (code === 0) return;
+  const heading = 'baseline requires a clean database plan with every migration already applied';
+  if (code !== 1) throw new DbVerbError(`${heading}; the plan refused: ${err.trim().replace(/^starci app db plan: /, '')}`);
+  const drift = Object.entries(JSON.parse(out)).filter(([, item]) => item && typeof item === 'object' && (item.missing || item.unexpected || item.changed))
+    .map(([kind, item]) => `${kind}: missing ${item.missing}, unexpected ${item.unexpected}, changed ${item.changed}`);
+  throw offendersError(`${heading}; the plan found drift`, drift);
+}
+
 /** Numeric CLI exit code, injectable IO/clock/CLI; neither files nor history change without --apply. */
 export function dbBaselineMain(argv, { cwd = invocationDir(), stdout = (text) => process.stdout.write(text), stderr = (text) => process.stderr.write(text), now = () => new Date(), ...ports } = {}) {
   try {
-    const opts = dbOptions(argv, ['cwd', 'stamp', 'apply', 'url-env']);
+    const opts = dbOptions(argv, ['cwd', 'stamp', 'apply', 'url-env', 'squash', 'drop-extension', 'drop-pgmq-queue', 'seed-tables', 'image']);
+    if (!opts.squash && ['drop-extension', 'drop-pgmq-queue', 'seed-tables', 'image'].some((name) => Object.hasOwn(opts, name))) throw new DbVerbError('--drop-extension, --drop-pgmq-queue, --seed-tables and --image belong to --squash');
+    if (opts.squash && !opts['url-env']) throw new DbVerbError('baseline --squash reads a database the whole active chain is applied to: pass --url-env NAME');
     const urlEnv = opts['url-env'] ?? 'DATABASE_URL';
     const root = dbAppRoot(cwd, opts);
     const plan = baselinePlan(root, opts, now);
     const files = baselineFiles(root);
-    if (baselineSourceFindings({ repoRoot: root, files, all: files, git: ports.git }).length) throw new DbVerbError('baseline sources and archives must match committed migration history');
+    const sourceFindings = baselineSourceFindings({ repoRoot: root, files, all: files, git: ports.git });
+    if (sourceFindings.length) throw offendersError('baseline sources and archives must match committed migration history', sourceFindings.map(findingLine));
+    const archive = () => {
+      fs.mkdirSync(path.join(root, MIGRATIONS_ARCHIVE_DIR), { recursive: true });
+      for (const entry of plan.active) fs.renameSync(path.join(root, entry.file), path.join(root, MIGRATIONS_ARCHIVE_DIR, entry.name));
+    };
+    if (opts.squash) {
+      const precondition = (runPostgres) => sourceHistory(root, plan, { ...ports, runPostgres }, urlEnv);
+      return squashBaseline({ root, opts, plan, urlEnv, connection: connectionOf(ports.env ?? process.env, urlEnv), commandOf, precondition, managedSchema: excludedSchema, archive, stdout });
+    }
     const metadata = baselineMetadata(root, plan, files);
-    if (dbPlanMain(['--json', '--url-env', urlEnv], { ...ports, cwd: root, stdout: () => {}, stderr: () => {} }) !== 0) throw new DbVerbError('baseline requires a clean database plan with every migration already applied');
+    cleanPlan(root, urlEnv, ports);
     const schema = runDbPostgres(root, { kind: 'dump' }, ports, urlEnv);
     if (!schema.trim() || baselineMarkers(schema).length || /\bvault\.(?:create_secret|update_secret)\s*\(|\bpassword\s+'[^']+'/i.test(schema)) throw new DbVerbError('schema dump is empty or contains unsupported secret/baseline statements');
     const repairs = [...plan.active.map((entry) => repairOp(entry.stamp, 'reverted')), repairOp(plan.stamp, 'applied', 'baseline')];
@@ -289,9 +362,8 @@ export function dbBaselineMain(argv, { cwd = invocationDir(), stdout = (text) =>
     for (const op of repairs) stdout(`${repairLine(op)}\n`);
     if (!opts.apply) return 0;
     for (const op of repairs) runDbPostgres(root, op, ports, urlEnv);
-    fs.mkdirSync(path.join(root, MIGRATIONS_ARCHIVE_DIR), { recursive: true });
     fs.writeFileSync(path.join(root, plan.file), `-- starci:baseline ${plan.last}\n${metadata.extensions}${metadata.roles}${schema.replace(/^\s+/, '')}\n${metadataSql(metadata)}`, { flag: 'wx' });
-    for (const entry of plan.active) fs.renameSync(path.join(root, entry.file), path.join(root, MIGRATIONS_ARCHIVE_DIR, entry.name));
+    archive();
     return 0;
   } catch (error) { return dbFailure('baseline', error, stderr); }
 }

package/runtime/scripts/hfs/db-errors.mjs

@@ -0,0 +1,2 @@
+// The refusal every `starci app db` verb module throws; dbFailure (db-baseline-verb.mjs) prints its message, any other error is withheld.
+export class DbVerbError extends Error {}

package/runtime/scripts/hfs/db-plan-verb.mjs

@@ -5,7 +5,7 @@ import { sha256 } from '../../engine/digest.mjs';
 import pg from 'libpg-query';
 import { invocationDir } from '../lib/roots.mjs';
 import { sqlStringParts, walkSqlAst as walk, sqlParser, sqlCallArgument as argument, sqlStringLiteral as literal } from '../lib/sql-ast.mjs';
-import { DbVerbError, dbOptions, dbAppRoot, dbSqlFiles, runDbPostgres, dbFailure } from './db-baseline-verb.mjs';
+import { DbVerbError, dbOptions, dbAppRoot, dbSqlFiles, runDbPostgres, dbFailure, offendersError } from './db-baseline-verb.mjs';
 
 await pg.loadModule();
 
@@ -75,6 +75,7 @@ function conditionalBlock(item) {
     && !(key === 'PLpgSQL_stmt_return' && item[key].lineno === undefined));
 }
 
+/** Inventories one DO block; returns the body-relative line of each EXECUTE, whose SQL is built at run time and so never inventoried. */
 function inventoryDo(state, node) {
   const option = (key) => (node.args ?? []).find((item) => item.DefElem?.defname === key)?.DefElem?.arg?.String?.sval;
   if (option('language') && option('language') !== 'plpgsql') throw new DbVerbError('database plan requires statically readable migration blocks');
@@ -85,29 +86,40 @@ function inventoryDo(state, node) {
   try { tree = pg.parsePlPgSQLSync(`create function starci_plan_probe() returns void language plpgsql as ${delimiter}${body}${delimiter};`); }
   catch { throw new DbVerbError('migration block could not be inventoried safely'); }
   const statements = [];
+  const dynamic = [];
   let conditional = false;
   walk(tree, (item) => {
     conditional ||= conditionalBlock(item);
-    if (item.PLpgSQL_stmt_dynexecute) throw new DbVerbError('dynamic migration blocks cannot prove database inventory');
+    if (item.PLpgSQL_stmt_dynexecute) dynamic.push(item.PLpgSQL_stmt_dynexecute.lineno);
     if (item.PLpgSQL_expr?.parseMode === 0) statements.push(parseSql(item.PLpgSQL_expr.query));
   });
   if (conditional && statements.some((statement) => ownershipCalls(statement).length)) throw new DbVerbError('conditional migration blocks cannot prove database inventory');
   for (const statement of statements) inventoryTree(state, statement);
+  return dynamic;
 }
 
+/** The 1-based line of `text` at libpg_query's byte offset `location`. */
+const lineAt = (text, location) => Buffer.from(text).subarray(0, location).toString().split('\n').length;
+
 function repositoryState(root) {
   const migrations = dbSqlFiles(root);
   const state = { migrations: new Map(migrations.map((entry) => [entry.stamp, null])), cron: new Map(), queues: new Map() };
   if (state.migrations.size !== migrations.length) throw new DbVerbError('active migrations must have unique versions');
+  // Every EXECUTE of every active migration is named (file:line) before the refusal.
+  const dynamic = [];
   for (const entry of migrations) {
-    const tree = parseSql(fs.readFileSync(path.join(root, entry.file), 'utf8'));
+    const text = fs.readFileSync(path.join(root, entry.file), 'utf8');
+    const tree = parseSql(text);
     for (const statement of tree.stmts ?? []) {
       // Creating a function does not execute its body. Inert SQL strings never declare remote state.
       if (statement.stmt.CreateFunctionStmt) continue;
-      if (statement.stmt.DoStmt) inventoryDo(state, statement.stmt.DoStmt);
-      else inventoryTree(state, statement.stmt);
+      if (!statement.stmt.DoStmt) { inventoryTree(state, statement.stmt); continue; }
+      // The body opens on the line of its quote (the `as` DefElem), so a body line maps to that line plus its offset.
+      const opening = lineAt(text, statement.stmt.DoStmt.args?.find((item) => item.DefElem?.defname === 'as')?.DefElem?.location ?? statement.stmt_location ?? 0);
+      for (const line of inventoryDo(state, statement.stmt.DoStmt)) dynamic.push(`${entry.file}:${opening + line - 1}: EXECUTE builds its SQL at run time`);
     }
   }
+  if (dynamic.length) throw offendersError('dynamic migration blocks cannot prove database inventory', dynamic);
   return state;
 }