@starci/hfs 4.3.0 → 4.4.1

package/runtime/scripts/hfs/db-plan-verb.mjs

@@ -0,0 +1,254 @@
+// app db plan: compare migration-owned state without fetching commands or Vault values.
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import pg from 'libpg-query';
+import { parse as parseToml } from 'smol-toml';
+import { invocationDir } from '../lib/roots.mjs';
+import { sqlStringParts, walkSqlAst as walk, sqlParser, sqlCallArgument as argument, sqlStringLiteral as literal } from '../lib/sql-ast.mjs';
+import { DbVerbError, dbOptions, dbAppRoot, dbSqlFiles, runDbSupabase, dbFailure } from './db-baseline-verb.mjs';
+
+await pg.loadModule();
+
+// The pinned Supabase CLI uses positional SQL, --output json and --agent no for a plain rows array.
+// Contract: supabase/cli apps/cli/src/commands/db/query/{query.command,query.handler}.ts at the pinned CLI tag.
+const PRESENCE_SQL = `select
+  pg_catalog.to_regclass('supabase_migrations.schema_migrations') is not null as migrations,
+  pg_catalog.to_regclass('cron.job') is not null as cron,
+  pg_catalog.to_regclass('pgmq.meta') is not null as queues,
+  pg_catalog.to_regclass('vault.secrets') is not null as vault;`;
+const INVENTORIES = {
+  migrations: 'select version from supabase_migrations.schema_migrations order by version;',
+  cron: 'select jobname as name, schedule, pg_catalog.md5(command) as "commandHash", active from cron.job order by jobid;',
+  queues: 'select queue_name as name from pgmq.meta order by queue_name;',
+  vault: 'select name from vault.secrets order by name;',
+};
+const names = (nodes) => sqlStringParts(nodes).join('.');
+const hash = (command) => createHash('md5').update(command).digest('hex');
+const OWNERSHIP_CALLS = new Set(['pgmq.create', 'pgmq.drop_queue', 'cron.schedule', 'cron.unschedule', 'cron.schedule_in_database', 'cron.alter_job']);
+
+const parseSql = sqlParser(pg, () => { throw new DbVerbError('migration SQL could not be inventoried safely'); });
+
+function requiredLiteral(call, index, aliases) {
+  const value = literal(argument(call, index, aliases));
+  if (value === null || value.length === 0) throw new DbVerbError('queue and cron ownership requires literal migration declarations');
+  return value;
+}
+
+function inventoryCall(state, call) {
+  const name = names(call.funcname);
+  if (name === 'pgmq.create') {
+    state.queues.set(requiredLiteral(call, 0, ['queue_name']), null);
+  } else if (name === 'pgmq.drop_queue') {
+    state.queues.delete(requiredLiteral(call, 0, ['queue_name']));
+  } else if (name === 'cron.schedule') {
+    const job = requiredLiteral(call, 0, ['job_name']);
+    const schedule = requiredLiteral(call, 1, ['schedule']);
+    const command = requiredLiteral(call, 2, ['command']);
+    state.cron.set(job, { schedule, commandHash: hash(command), active: true });
+  } else if (name === 'cron.unschedule') {
+    state.cron.delete(requiredLiteral(call, 0, ['job_name']));
+  } else if (name === 'cron.schedule_in_database' || name === 'cron.alter_job') {
+    throw new DbVerbError('database plan requires cron.schedule and named cron.unschedule ownership');
+  }
+}
+
+function ownershipCalls(tree) {
+  const calls = [];
+  walk(tree, (node) => { if (node.FuncCall && OWNERSHIP_CALLS.has(names(node.FuncCall.funcname))) calls.push(node.FuncCall); });
+  return calls;
+}
+
+function flatDeclaration(tree, call) {
+  const node = tree.SelectStmt ?? (tree.stmts?.length === 1 && tree.stmts[0].stmt.SelectStmt);
+  const constrained = ['fromClause', 'withClause', 'whereClause', 'intoClause', 'limitCount', 'limitOffset', 'groupClause', 'havingClause', 'distinctClause', 'windowClause'];
+  return node && /^SETOP_NONE$/.test(node.op) && node.targetList?.length === 1
+    && !constrained.some((key) => node[key]) && node.targetList[0].ResTarget?.val?.FuncCall === call;
+}
+
+function inventoryTree(state, tree) {
+  const calls = ownershipCalls(tree);
+  if (!calls.length) return;
+  if (calls.length !== 1 || !flatDeclaration(tree, calls[0])) throw new DbVerbError('database ownership requires one unconditional migration SELECT per declaration');
+  inventoryCall(state, calls[0]);
+}
+
+function conditionalBlock(item) {
+  if (item.PLpgSQL_stmt_block?.exceptions) return true;
+  return Object.keys(item).some((key) => key.startsWith('PLpgSQL_stmt_')
+    && !['PLpgSQL_stmt_block', 'PLpgSQL_stmt_perform', 'PLpgSQL_stmt_execsql'].includes(key)
+    && !(key === 'PLpgSQL_stmt_return' && item[key].lineno === undefined));
+}
+
+function inventoryDo(state, node) {
+  const option = (key) => (node.args ?? []).find((item) => item.DefElem?.defname === key)?.DefElem?.arg?.String?.sval;
+  if (option('language') && option('language') !== 'plpgsql') throw new DbVerbError('database plan requires statically readable migration blocks');
+  const body = option('as') ?? '';
+  let delimiter = '$starci_plan$';
+  while (body.includes(delimiter)) delimiter = `${delimiter.slice(0, -1)}_$`;
+  let tree;
+  try { tree = pg.parsePlPgSQLSync(`create function starci_plan_probe() returns void language plpgsql as ${delimiter}${body}${delimiter};`); }
+  catch { throw new DbVerbError('migration block could not be inventoried safely'); }
+  const statements = [];
+  let conditional = false;
+  walk(tree, (item) => {
+    conditional ||= conditionalBlock(item);
+    if (item.PLpgSQL_stmt_dynexecute) throw new DbVerbError('dynamic migration blocks cannot prove database inventory');
+    if (item.PLpgSQL_expr?.parseMode === 0) statements.push(parseSql(item.PLpgSQL_expr.query));
+  });
+  if (conditional && statements.some((statement) => ownershipCalls(statement).length)) throw new DbVerbError('conditional migration blocks cannot prove database inventory');
+  for (const statement of statements) inventoryTree(state, statement);
+}
+
+function quotedEnd(text, start) {
+  const quote = text[start];
+  const delimiter = text.startsWith(quote.repeat(3), start) ? quote.repeat(3) : quote;
+  let cursor = start + delimiter.length;
+  while (cursor < text.length) {
+    if (quote === '"' && text[cursor] === '\\') { cursor += 2; continue; }
+    if (text.startsWith(delimiter, cursor)) {
+      let end = cursor + delimiter.length;
+      if (delimiter.length === 3) while (text[end] === quote) end += 1;
+      return end;
+    }
+    cursor += 1;
+  }
+  throw new DbVerbError('Supabase config contains an unterminated string');
+}
+
+// Advance over an assignment value without collecting, decoding or resolving its content.
+function valueEnd(text, start) {
+  let cursor = start;
+  let depth = 0;
+  while (cursor < text.length) {
+    const character = text[cursor];
+    if (character === '"' || character === "'") { cursor = quotedEnd(text, cursor); continue; }
+    if (character === '#') {
+      cursor = text.indexOf('\n', cursor);
+      if (cursor < 0) return text.length;
+      if (depth === 0) return cursor + 1;
+    } else if (character === '[' || character === '{') depth += 1;
+    else if (character === ']' || character === '}') depth -= 1;
+    else if (character === '\n' && depth === 0) return cursor + 1;
+    cursor += 1;
+  }
+  return cursor;
+}
+
+function assignmentEnd(text, start) {
+  let cursor = start;
+  while (cursor < text.length && !['=', '\n', '#'].includes(text[cursor])) {
+    if (text[cursor] === '"' || text[cursor] === "'") cursor = quotedEnd(text, cursor);
+    else cursor += 1;
+  }
+  return cursor;
+}
+
+function vaultKeys(table, key) {
+  // The TOML parser sees only the table/key syntax and an invented empty value.
+  let metadata;
+  try { metadata = parseToml(`${table}\n${key} = ""`); }
+  catch { throw new DbVerbError('Supabase config keys could not be inventoried safely'); }
+  const vault = metadata.db?.vault ?? {};
+  if (typeof vault !== 'object' || Object.values(vault).some((value) => typeof value !== 'string')) throw new DbVerbError('Vault config requires individual named keys');
+  return Object.keys(vault);
+}
+
+function configToken(text, cursor, table) {
+  if (/\s/.test(text[cursor])) return { cursor: cursor + 1, table, keys: [] };
+  if (text[cursor] === '#' || text[cursor] === '[') {
+    const end = text.indexOf('\n', cursor);
+    const line = text.slice(cursor, end < 0 ? undefined : end);
+    return { cursor: end < 0 ? text.length : end + 1, table: text[cursor] === '[' ? line : table, keys: [] };
+  }
+  const end = assignmentEnd(text, cursor);
+  if (text[end] !== '=') throw new DbVerbError('Supabase config keys could not be inventoried safely');
+  return { cursor: valueEnd(text, end + 1), table, keys: vaultKeys(table, text.slice(cursor, end)) };
+}
+
+function configVaultNames(root) {
+  const file = path.join(root, 'supabase/config.toml');
+  if (!fs.existsSync(file)) return new Map();
+  const text = fs.readFileSync(file, 'utf8');
+  const configured = new Map();
+  let token = { cursor: 0, table: '' };
+  while (token.cursor < text.length) {
+    token = configToken(text, token.cursor, token.table);
+    for (const name of token.keys) {
+      if (configured.has(name)) throw new DbVerbError('Vault config names must be unique');
+      configured.set(name, null);
+    }
+  }
+  return configured;
+}
+
+function repositoryState(root) {
+  const migrations = dbSqlFiles(root);
+  const state = { migrations: new Map(migrations.map((entry) => [entry.stamp, null])), cron: new Map(), queues: new Map(), vault: configVaultNames(root) };
+  if (state.migrations.size !== migrations.length) throw new DbVerbError('active migrations must have unique versions');
+  for (const entry of migrations) {
+    const tree = parseSql(fs.readFileSync(path.join(root, entry.file), 'utf8'));
+    for (const statement of tree.stmts ?? []) {
+      // Creating a function does not execute its body. Inert SQL strings never declare remote state.
+      if (statement.stmt.CreateFunctionStmt) continue;
+      if (statement.stmt.DoStmt) inventoryDo(state, statement.stmt.DoStmt);
+      else inventoryTree(state, statement.stmt);
+    }
+  }
+  return state;
+}
+
+function queryRows(root, sql, local, ports) {
+  // The pinned CLI otherwise creates this cache in its linked-query finalizer.
+  // Existing metadata is sufficient; never inspect the cache or credential files.
+  if (!local && !ports.runSupabase && !ports.run && !fs.existsSync(path.join(root, 'supabase/.temp/linked-project.json'))) throw new DbVerbError('link the existing project before running a read-only database plan');
+  const output = runDbSupabase(root, ['db', 'query', sql, local ? '--local' : '--linked', '--output', 'json', '--agent', 'no'], ports);
+  let rows;
+  try { rows = JSON.parse(output); } catch { throw new DbVerbError('Supabase returned unreadable inventory; database output was withheld'); }
+  if (!Array.isArray(rows) || rows.some((row) => !row || typeof row !== 'object' || Array.isArray(row))) throw new DbVerbError('Supabase returned an invalid inventory; database output was withheld');
+  return rows;
+}
+
+function remoteState(root, local, ports) {
+  const rows = queryRows(root, PRESENCE_SQL, local, ports);
+  if (rows.length !== 1 || Object.keys(INVENTORIES).some((key) => typeof rows[0][key] !== 'boolean')) throw new DbVerbError('Supabase returned invalid inventory availability');
+  const state = {};
+  for (const [key, sql] of Object.entries(INVENTORIES)) state[key] = rows[0][key] ? queryRows(root, sql, local, ports) : [];
+  return state;
+}
+
+function compare(expected, actual, kind) {
+  const unmatched = new Set(expected.keys());
+  let unexpected = 0;
+  let changed = 0;
+  for (const row of actual) {
+    const key = kind === 'migrations' ? row.version : row.name;
+    if (key !== null && (typeof key !== 'string' || key.length === 0)) throw new DbVerbError('Supabase returned an invalid inventory identifier');
+    if (kind === 'cron' && (typeof row.schedule !== 'string' || !/^[a-f0-9]{32}$/.test(row.commandHash ?? '') || typeof row.active !== 'boolean')) throw new DbVerbError('Supabase returned invalid cron metadata');
+    if (!unmatched.delete(key)) unexpected += 1;
+    else if (kind === 'cron') {
+      const desired = expected.get(key);
+      if (desired.schedule !== row.schedule || desired.commandHash !== row.commandHash || desired.active !== row.active) changed += 1;
+    }
+  }
+  return { expected: expected.size, remote: actual.length, missing: unmatched.size, unexpected, changed };
+}
+
+/** Numeric exit codes: 0 equal, 1 drift, 2 invalid/failed; all IO and CLI calls are injectable. */
+export function dbPlanMain(argv, { cwd = invocationDir(), stdout = (text) => process.stdout.write(text), stderr = (text) => process.stderr.write(text), ...ports } = {}) {
+  try {
+    const opts = dbOptions(argv, ['cwd', 'local', 'json', 'apply']);
+    const root = dbAppRoot(cwd, opts);
+    const expected = repositoryState(root);
+    const actual = remoteState(root, opts.local, ports);
+    const summaries = Object.fromEntries(Object.keys(INVENTORIES).map((kind) => [kind, compare(expected[kind], actual[kind], kind)]));
+    const drift = Object.values(summaries).some((item) => item.missing || item.unexpected || item.changed);
+    // --apply is accepted for shared verb ergonomics; a plan never mutates history, queues or jobs.
+    if (opts.json) stdout(`${JSON.stringify({ readOnly: true, drift, ...summaries })}\n`);
+    else {
+      stdout(`read-only database plan: ${drift ? 'drift detected' : 'repo and database agree'}\n`);
+      for (const [kind, item] of Object.entries(summaries)) stdout(`${kind}: repo ${item.expected}, remote ${item.remote}, missing ${item.missing}, unexpected ${item.unexpected}, changed ${item.changed}\n`);
+    }
+    return drift ? 1 : 0;
+  } catch (error) { return dbFailure('plan', error, stderr); }
+}

package/runtime/scripts/hfs/db-restamp-verb.mjs

@@ -0,0 +1,60 @@
+// app db restamp: an explicit version map previews file renames and history repairs.
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseYaml } from '../../engine/yaml.mjs';
+import { invocationDir } from '../lib/roots.mjs';
+import { migrationStamp } from './rules/database-migrations.mjs';
+import { MIGRATIONS_DIR, MIGRATIONS_ARCHIVE_DIR } from './rules/database-constants.mjs';
+import { DbVerbError, dbOptions, dbAppRoot, dbSqlFiles, runDbSupabase, repairArgs, dbFailure } from './db-baseline-verb.mjs';
+
+/** YAML maps use quoted UTC stamps as keys and values (or a versions: mapping). */
+function restampPlan(root, mapFile) {
+  if (!mapFile) throw new DbVerbError('restamp requires --map <restamp.yaml>');
+  let document;
+  try { document = parseYaml(fs.readFileSync(path.resolve(root, mapFile), 'utf8')); }
+  catch { throw new DbVerbError('restamp map must be readable YAML with unique version keys'); }
+  const mapping = document?.versions ?? document;
+  if (!mapping || typeof mapping !== 'object' || Array.isArray(mapping) || !Object.keys(mapping).length) throw new DbVerbError('restamp map must contain old-to-new version pairs');
+  const active = dbSqlFiles(root);
+  const archived = dbSqlFiles(root, MIGRATIONS_ARCHIVE_DIR);
+  const occupied = new Set([...active, ...archived].map((entry) => entry.stamp));
+  const targets = new Set();
+  const steps = Object.entries(mapping).map(([old, raw]) => {
+    const next = String(raw);
+    if (![old, next].every((stamp) => /^\d{14}$/.test(stamp) && migrationStamp(stamp) !== null) || old === next) throw new DbVerbError('restamp pairs require distinct valid UTC versions');
+    const sources = active.filter((entry) => entry.stamp === old);
+    if (sources.length !== 1) throw new DbVerbError('each restamp source must identify exactly one active migration; archives never change');
+    if (occupied.has(next) || targets.has(next)) throw new DbVerbError('restamp targets must be unique unoccupied versions');
+    targets.add(next);
+    const source = sources[0];
+    const name = source.name.replace(old, next);
+    return { old, next, source: source.file, target: `${MIGRATIONS_DIR}/${name}` };
+  });
+  const replacements = new Map(steps.map((step) => [step.old, step.next]));
+  const resulting = active.map((entry) => replacements.get(entry.stamp) ?? entry.stamp);
+  if (resulting.some((stamp, index) => index > 0 && stamp <= resulting[index - 1])) throw new DbVerbError('the map must cover every version needed to preserve migration order');
+  return steps.sort((left, right) => left.old < right.old ? -1 : 1);
+}
+
+/** The complete map is validated before any repair or file mutation. */
+export function dbRestampMain(argv, { cwd = invocationDir(), stdout = (text) => process.stdout.write(text), stderr = (text) => process.stderr.write(text), ...ports } = {}) {
+  try {
+    const opts = dbOptions(argv, ['cwd', 'map', 'apply', 'local']);
+    const root = dbAppRoot(cwd, opts);
+    const steps = restampPlan(root, opts.map);
+    const repairs = steps.flatMap((step) => [repairArgs(step.old, 'reverted', opts.local), repairArgs(step.next, 'applied', opts.local)]);
+    stdout(`${opts.apply ? 'apply' : 'preview'} restamp ${steps.length} migrations\n`);
+    for (const step of steps) stdout(`${step.source} -> ${step.target}\n`);
+    for (const args of repairs) stdout(`supabase ${args.join(' ')}\n`);
+    if (!opts.apply) return 0;
+    // CLI failures never echo credentials and leave the local files intact for an explicit retry.
+    for (const args of repairs) runDbSupabase(root, args, ports);
+    for (const step of steps) {
+      const source = path.join(root, step.source);
+      const text = fs.readFileSync(source, 'utf8').replace(/^\s*--\s*starci:restamp\s+\d{14}\s+\d{14}\s*\r?\n/, '');
+      fs.writeFileSync(path.join(root, step.target), `-- starci:restamp ${step.old} ${step.next}\n${text}`, { flag: 'wx' });
+      fs.unlinkSync(source);
+    }
+    return 0;
+  } catch (error) { return dbFailure('restamp', error, stderr); }
+}

package/runtime/scripts/hfs/declaration-shape.mjs

@@ -60,26 +60,62 @@ function connectionProblems(side, value) {
 /** Whether a declared list is present but not a list of distinct entries that all satisfy `valid`. */
 const isBadUniqueList = (list, valid) => list !== undefined && (!Array.isArray(list) || !list.every((entry) => valid(entry)) || new Set(list).size !== list.length);
 
+const PACKAGE_NAME = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/;
+
+function validExtraPackage(entry) {
+  return isPlainObject(entry) && Object.keys(entry).every(key => key === 'name' || key === 'reason')
+    && typeof entry.name === 'string' && entry.name.length <= 214 && PACKAGE_NAME.test(entry.name)
+    && typeof entry.reason === 'string' && entry.reason.trim().length > 0;
+}
+
+function extraPackageProblems(side, entries) {
+  if (entries === undefined) return [];
+  if (side !== 'fe') return ['extraPackages belong to the fe side'];
+  if (!Array.isArray(entries) || !entries.every(validExtraPackage)) return ['sides.fe.extraPackages must be a list of {name, reason} with a package name and a nonempty reason'];
+  return [];
+}
+
 function declarationSideProblems(side, value, names) {
   const bad = [];
   const at = `sides.${side}`;
   if (!isPlainObject(value)) return [`${at} must be an object`];
-  for (const key of Object.keys(value)) if (!['apps', 'optionalSlots', 'patterns', 'kinds', 'connections', 'reads'].includes(key)) bad.push(`${at} has unknown key ${key}`);
+  for (const key of Object.keys(value)) if (!['apps', 'optionalSlots', 'patterns', 'kinds', 'connections', 'reads', 'extraPackages'].includes(key)) bad.push(`${at} has unknown key ${key}`);
   bad.push(...appShapeProblems(value.apps, at, side, names));
   if (isBadUniqueList(value.optionalSlots, (slot) => SLOT_ID.test(String(slot)))) bad.push(`${at}.optionalSlots must be a unique list of slot ids`);
   bad.push(...patternShapeProblems(value, at, NAME), ...kindShapeProblems(value, at, NAME));
   if (isBadUniqueList(value.reads, (read) => typeof read === 'string' && read.length > 0)) bad.push(`${at}.reads must be a unique list of paths`);
   if (value.connections !== undefined) bad.push(...connectionProblems(side, value));
+  bad.push(...extraPackageProblems(side, value.extraPackages));
+  return bad;
+}
+
+/** Product vocabulary is a declaration, never a switch for a rule. */
+function lexiconBlockProblems(lexicon) {
+  if (lexicon === undefined) return [];
+  if (!isPlainObject(lexicon)) return ['lexicon must be an object'];
+  const bad = [];
+  for (const key of Object.keys(lexicon)) if (key !== 'removed') bad.push(`lexicon has unknown key ${key}`);
+  if (isBadUniqueList(lexicon.removed, (word) => typeof word === 'string' && word.trim().length > 0)) bad.push('lexicon.removed must be a unique list of nonempty words');
+  return bad;
+}
+
+/** Front-end error reporting remains optional in both editions. */
+function observeBlockProblems(observe) {
+  if (observe === undefined) return [];
+  if (!isPlainObject(observe)) return ['observe must be an object'];
+  const bad = [];
+  for (const key of Object.keys(observe)) if (key !== 'reporter') bad.push(`observe has unknown key ${key}`);
+  if (observe.reporter !== undefined && (typeof observe.reporter !== 'string' || !observe.reporter.trim())) bad.push('observe.reporter must name the optional error reporter');
   return bad;
 }
 
-const DECLARATION_KEYS = new Set(['hfs', 'kind', 'project', 'edition', 'supabase', 'sides', 'browser']);
+const DECLARATION_KEYS = new Set(['hfs', 'kind', 'project', 'edition', 'supabase', 'sides', 'browser', 'lexicon', 'observe']);
 const RUNTIME_DECLARATION_KEYS = new Set(['hfs', 'kind', 'project']);
 
 function productDeclarationProblems(declaration, profiles) {
   const bad = [];
   for (const key of Object.keys(declaration)) if (!DECLARATION_KEYS.has(key)) bad.push(`unknown key ${key}`);
-  bad.push(...supabaseBlockProblems(declaration.supabase));
+  bad.push(...supabaseBlockProblems(declaration.supabase), ...lexiconBlockProblems(declaration.lexicon), ...observeBlockProblems(declaration.observe));
   if (declaration.browser !== undefined && declaration.browser !== true) bad.push('browser is `true` when the app has a browser journey (slot app.browser), and is left out otherwise');
   if (declaration.kind !== APP_KIND) bad.push(`kind must be ${APP_KIND} (a product is one app repository with a be and an fe side) or ${RUNTIME_KIND} (the StarCi runtime repository)`);
   if (!isPlainObject(declaration.sides) || Object.keys(declaration.sides).sort(byCodeUnit).join() !== profiles.join()) { bad.push('sides must declare exactly be and fe'); return bad; }