@starci/hfs 4.11.21 → 4.12.0

package/db/dump.mjs

@@ -1,79 +1,86 @@
-// dump.mjs - `starci app db dump`: a full backup of the database --url-env names (pg_dumpall roles, pg_dump schema and data),
+// dump.mjs - `starci app db dump`: a full backup of the database --db-url-env names (pg_dumpall roles, pg_dump schema and data),
 // encrypted outside Git; every plaintext part is removed even when a tool fails.
 import fs from 'node:fs';
 import path from 'node:path';
 import { runVerb } from '../release/index.mjs';
 import { failure, refusal } from '../stack/errors.mjs';
 import { RECEIPT_SCHEMA, outsideRepository, backupSession, backupRecipients, backupTool, encryptFile, queryRows, sha256, runPgTool } from './backup-receipt.mjs';
-
 /** Counts every ordinary table with quoted identifiers, every schema but the catalogs (auth included). */
 export function rowcounts(session) {
-  const tables = queryRows(session, "select schemaname, tablename from pg_catalog.pg_tables where schemaname not in ('pg_catalog', 'information_schema') order by schemaname, tablename;");
-  const quote = (name) => `"${String(name).replaceAll('"', '""')}"`;
-  return tables.map((table) => {
-    if (typeof table.schemaname !== 'string' || typeof table.tablename !== 'string') throw failure('DB_BACKUP_QUERY_FAILED', 'invalid table inventory');
-    const rows = queryRows(session, `select count(*)::text as count from ${quote(table.schemaname)}.${quote(table.tablename)}`);
-    if (rows.length !== 1 || !/^\d+$/u.test(String(rows[0].count))) throw failure('DB_BACKUP_QUERY_FAILED', 'invalid rowcount');
-    return { schema: table.schemaname, table: table.tablename, count: String(rows[0].count) };
-  });
+    const tables = queryRows(session, "select schemaname, tablename from pg_catalog.pg_tables where schemaname not in ('pg_catalog', 'information_schema') order by schemaname, tablename;");
+    const quote = (name) => `"${String(name).replaceAll('"', '""')}"`;
+    return tables.map((table) => {
+        if (typeof table.schemaname !== 'string' || typeof table.tablename !== 'string')
+            throw failure('DB_BACKUP_QUERY_FAILED', 'invalid table inventory');
+        const rows = queryRows(session, `select count(*)::text as count from ${quote(table.schemaname)}.${quote(table.tablename)}`);
+        if (rows.length !== 1 || !/^\d+$/u.test(String(rows[0]?.count)))
+            throw failure('DB_BACKUP_QUERY_FAILED', 'invalid rowcount');
+        return { schema: table.schemaname, table: table.tablename, count: String(rows[0]?.count) };
+    });
 }
-
 /** The archive name inside the plaintext scratch directory. */
 export const ARCHIVE_NAME = 'backup.tar';
-
 /**
  * Archive `parts` of `scratch` into scratch/backup.tar. tar runs inside scratch with relative names only: no -C and no
  * absolute path, because GNU tar (Git Bash puts it first on Windows) reads a drive-letter path (the `C:` of an absolute Windows path) as a remote `host:path`, while bsdtar
  * does not; relative arguments mean the same to both. Returns the archive's absolute path.
  */
 export function archiveParts(ctx, scratch, parts) {
-  backupTool(ctx, 'tar', ['-cf', ARCHIVE_NAME, ...parts], { cwd: scratch });
-  return path.join(scratch, ARCHIVE_NAME);
+    backupTool(ctx, 'tar', ['-cf', ARCHIVE_NAME, ...parts], { cwd: scratch });
+    return path.join(scratch, ARCHIVE_NAME);
 }
-
 /** Writes only ciphertext and its receipt permanently; a receipt is published only after header, size and hash checks. */
 export function dumpDatabase(ctx, options = {}) {
-  if (!options.out) throw refusal('DB_USAGE', '--out is required');
-  const out = outsideRepository(ctx.cwd, path.resolve(ctx.cwd, options.out));
-  const session = backupSession(ctx, options);
-  const recipients = backupRecipients(ctx.cwd);
-  fs.mkdirSync(out, { recursive: true, mode: 0o700 });
-  const at = new Date(ctx.now()).toISOString();
-  const stamp = at.replace(/[-:.]/gu, '');
-  const name = `${session.project}-${stamp}`;
-  const dir = path.join(out, name);
-  fs.mkdirSync(dir, { mode: 0o700 });
-  const scratch = path.join(dir, '.plaintext');
-  const archive = path.join(dir, `${name}.tar.age`);
-  const receiptPath = path.join(dir, `${name}.receipt.json`);
-  let complete = false;
-  try {
-    fs.mkdirSync(scratch, { mode: 0o700 });
-    const jobs = [['roles.sql', 'pg_dumpall', ['--roles-only', '--no-role-passwords']], ['schema.sql', 'pg_dump', ['--schema-only']]];
-    if (!options.schemaOnly) jobs.push(['data.sql', 'pg_dump', ['--data-only']]);
-    for (const [file, command, flags] of jobs) runPgTool(session, command, [...flags, '--file', path.join(scratch, file), '--dbname', session.target.url]);
-    const counts = rowcounts(session);
-    fs.writeFileSync(path.join(scratch, 'rowcounts.json'), `${JSON.stringify(counts, null, 2)}\n`, { mode: 0o600 });
-    const parts = [...jobs.map(([file]) => file), 'rowcounts.json'];
-    const tar = archiveParts(ctx, scratch, parts);
-    const bytes = encryptFile(ctx, tar, archive, recipients);
-    const receipt = { schema: RECEIPT_SCHEMA, project: session.project, target: session.target.kind, ref: session.target.urlEnv,
-      at, file: path.basename(archive), bytes, sha256: sha256(archive), parts,
-      rowcounts: { tables: counts.length, rows: counts.reduce((sum, row) => sum + BigInt(row.count), 0n).toString(), tablesByName: counts }, schemaOnly: options.schemaOnly === true };
-    fs.writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
-    complete = true;
-    return { ok: true, lines: [JSON.stringify(receipt, null, 2), `receipt: ${receiptPath}`, 'age header, size and sha256 verified; decryption requires the owner identity'], data: { receipt, receiptPath } };
-  } finally {
-    fs.rmSync(scratch, { recursive: true, force: true });
-    if (!complete) { fs.rmSync(archive, { force: true }); fs.rmSync(receiptPath, { force: true }); }
-  }
+    if (!options.out)
+        throw refusal('DB_USAGE', '--out is required');
+    const out = outsideRepository(ctx.cwd, path.resolve(ctx.cwd, options.out));
+    const session = backupSession(ctx, options);
+    const recipients = backupRecipients(ctx.cwd);
+    fs.mkdirSync(out, { recursive: true, mode: 0o700 });
+    const at = new Date(ctx.now()).toISOString();
+    const stamp = at.replace(/[-:.]/gu, '');
+    const name = `${session.project}-${stamp}`;
+    const dir = path.join(out, name);
+    fs.mkdirSync(dir, { mode: 0o700 });
+    const scratch = path.join(dir, '.plaintext');
+    const archive = path.join(dir, `${name}.tar.age`);
+    const receiptPath = path.join(dir, `${name}.receipt.json`);
+    let complete = false;
+    try {
+        fs.mkdirSync(scratch, { mode: 0o700 });
+        const jobs = [['roles.sql', 'pg_dumpall', ['--roles-only', '--no-role-passwords']], ['schema.sql', 'pg_dump', ['--schema-only']]];
+        if (!options.schemaOnly)
+            jobs.push(['data.sql', 'pg_dump', ['--data-only']]);
+        for (const [file, command, flags] of jobs)
+            runPgTool(session, command, [...flags, '--file', path.join(scratch, file), '--dbname', session.target.url]);
+        const counts = rowcounts(session);
+        fs.writeFileSync(path.join(scratch, 'rowcounts.json'), `${JSON.stringify(counts, null, 2)}\n`, { mode: 0o600 });
+        const parts = [...jobs.map(([file]) => file), 'rowcounts.json'];
+        const tar = archiveParts(ctx, scratch, parts);
+        const bytes = encryptFile(ctx, tar, archive, recipients);
+        const receipt = { schema: RECEIPT_SCHEMA, project: session.project, target: session.target.kind, ref: session.target.urlEnv,
+            at, file: path.basename(archive), bytes, sha256: sha256(archive), parts,
+            rowcounts: { tables: counts.length, rows: counts.reduce((sum, row) => sum + BigInt(row.count), 0n).toString(), tablesByName: counts }, schemaOnly: options.schemaOnly === true };
+        fs.writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
+        complete = true;
+        return { ok: true, lines: [JSON.stringify(receipt, null, 2), `receipt: ${receiptPath}`, 'age header, size and sha256 verified; decryption requires the owner identity'], data: { receipt, receiptPath } };
+    }
+    finally {
+        fs.rmSync(scratch, { recursive: true, force: true });
+        if (!complete) {
+            fs.rmSync(archive, { force: true });
+            fs.rmSync(receiptPath, { force: true });
+        }
+    }
 }
-
+/** The flags of `starci app db dump`; the parser refuses every other. */
+export const SPEC = { boolean: ['--schema-only', '--json'], value: ['--db-url-env', '--out'] };
 export function dbDumpMain(argv, io = {}) {
-  return runVerb({ label: 'starci app db dump', name: 'db dump', code: 'DB_USAGE', io, argv,
-    spec: { boolean: ['--schema-only', '--json'], value: ['--url-env', '--out'] },
-    run: (ctx, flags, positional) => {
-      if (positional.length) throw refusal('DB_USAGE', 'dump takes flags only');
-      return dumpDatabase(ctx, { urlEnv: flags['url-env'], out: flags.out, schemaOnly: flags['schema-only'] });
-    } });
+    return runVerb({ label: 'starci app db dump', name: 'db dump', code: 'DB_USAGE', io, argv,
+        spec: SPEC,
+        run: (ctx, flags, positional) => {
+            if (positional.length)
+                throw refusal('DB_USAGE', 'dump takes flags only');
+            return dumpDatabase(ctx, { urlEnv: flags['db-url-env'], out: flags.out, schemaOnly: flags['schema-only'] });
+        } });
 }

package/db/entity-list.d.mts

@@ -0,0 +1,4 @@
+/** The app's one entity list: it re-exports the arrays each capability's persistence/entities/index.ts exports. */
+export declare const ENTITY_LIST = "be/src/modules/platform/database/entities.ts";
+/** True when the app at `root` keeps an entity list: the apps whose entities `starci app db drift` checks and whose release gate runs it. */
+export declare const hasEntityList: (root: string) => boolean;

package/db/entity-list.mjs

@@ -1,9 +1,7 @@
 // entity-list.mjs - where an app keeps its TypeORM entity list, read by `starci app db drift` and by the release gate that runs it.
 import fs from 'node:fs';
 import path from 'node:path';
-
 /** The app's one entity list: it re-exports the arrays each capability's persistence/entities/index.ts exports. */
 export const ENTITY_LIST = 'be/src/modules/platform/database/entities.ts';
-
 /** True when the app at `root` keeps an entity list: the apps whose entities `starci app db drift` checks and whose release gate runs it. */
 export const hasEntityList = (root) => fs.existsSync(path.join(root, ENTITY_LIST));

package/db/image.d.mts

@@ -0,0 +1,8 @@
+/** The image app name a deploy block gives the db service. */
+export declare const DB_IMAGE_APP = "db";
+/** The build context of the db image, app-relative: the folder of its Dockerfile, which copies nothing from it. */
+export declare const DB_IMAGE_CONTEXT = ".starcistacks/images/db";
+/** The managed Dockerfile of the db image, app-relative. */
+export declare const DB_DOCKERFILE = ".starcistacks/images/db/Dockerfile";
+/** Where the db image's Dockerfile lived before it moved into the stacks; `starci app sync --execute` removes it. */
+export declare const RETIRED_DB_DOCKERFILE = "db/Dockerfile";

package/db/image.mjs

@@ -2,12 +2,11 @@
 // the managed .starcistacks/images/db/Dockerfile builds it, `deploy.images.db` of .starcistacks/application-stacks.yaml names it as app
 // `db`, and the stack build, the images and stack-deploy workflows, the local db container, the drift scratch database and the squash
 // replay all build that one file. db/ keeps only schema and data (migrations, bootstrap, seed).
-
 /** The image app name a deploy block gives the db service. */
 export const DB_IMAGE_APP = 'db';
 /** The build context of the db image, app-relative: the folder of its Dockerfile, which copies nothing from it. */
 export const DB_IMAGE_CONTEXT = '.starcistacks/images/db';
 /** The managed Dockerfile of the db image, app-relative. */
 export const DB_DOCKERFILE = `${DB_IMAGE_CONTEXT}/Dockerfile`;
-/** Where the db image's Dockerfile lived before it moved into the stacks; `starci app sync --write` removes it. */
+/** Where the db image's Dockerfile lived before it moved into the stacks; `starci app sync --execute` removes it. */
 export const RETIRED_DB_DOCKERFILE = 'db/Dockerfile';

package/db/migrate.d.mts

@@ -0,0 +1,104 @@
+import { BOOTSTRAP_FILE } from '../runtime/scripts/hfs/rules/database-constants.mjs';
+export { BOOTSTRAP_FILE };
+import type { DbTarget } from './postgres.mjs';
+import type { FlagSpec, VerbContext, VerbIo, VerbResult } from '../release/index.mjs';
+/** The function families `db post-cutover` checks (database.postCutover). */
+export interface PostCutover {
+    schema?: string;
+    functionPrefixes?: string[];
+    functionNames?: string[];
+    functions?: string[];
+    minAgeHours?: number;
+}
+/** A restamped migration: the old stamp and the new one. */
+export interface RestampPair {
+    from: string;
+    to: string;
+}
+/** The normalised `database:` block of .starcistacks/application-stacks.yaml. */
+export interface DatabaseDecl {
+    migrationsDir: string;
+    archiveDir: string | null;
+    baseline: string | null;
+    standard: string | null;
+    heldBack: string[];
+    restamp: RestampPair[];
+    postCutover: PostCutover | null;
+}
+/** An archived migration: its file name under the restamped version, and that version. */
+interface ArchiveRow {
+    name: string;
+    version: string;
+}
+/** The migration files of the app classified: baseline pair, held-back, forward, restamp pairs and archive. */
+export interface MigrationModel {
+    root: string;
+    decl: DatabaseDecl;
+    files: string[];
+    baselineFile: string | undefined;
+    standardFile: string | undefined;
+    baseline: string | null;
+    standard: string | null;
+    held: string[];
+    push: string[];
+    heldVersions: string[];
+    pushPending: string[];
+    archive: ArchiveRow[];
+    pushVersions: Set<string>;
+    allowed: Set<string>;
+}
+/** What a database verb works with: the context, the model and the target (and, for post-cutover, the declaration). */
+export interface MigrationSession {
+    ctx: VerbContext;
+    model: MigrationModel;
+    target: DbTarget;
+    decl?: DatabaseDecl;
+}
+/** What the plan decides, from the model and the recorded history. */
+interface MigrationPlan {
+    done: boolean;
+    repairs: RestampPair[];
+    reverts: string[];
+    mark: boolean;
+    push: string[];
+    bootstrap: boolean;
+    missingFiles: string[];
+    unexpected: string[];
+    final: number;
+}
+/** The normalised `database:` block of the app at `root` (all defaults when absent); an invalid block is a refusal. */
+export declare function loadDatabase(root: string): DatabaseDecl;
+/** Classifies the migration files of the app: baseline pair, held-back, forward, restamp pairs and archive. Refuses an inconsistent repository. */
+export declare function buildModel(root: string, decl: DatabaseDecl): MigrationModel;
+/** The database of `--db-url-env` (default DATABASE_URL). */
+export declare const resolveTarget: (ctx: VerbContext, options?: {
+    target?: DbTarget;
+    urlEnv?: string | undefined;
+}) => DbTarget;
+/** True for a database on this machine's loopback (the local db service): applying to it needs no arming. */
+export declare const isLoopback: (url: string) => boolean;
+/** Refuses a mutation unless the operator armed it with STARCI_DB_ARMED=1. */
+export declare function requireArmed(ctx: VerbContext): void;
+/** The recorded history as a Set of versions. */
+export declare const listRemote: (session: MigrationSession) => Set<string>;
+/** Pure: what to do given the recorded history. `done` means baseline + forward are present and nothing else is. */
+export declare function planMigration(model: MigrationModel, remote: ReadonlySet<string>): MigrationPlan;
+/** Applies `versions` in order: each file and its history row in one transaction, so a failure leaves the history true. */
+export declare function applyVersions(session: MigrationSession, versions: string[]): string[];
+/**
+ * `starci app db migrate`. options: { execute, urlEnv, target }. Dry run reads the history once and prints the plan (the entry adds the
+ * read-only inventory of cron jobs and queues, see withInventory; this function stays ok while work is pending, for the urgent floor); --execute applies it,
+ * armed with STARCI_DB_ARMED=1 unless the database is on the loopback (the local db service of `npm run db:push`). Already baseline + forward: `nothing to do`, no mutating call, ok.
+ */
+export declare function migrateDatabase(ctx: VerbContext, decl: DatabaseDecl, options?: {
+    execute?: boolean;
+    urlEnv?: string | undefined;
+    target?: DbTarget;
+}): Promise<VerbResult>;
+/** The flags of `starci app db migrate`; the parser refuses every other. */
+export declare const SPEC: FlagSpec;
+/**
+ * Entry of the verb: `starci app db migrate [--execute] [--db-url-env <NAME>] [--json]`. Execute requires database arming. Without
+ * --execute it is the read-only plan: exit 0 when the repository and the database agree, 1 on drift, 2 when refused.
+ */
+export declare function dbMigrateMain(argv: string[], io?: VerbIo): Promise<number>;