@everystack/cli 0.3.23 → 0.3.27

package/src/cli/db-build.ts

@@ -0,0 +1,161 @@
+/**
+ * db-build — build a database FROM the declared state (brick 5-remainder's
+ * shared core; decision 14: models-built, never data copies).
+ *
+ * One builder, three consumers: `db:check`'s ephemeral compose ring, the
+ * dev template (`db:template:refresh`), and `createEphemeralDatabase` — the
+ * exported test-database helper (the create-sync-drop shape this repo's own
+ * integration harness proves on every run, packaged for consumers). The bar
+ * is always the brick-2 identity: a fresh database built from the declared
+ * state lands EXACTLY on the models' fingerprint.
+ */
+
+import type { ModelDescriptor } from '@everystack/model';
+import type { TableContract, QueryRunner } from './authz-contract.js';
+import type { SourceFile } from './derived-source.js';
+import { compileDeclaredState } from './declared-diff.js';
+import { createUrlRunner } from './db-source.js';
+import { executeSync, buildSyncReport } from './commands/db-sync.js';
+
+const SAFE_NAME = /^[a-z_][a-z0-9_$]*$/;
+
+function assertSafeName(name: string, what: string): void {
+  if (!SAFE_NAME.test(name)) {
+    throw new Error(`${what} ${JSON.stringify(name)} is not a plain lowercase identifier — everystack manages databases it can name without quoting games.`);
+  }
+}
+
+/** Swap the database path segment of a connection URL. */
+export function withDatabase(url: string, database: string): string {
+  if (!/\/[^/?]+(\?|$)/.test(url)) {
+    throw new Error('the connection URL names no database — add the path segment (postgres://…/mydb)');
+  }
+  return url.replace(/\/[^/?]+(\?|$)/, `/${database}$1`);
+}
+
+/** Every role the contract grants to or names in a policy — a fresh database needs them. */
+export function rolesFromContract(tables: TableContract[]): string[] {
+  const roles = new Set<string>();
+  for (const t of tables) {
+    for (const grantee of Object.keys(t.grants)) roles.add(grantee);
+    for (const grantee of Object.keys(t.columnGrants ?? {})) roles.add(grantee);
+    for (const p of t.policies) for (const r of p.roles) roles.add(r);
+  }
+  roles.delete('PUBLIC');
+  roles.delete('public');
+  return [...roles].sort();
+}
+
+/** Create the contract's roles, NOLOGIN, only if absent (roles are cluster-level). */
+export async function ensureContractRoles(runner: QueryRunner, models: ModelDescriptor[]): Promise<string[]> {
+  const roles = rolesFromContract(compileDeclaredState(models).contract.tables);
+  for (const role of roles) {
+    assertSafeName(role, 'contract role');
+    await runner(
+      `DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN CREATE ROLE "${role}" NOLOGIN; END IF; END $$`,
+    );
+  }
+  return roles;
+}
+
+export async function createDatabase(adminUrl: string, name: string, template?: string): Promise<void> {
+  assertSafeName(name, 'database name');
+  if (template !== undefined) assertSafeName(template, 'template name');
+  const { runner, end } = await createUrlRunner(adminUrl);
+  try {
+    await runner(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ''}`);
+  } finally {
+    await end?.();
+  }
+}
+
+export async function dropDatabase(adminUrl: string, name: string): Promise<void> {
+  assertSafeName(name, 'database name');
+  const { runner, end } = await createUrlRunner(adminUrl);
+  try {
+    await runner(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
+  } finally {
+    await end?.();
+  }
+}
+
+export interface BuildResult {
+  converged: boolean;
+  fingerprintMatch: boolean;
+  fingerprint: string;
+  createdRoles: string[];
+  report: string[];
+}
+
+export interface BuildOptions {
+  sources?: SourceFile[] | null;
+  actor?: string | null;
+  gitRef?: string | null;
+}
+
+/**
+ * Sync a (typically fresh) database to the declared state — both layers —
+ * and report against the MATCH bar. Opens and closes its own connection,
+ * so the database is immediately usable as a `TEMPLATE` source afterwards.
+ */
+export async function buildIntoDatabase(
+  url: string,
+  models: ModelDescriptor[],
+  options: BuildOptions = {},
+): Promise<BuildResult> {
+  const { runner, end } = await createUrlRunner(url);
+  try {
+    const createdRoles = await ensureContractRoles(runner, models);
+    const run = await executeSync(runner, models, options.sources ?? null, {
+      actor: options.actor ?? 'db-build',
+      gitRef: options.gitRef ?? null,
+    });
+    return {
+      converged: run.converged,
+      fingerprintMatch: run.fingerprintMatch,
+      fingerprint: run.state.toFingerprint,
+      createdRoles,
+      report: buildSyncReport(run),
+    };
+  } finally {
+    await end?.();
+  }
+}
+
+export interface EphemeralDatabase {
+  database: string;
+  url: string;
+  fingerprint: string;
+  drop: () => Promise<void>;
+}
+
+/**
+ * The packaged create-sync-drop shape for consumer test suites: a fresh
+ * database at the declared state (state + authz + derived layer), never
+ * long-lived, so never poisoned. Throws (and drops) unless the build lands
+ * MATCH — a test database that isn't the declared state proves nothing.
+ */
+export async function createEphemeralDatabase(
+  adminUrl: string,
+  models: ModelDescriptor[],
+  options: BuildOptions & { name?: string } = {},
+): Promise<EphemeralDatabase> {
+  const database = options.name ?? `eph_${process.pid}_${Date.now()}`;
+  await createDatabase(adminUrl, database);
+  const url = withDatabase(adminUrl, database);
+  try {
+    const built = await buildIntoDatabase(url, models, options);
+    if (!(built.converged && built.fingerprintMatch)) {
+      throw new Error(`the ephemeral database did not land on the declared state:\n${built.report.join('\n')}`);
+    }
+    return {
+      database,
+      url,
+      fingerprint: built.fingerprint,
+      drop: () => dropDatabase(adminUrl, database),
+    };
+  } catch (err) {
+    await dropDatabase(adminUrl, database).catch(() => {});
+    throw err;
+  }
+}

package/src/cli/derived-source.ts

@@ -62,7 +62,7 @@ export interface ParsedSources {
 // Lexer.
 // ---------------------------------------------------------------------------
 
-type LexState =
+export type LexState =
   | { in: 'normal' }
   | { in: 'single'; escaped: boolean }
   | { in: 'double' }
@@ -82,7 +82,8 @@ function dollarTagAt(text: string, i: number): string | null {
  * (comments/strings/normal). The shared core under both the statement splitter
  * and the normalizer, so they can never disagree about where a string ends.
  */
-function lex(
+/** The dollar-quote-aware SQL lexer — exported for siblings that need string-safe scans (the backfill lane). */
+export function lex(
   text: string,
   emit: (ch: string, state: LexState['in'], index: number) => void,
 ): void {

package/src/cli/index.ts

@@ -19,6 +19,9 @@ import { dbPlanCommand } from './commands/db-plan.js';
 import { dbApplyCommand } from './commands/db-apply.js';
 import { dbCheckCommand } from './commands/db-check.js';
 import { dbApproversCommand } from './commands/db-approvers.js';
+import { dbBackfillCommand } from './commands/db-backfill.js';
+import { dbTemplateRefreshCommand, dbBranchCommand } from './commands/db-branch.js';
+import { dbForkCommand } from './commands/db-fork.js';
 import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
 import { dbBackupProbeCommand, dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
 import { consoleCommand } from './commands/console.js';
@@ -204,6 +207,18 @@ async function main() {
     case 'db:approvers':
       await dbApproversCommand(flags);
       break;
+    case 'db:backfill':
+      await dbBackfillCommand(flags);
+      break;
+    case 'db:template:refresh':
+      await dbTemplateRefreshCommand(flags);
+      break;
+    case 'db:branch':
+      await dbBranchCommand(flags);
+      break;
+    case 'db:fork':
+      await dbForkCommand(flags);
+      break;
     case 'db:authz:pull':
       await dbAuthzPullCommand(flags);
       break;
@@ -334,6 +349,10 @@ Usage:
   everystack db:apply          --plan <file.plan.json> [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--snapshot-ref <ref>] [--force-descent <snapshot-ref> --confirm]  Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule, else refuse — "rebase first"), and for DESTRUCTIVE plans require --confirm always + a snapshot (automatic via db:backup with --stage, else --snapshot-ref) + the stage's approver set when declared (STS identity-verified). Every refusal is recorded in schema_log. Apply as one transaction (plan_ref stamped), verify it landed exactly on plan.to; idempotent when already there; direct connection required
   everystack db:check          [--models <barrel>] [--sql-dir db/sql] [--schema-out <file.ts>] [--database-url <url>] [--json]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, db/sql parses) and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
   everystack db:approvers      --stage <name> [--set "cto,arn:..."] [--remove]  Declare who can DESTROY: the stage's destructive-approver set (SSM parameter, admin-writable). Destructive db:apply runs are then identity-verified (STS) against it; --set '' disables destructive applies; --remove returns the stage to ceremony-only
+  everystack db:backfill       [--database-url <url>] [--dir db/backfills] [--apply] [--mark-applied <file.sql>] [--json]  One-shot data jobs in their own lane: plan shows applied (by CONTENT identity — renames/comment edits are no-ops) / pending (in order, unbounded-pass advisories) / blocked (a name that already ran in a different form — one-shot jobs are immutable). --apply runs each pending job as its own transaction, recorded in everystack.backfill_log (a failure rolls back alone, is recorded, stops the run); --mark-applied records without running. Never runs as a schema side effect; direct connection required
+  everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--sql-dir db/sql] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
+  everystack db:branch         [--list | --drop --confirm | --prune --confirm] [--database-url <url>]  Mint (or find) the current git branch's database from the template (CREATE DATABASE … TEMPLATE — schema, authz, derived layer, and seed rows inherited), print its DATABASE_URL, then db:sync evolves it with the checkout. --list maps every branch DB to its branch; --prune drops the ones whose branch is gone (never guesses: unknown mappings are kept)
+  everystack db:fork           --from-stage <src> --stage <target> --confirm [--backup <id>]  Fork one DEPLOYED stage's database into another: back up the source (or reuse --backup <id>), presign the dump (the operator's credentials ARE the cross-stage authorization; expires in 1h), restore into the target via its ops Lambda. Production is never a target (that's db:restore); forking FROM production warns about PII; the branch's schema edge then lands via db:plan → db:apply (descent composes). Teardown: sst remove --stage <target>
   everystack db:authz:pull     [--stage <name>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
   everystack db:authz:diff     [--stage <name>] [--dir authz]  Validate the live DB against the committed contract (non-zero exit on drift)
   everystack db:authz:test     [--stage <name>] [--dir authz]  Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)

package/src/cli/model-api.ts

@@ -34,3 +34,10 @@ export * from './authz-owner-probe.js';
 
 // --- the SECDEF function catalog the contract introspection needs -----------
 export { FUNCTIONS_SQL, catalogFunctionToDescriptor } from './security-catalog.js';
+
+// --- databases FROM the declared state: ephemeral test DBs + the builder ----
+// `createEphemeralDatabase(adminUrl, models, { sources })` is the packaged
+// create-sync-drop shape for consumer test suites: a fresh database at the
+// declared state (state + authz + derived), verified to fingerprint MATCH,
+// dropped when you're done — never long-lived, never poisoned.
+export * from './db-build.js';

package/src/cli/runbook/sections.ts

@@ -150,9 +150,14 @@ function extend(r: ProjectReality): Block {
   ];
   if (r.models.dir) {
     lines.push(
-      '- **Add a table or field:** edit the Model in `models/`, run `everystack db:generate`,',
-      '  review the emitted migration, then `everystack db:migrate`. Never hand-write a SQL',
-      '  migration, never edit `db/schema.ts` — after any change `db:generate` must be a clean no-op.',
+      `- **Add a table or field:** edit the Model in \`${r.models.dir}/\`, run \`everystack db:sync\` —`,
+      '  the dev database follows your checkout (state + authz + derived, verified by fingerprint).',
+      '  Never hand-write a SQL migration, never edit generated schema — CI runs `everystack db:check`',
+      '  and refuses artifacts that do not match regeneration.',
+      '- **Ship a schema change:** protected stages take plans, not syncs: `everystack db:plan`',
+      '  (a reviewable edge, fingerprint-pinned at both ends) → review → `everystack db:apply`.',
+      '- **Move data:** one-shot jobs in `db/backfills/*.sql`, run deliberately via',
+      '  `everystack db:backfill --apply` — never as a schema side effect.',
     );
     if (r.migrationMode === 'transitional') {
       lines.push(
@@ -165,8 +170,9 @@ function extend(r: ProjectReality): Block {
     );
   } else if (r.hasDatabase) {
     lines.push(
-      '- **Schema changes:** migrate to Models (`everystack db:pull` writes `models/` from the',
-      '  live database), then the flow is: edit Model → `everystack db:generate` → `db:migrate`.',
+      '- **Schema changes:** migrate to Models (`everystack db:pull` writes `db/models/` from the',
+      '  live database — COMMIT that state before evolving it), then the loop is: edit Model →',
+      '  `everystack db:sync` (dev) / `db:plan` → `db:apply` (protected stages).',
     );
   }
   if (r.handlers.api) {
@@ -193,17 +199,24 @@ function extend(r: ProjectReality): Block {
 function database(r: ProjectReality): Block {
   const flow =
     r.migrationMode === 'everystack'
-      ? ['- `everystack db:generate` — compile Models → next migration (schema + RLS + grants)']
+      ? [
+          '- `everystack db:sync` — dev edit loop: the database follows your checkout (state + authz + derived)',
+          '- `everystack db:check` — the CI gate: merged Models compose, generated artifacts match regeneration',
+          '- `everystack db:plan` → `everystack db:apply` — protected stages: reviewable, fingerprint-verified edges',
+          '- `everystack db:branch` / `db:template:refresh` — per-git-branch dev databases from a seeded template',
+          '- `everystack db:backfill` — one-shot data jobs (`db/backfills/*.sql`), own record, run deliberately',
+        ]
       : r.migrationMode === 'transitional'
         ? [
             '- **Mid-migration to Models:** the journal is still hand-authored. Target flow is',
-            '  `everystack db:generate`; it must produce a clean, empty data diff before the',
-            '  generated flow takes over. Until then, review hand-written SQL like the liability it is.',
+            '  `everystack db:sync` (dev) / `db:plan` → `db:apply` (protected); `everystack db:generate`',
+            '  must produce a clean, empty data diff before the generated flow takes over. Until then,',
+            '  review hand-written SQL like the liability it is.',
           ]
         : [
             '- Migrations are hand-authored (drizzle-kit). The Models on-ramp: `everystack db:pull`',
-            '  writes `models/` from the live database; from there `everystack db:generate` compiles',
-            '  migrations (schema + RLS + grants) and hand-written SQL retires.',
+            '  writes `db/models/` from the live database (COMMIT that state before evolving it); from',
+            '  there `everystack db:sync` / `db:plan` → `db:apply` take over and hand-written SQL retires.',
           ];
   return gen('database', [
     '## Database operations',