@everystack/cli 0.4.30 → 0.4.31

package/src/cli/commands/db-fork.ts

@@ -22,6 +22,7 @@
 import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
 import { invokeAction, presignGet } from '../aws.js';
 import { keyForId, parseBackupRef, isProductionTier, crossStageGuard } from '../backup.js';
+import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info, warn } from '../output.js';
 
 export type ForkGuardVerdict =
@@ -92,13 +93,19 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
     }
     info(`Reusing backup ${backupId}.`);
   } else {
-    step(`Backing up ${from} (pg_dump → S3; may take a while for large databases)...`);
-    const result: any = await invokeAction(source.region, opsFunction(source), 'db:backup', { stage: from });
-    if (result?.error) {
-      fail(`Source backup failed: ${result.error}`);
+    step(`Backing up ${from} (pg_dump → S3 in the Task; may take a while for large databases)...`);
+    const dispatched: any = await invokeAction(source.region, opsFunction(source), 'db:backup', { stage: from, actor: process.env.USER ?? null });
+    if (dispatched?.error) {
+      fail(`Source backup failed: ${dispatched.error}`);
+      process.exit(1);
+    }
+    backupId = dispatched.id;
+    const poll = await pollTaskUntilStopped(source.region, opsFunction(source), { runId: dispatched.runId, taskArn: dispatched.taskArn });
+    if (poll.outcome !== 'stopped' || poll.status.exitCode !== 0) {
+      const why = poll.outcome === 'stopped' ? `exit ${poll.status.exitCode}${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}` : poll.outcome;
+      fail(`Source backup failed (${why}). Reconcile run ${dispatched.runId}.`);
       process.exit(1);
     }
-    backupId = result.id;
     info(`Backup ${backupId}.`);
   }
 
@@ -121,15 +128,22 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
     fail(`${err.message}\nDeploy the feature stage first: sst deploy --stage ${target}`);
     process.exit(1);
   }
-  step(`Restoring into ${target} (streamed; replaces its database)...`);
+  step(`Restoring into ${target} (pg_restore in the Task; replaces its database)...`);
   const restored: any = await invokeAction(targetConfig.region, opsFunction(targetConfig), 'db:restore', {
     url,
     confirm: true,
+    actor: process.env.USER ?? null,
   });
   if (restored?.error) {
     fail(`Restore failed: ${restored.error}`);
     process.exit(1);
   }
+  const rPoll = await pollTaskUntilStopped(targetConfig.region, opsFunction(targetConfig), { runId: restored.runId, taskArn: restored.taskArn });
+  if (rPoll.outcome !== 'stopped' || rPoll.status.exitCode !== 0) {
+    const why = rPoll.outcome === 'stopped' ? `exit ${rPoll.status.exitCode}${rPoll.status.stoppedReason ? ` — ${rPoll.status.stoppedReason}` : ''}` : rPoll.outcome;
+    fail(`Restore failed (${why}). Intent ${restored.intentKey} — the target DB may be partially restored.`);
+    process.exit(1);
+  }
 
   success(`Forked ${from} → ${target} (backup ${backupId}).`);
   info('Next:');

package/src/cli/commands/deploy.ts

@@ -38,11 +38,10 @@ export async function deployCommand(flags: Record<string, string>): Promise<void
   const args = buildDeployArgs(flags);
 
   step(`Deploying stage "${stage}"...`);
-  // PGDUMP_LAYER_ARN is deprecated: the layer now ships via @everystack/pg-tools and pgDumpLayer()
-  // needs no env var. Still forwarded to sst for one more minor (pgDumpLayer honors it as an alias),
-  // then removed — surface it loudly so the operator migrates off it.
+  // The pg_dump Lambda layer is gone — all pg-binary work (backup/export/restore) runs in the
+  // ephemeral Fargate Task lane. PGDUMP_LAYER_ARN no longer does anything; flag it so it's dropped.
   if (process.env.PGDUMP_LAYER_ARN) {
-    warn('PGDUMP_LAYER_ARN is DEPRECATED and will be removed next minor. The pg_dump layer now ships via @everystack/pg-tools + pgDumpLayer() with no env var. Drop it once your sst.config.ts uses pgDumpLayer().');
+    warn('PGDUMP_LAYER_ARN is set but IGNORED — the pg_dump layer was removed; pg-binary work runs in the Fargate Task lane. Drop the env var.');
   }
   await new Promise<void>((resolve) => {
     // `npx` resolves the consumer's local sst (the same way the export step runs expo).

package/src/cli/commands/task-probe.ts

@@ -0,0 +1,66 @@
+/**
+ * `everystack task:probe --stage <name>` — the ephemeral Task lane's smoke test.
+ *
+ * Dispatches the version-handshake task (via the ops Lambda's `task:probe` action — the operator
+ * holds no URL), then polls `task:status` until the task stops. A pass proves the whole substrate
+ * end to end: the task started, resolved its injected credential, reached the private DB, and the
+ * image's pg tools are compatible. Phase 3+ reuses this dispatch/poll for real pg_dump/pg_restore.
+ *
+ * The poll is BOUNDED (a deadline) — a Fargate task can sit in PROVISIONING/PENDING on capacity or
+ * ENI trouble, and a naked loop would hang the CLI. On deadline it reports the last status and the
+ * run id so the operator can reconcile (the task_log row + ECS both carry it).
+ */
+
+import { resolveConfig, opsFunction } from '../config.js';
+import { invokeAction } from '../aws.js';
+import { pollTaskUntilStopped } from '../task-poll.js';
+import { step, success, fail, info } from '../output.js';
+
+const DEADLINE_MS = 10 * 60_000; // 10 min — Fargate cold start + pull + handshake, with headroom.
+
+export async function taskProbeCommand(flags: Record<string, string>): Promise<void> {
+  const stage = flags.stage;
+  if (!stage) {
+    fail('task:probe needs --stage <name> (the deployed stage whose Task lane to probe).');
+    process.exit(1);
+  }
+
+  step('Resolving deployed config...');
+  const config = await resolveConfig(stage);
+  const fn = opsFunction(config);
+
+  step('Dispatching the handshake task (credential-free, via the ops Lambda)...');
+  const dispatched: any = await invokeAction(config.region, fn, 'task:probe', {
+    kind: 'probe', actor: process.env.USER ?? null, stage,
+  });
+  if (dispatched?.error) {
+    fail(`task:probe failed to dispatch: ${dispatched.error}`);
+    if (/Unknown action/i.test(String(dispatched.error))) {
+      info('The deployed handler predates the Task lane. Deploy the dbTask() wiring first (Step 5).');
+    }
+    process.exit(1);
+  }
+  const { runId, taskArn, warning } = dispatched as { runId: string; taskArn: string; warning?: string };
+  info(`task started: ${taskArn}`);
+  info(`run id: ${runId} (everystack.task_log)`);
+  if (warning) info(`note: ${warning}`);
+
+  step('Polling until the task stops...');
+  const poll = await pollTaskUntilStopped(config.region, fn, { runId, taskArn }, { deadlineMs: DEADLINE_MS });
+  if (poll.outcome === 'timeout') {
+    fail(`task:probe timed out after ${DEADLINE_MS / 60_000} min (last status: ${poll.lastStatus}). Run id ${runId} — reconcile via ECS/task_log.`);
+    process.exit(1);
+  }
+  if (poll.outcome === 'error') {
+    fail(`task:status failed repeatedly: ${poll.status.error}`);
+    info(`The task may still be running — reconcile run id ${runId} via ECS/everystack.task_log.`);
+    process.exit(1);
+  }
+  if (poll.status.exitCode === 0) {
+    success('Task lane verified: the handshake passed (image pg tools are compatible with the live server).');
+    return;
+  }
+  fail(`Task stopped with exit ${poll.status.exitCode ?? 'unknown'}${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}.`);
+  info('Read the task logs (CloudWatch) for the handshake verdict JSON.');
+  process.exit(1);
+}

package/src/cli/exec-digest.ts

@@ -0,0 +1,79 @@
+/**
+ * exec-digest — the catalog content-address that makes db:exec DML-only WITHOUT a role, a
+ * superuser, or a regex. A self-contained SQL query that md5s the schema-relevant catalog
+ * (pg_class / pg_attribute / pg_constraint / pg_index / pg_proc / pg_type) for the given app
+ * schemas. db:exec takes this digest before and after the file inside one transaction; any
+ * difference is a schema change — however it was caused (a CREATE/ALTER/DROP, a function, a DO
+ * block, dynamic SQL) — and rolls the whole transaction back.
+ *
+ * The guarantee rests on TWO properties of the projection:
+ *  - It sees every way the schema can move: relation kind, every column's number/type/typmod/
+ *    not-null, the full textual def of every constraint and index, each routine's args + body
+ *    hash, and each type's kind + ordered enum labels. A column TYPE change moves atttypid/
+ *    atttypmod; a DROP removes rows; a CREATE adds them; a hidden CREATE inside a called function
+ *    lands a new pg_class/pg_type row before the after-digest is taken.
+ *  - It is STABLE across pure DML. It deliberately excludes the volatile planner-stats columns
+ *    (reltuples, relpages, relallvisible, relfrozenxid, …) that INSERT/UPDATE/DELETE and
+ *    autovacuum move — so an INSERT never trips the guard, only a catalog change does.
+ *
+ * This reuses everystack's own live-catalog content-addressing idea (schema-fingerprint), scoped
+ * to the exact question db:exec asks: "did the schema move at all?" — not "does it match the
+ * models?". Server-side, no CLI/model dependency: a plain string that any runner can execute.
+ */
+
+import { escapeLiteral } from './derived-apply.js';
+
+/**
+ * Build the digest query for `schemas`. Returns SQL selecting a single `digest` column (md5 hex,
+ * or md5('') for an empty catalog). The schemas are embedded as a quoted `text[]` literal — they
+ * come from the deploy's declared schema set, never user input, but they are escaped regardless.
+ */
+export function catalogDigestQuery(schemas: string[]): string {
+  if (schemas.length === 0) {
+    throw new Error('catalogDigestQuery needs at least one schema to digest — pass the app schemas.');
+  }
+  const arr = `ARRAY[${schemas.map(escapeLiteral).join(', ')}]::text[]`;
+  return `
+SELECT md5(coalesce(string_agg(line, E'\\n' ORDER BY line), '')) AS digest
+FROM (
+  -- Relations: kind + persistence ONLY, never the volatile planner-stats columns (row-count,
+  -- page-count, freeze-horizon) — those move under INSERT/UPDATE/DELETE and autovacuum, and
+  -- would make the digest lie about pure DML.
+  SELECT format('rel:%s.%s:%s:%s', n.nspname, c.relname, c.relkind, c.relpersistence) AS line
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = ANY(${arr}) AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
+  UNION ALL
+  -- Columns: number, type, typmod, not-null — the shape ALTER TABLE moves, DML never.
+  SELECT format('att:%s.%s.%s:%s:%s:%s:%s',
+                n.nspname, c.relname, a.attname, a.attnum, a.atttypid, a.atttypmod, a.attnotnull)
+    FROM pg_attribute a
+    JOIN pg_class c ON c.oid = a.attrelid
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = ANY(${arr}) AND a.attnum > 0 AND NOT a.attisdropped
+  UNION ALL
+  -- Constraints: the full textual definition (PK/FK/unique/check predicate).
+  SELECT format('con:%s.%s:%s', n.nspname, con.conname, pg_get_constraintdef(con.oid))
+    FROM pg_constraint con JOIN pg_namespace n ON n.oid = con.connamespace
+    WHERE n.nspname = ANY(${arr})
+  UNION ALL
+  -- Indexes: the full textual definition (columns, uniqueness, partial WHERE).
+  SELECT format('idx:%s.%s:%s', n.nspname, ic.relname, pg_get_indexdef(i.indexrelid))
+    FROM pg_index i
+    JOIN pg_class ic ON ic.oid = i.indexrelid
+    JOIN pg_namespace n ON n.oid = ic.relnamespace
+    WHERE n.nspname = ANY(${arr})
+  UNION ALL
+  -- Routines: name + arg types + a hash of the body. CREATE/REPLACE/DROP FUNCTION all move it.
+  SELECT format('proc:%s.%s:%s:%s', n.nspname, p.proname, p.proargtypes::text, md5(coalesce(p.prosrc, '')))
+    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
+    WHERE n.nspname = ANY(${arr})
+  UNION ALL
+  -- Types: kind + ordered enum labels. CREATE TYPE / ALTER TYPE ADD VALUE move it.
+  SELECT format('type:%s.%s:%s:%s', n.nspname, t.typname, t.typtype,
+                coalesce((SELECT string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder)
+                          FROM pg_enum e WHERE e.enumtypid = t.oid), ''))
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+    WHERE n.nspname = ANY(${arr})
+) s
+`.trim();
+}

package/src/cli/exec-execute.ts

@@ -0,0 +1,115 @@
+/**
+ * exec-execute — the db:exec decision spine: credential-free write SQL applied as ONE transaction,
+ * bracketed by a crash-truthful audit (intent committed BEFORE the run, outcome AFTER). The IO —
+ * audit persistence and the SQL run (single txn, txn rails, rows-affected) — is injected, so this
+ * spine is proven without a DB.
+ *
+ * db:exec runs through the SAME ops connection db:seed uses (opsDb) — no new role, no BYPASSRLS,
+ * no superuser dependency; it inherits whatever RLS-bypass the deploy's admin connection already has.
+ * DML-only is enforced SEMANTICALLY, in the database, by the concrete run: the file executes in one
+ * transaction, and a catalog digest taken before vs after detects ANY schema change (however caused —
+ * a function, a DO block, dynamic SQL). If the schema moved, the file did DDL → the whole transaction
+ * ROLLS BACK and is rejected. No regex, no string-matching — the guard compares the actual catalog.
+ * This spine owns the crash-truthful audit ordering and the single-transaction invariant; the digest
+ * guard lives in the injected run (it needs the DB).
+ */
+
+import { createHash } from 'node:crypto';
+import { lex } from './derived-source.js';
+
+export interface ExecIntent {
+  /** sha256 of the exact file bytes — content identity + a free join to a consumer's stage sha. */
+  sha: string;
+  actor: string;
+  stage: string;
+}
+
+export interface ExecOutcome {
+  status: 'succeeded' | 'failed';
+  /** Per-statement rows-affected (feeds a consumer's row-count verify). */
+  rowsAffected?: number[];
+  error?: string;
+}
+
+export interface ExecuteExecOptions {
+  /** The SQL file content. */
+  sql: string;
+  actor: string;
+  stage: string;
+  /** Persist the intent row and return its id. MUST be committed before run() (crash-truthful). */
+  writeIntent: (intent: ExecIntent) => Promise<string>;
+  /** Run the SQL as the exec role in ONE transaction; returns per-statement rows-affected. */
+  run: () => Promise<number[]>;
+  /** Persist the terminal outcome against the intent id. */
+  writeOutcome: (id: string, outcome: ExecOutcome) => Promise<void>;
+}
+
+export interface ExecResult {
+  id: string;
+  status: 'succeeded';
+  rowsAffected: number[];
+}
+
+/** Statement-boundary transaction-control keywords — the spine owns the single txn, a file must not. */
+const TXN_CONTROL = /(^|;)\s*(begin|start\s+transaction|commit|rollback|savepoint|release\s+savepoint)\b/i;
+
+/**
+ * Blank comments and string/dollar-quoted content so the txn-control regex reads STRUCTURE, not
+ * payload — a `BEGIN` inside a plpgsql function body (`$$ DECLARE x int; BEGIN … $$`), a DO block,
+ * or a string literal is not transaction control and must not be rejected. Only real top-level
+ * BEGIN/COMMIT/SAVEPOINT survive into the skeleton. (Same lexing the reconciler/backfill use.)
+ */
+function txnControlSkeleton(sql: string): string {
+  let out = '';
+  lex(sql, (ch, state) => {
+    if (state === 'line-comment' || state === 'block-comment') return;
+    if (state === 'single' || state === 'dollar') return; // quoted/dollar content blanked
+    out += ch;
+  });
+  return out;
+}
+
+/**
+ * Reject a file that opens/closes its own transaction. db:exec wraps the whole file in ONE
+ * transaction (all-or-nothing, with the before/after catalog-digest guard); an embedded
+ * BEGIN/COMMIT would break that atomicity. A pure, courtesy guard — the real DML-only guarantee
+ * is the semantic catalog digest, not keyword matching — so it reads the skeleton, never the
+ * payload, and never false-positives on a BEGIN inside a function body.
+ */
+export function assertNoTxnControl(sql: string): void {
+  if (TXN_CONTROL.test(txnControlSkeleton(sql))) {
+    throw new Error(
+      'db:exec runs the whole file as ONE transaction — remove embedded BEGIN/COMMIT/ROLLBACK/SAVEPOINT (the file is atomic; the spine owns the transaction).',
+    );
+  }
+}
+
+/** sha256 hex of the exact file bytes — the content identity stamped on the audit row. */
+export function execSha(sql: string): string {
+  return createHash('sha256').update(sql).digest('hex');
+}
+
+/**
+ * Run the ceremony: reject self-transacting files → commit an intent row → run → record the outcome.
+ * A run that throws still records a FAILED outcome (never a silent loss), then rethrows.
+ */
+export async function executeExec(opts: ExecuteExecOptions): Promise<ExecResult> {
+  // Reject a self-transacting file BEFORE any record is written — db:exec wraps the whole file in one
+  // transaction (with the before/after catalog-digest guard), so an embedded BEGIN/COMMIT would break
+  // both the atomicity and the guard. The DDL check itself is semantic and lives in the injected run.
+  assertNoTxnControl(opts.sql);
+
+  const sha = execSha(opts.sql);
+  // Intent FIRST, committed before the run: a run that dies mid-flight still leaves a truthful
+  // record of what was attempted (the M1 fix for the #25 crash-hole).
+  const id = await opts.writeIntent({ sha, actor: opts.actor, stage: opts.stage });
+
+  try {
+    const rowsAffected = await opts.run();
+    await opts.writeOutcome(id, { status: 'succeeded', rowsAffected });
+    return { id, status: 'succeeded', rowsAffected };
+  } catch (err: any) {
+    await opts.writeOutcome(id, { status: 'failed', error: String(err?.message ?? err) });
+    throw err;
+  }
+}

package/src/cli/exec-log.ts

@@ -0,0 +1,63 @@
+/**
+ * exec-log — the db:exec ledger (`everystack.exec_log`), a per-database record of every file run.
+ *
+ * The audit is CRASH-TRUTHFUL: the intent row (sha, actor, stage, started_at) is committed on its
+ * own — BEFORE the file's transaction opens — so a run that dies mid-flight (SIGKILL, a Lambda
+ * timeout) still leaves a truthful record of what was attempted. The M1 fix for the #25 crash-hole.
+ * The outcome (status, per-statement rows-affected, finished_at, error) is written AFTER, against
+ * the intent's id. The ledger lives in the `everystack` schema — never in the app schemas the
+ * digest guard covers — so it is invisible to the guard and untouched by the file txn's rollback.
+ *
+ * Mirrors everystack.backfill_log: a per-database ledger the way schema_log is a write-only memoir.
+ */
+
+import { escapeLiteral } from './derived-apply.js';
+
+export const ENSURE_EXEC_LOG_SQL: string[] = [
+  'CREATE SCHEMA IF NOT EXISTS everystack',
+  `CREATE TABLE IF NOT EXISTS everystack.exec_log (
+  id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+  started_at timestamptz NOT NULL DEFAULT now(),
+  finished_at timestamptz,
+  sha text NOT NULL,
+  actor text,
+  stage text,
+  status text NOT NULL DEFAULT 'started',
+  rows_affected integer[],
+  error text
+)`,
+];
+
+export interface ExecIntentRow {
+  /** sha256 of the exact file bytes — content identity + a free join to a consumer's stage sha. */
+  sha: string;
+  actor?: string | null;
+  stage?: string | null;
+}
+
+export interface ExecOutcomeRow {
+  status: 'succeeded' | 'failed';
+  /** Per-statement rows-affected (feeds a consumer's row-count verify). */
+  rowsAffected?: number[];
+  error?: string | null;
+}
+
+const opt = (v: string | null | undefined): string => (v == null ? 'NULL' : escapeLiteral(v));
+
+/** The intent INSERT — committed BEFORE the run. RETURNING id so the outcome can update this row. */
+export function renderExecIntentInsert(intent: ExecIntentRow): string {
+  return `INSERT INTO everystack.exec_log (sha, actor, stage, status)
+VALUES (${escapeLiteral(intent.sha)}, ${opt(intent.actor)}, ${opt(intent.stage)}, 'started')
+RETURNING id`;
+}
+
+/** The terminal outcome UPDATE against the intent id. */
+export function renderExecOutcomeUpdate(id: string, outcome: ExecOutcomeRow): string {
+  const rows =
+    outcome.rowsAffected == null
+      ? 'NULL'
+      : `ARRAY[${outcome.rowsAffected.map((n) => String(Math.trunc(n))).join(', ')}]::integer[]`;
+  return `UPDATE everystack.exec_log
+SET status = ${escapeLiteral(outcome.status)}, rows_affected = ${rows}, error = ${opt(outcome.error)}, finished_at = now()
+WHERE id = ${escapeLiteral(id)}`;
+}

package/src/cli/exec-run.ts

@@ -0,0 +1,109 @@
+/**
+ * exec-run — the concrete db:exec run, inside ONE transaction, guarded by the catalog digest.
+ *
+ * `runExecInTx` runs against a transaction that the caller has already opened (the `run` closure
+ * is pinned to a single connection — drizzle `.transaction()` in the ops Lambda, postgres.js
+ * `sql.begin` on the direct path). It:
+ *   1. sets the txn rails (statement/lock/idle timeouts) so a runaway file can't wedge the plane,
+ *   2. digests the app-schema catalog BEFORE the file,
+ *   3. executes every statement in order, capturing per-statement rows-affected,
+ *   4. digests AFTER, and
+ *   5. throws SchemaChangedError if the digest moved — the file did DDL, so the caller ROLLS the
+ *      whole transaction back and nothing lands.
+ *
+ * This is the semantic DML-only enforcement: not a keyword regex (theatre — `SELECT ddl_fn()`,
+ * `DO $$…$$`, dynamic SQL slip past) but a before/after comparison of the actual catalog. It
+ * needs no BYPASSRLS role and no superuser — it inherits the ops connection db:seed already uses.
+ *
+ * The orchestration is driver-agnostic and unit-tested with a fake `run`; the real behaviour
+ * (against PostgreSQL) is in __tests__/integration/exec.test.ts.
+ */
+
+import { splitSqlStatements } from './derived-source.js';
+import { catalogDigestQuery } from './exec-digest.js';
+
+/** The normalized result of one statement: its rows and how many it affected. */
+export interface ExecStmtResult {
+  rows: any[];
+  count: number;
+}
+
+/** A statement runner pinned to the open transaction (one connection). */
+export type TxRunner = (stmt: string) => Promise<ExecStmtResult>;
+
+/** The DML-only violation: the file moved the schema catalog, so the whole txn was rolled back. */
+export class SchemaChangedError extends Error {
+  readonly before: string;
+  readonly after: string;
+  constructor(before: string, after: string) {
+    super(
+      `db:exec is DML-only — the file changed the schema catalog (digest ${before.slice(0, 12)} → ${after.slice(0, 12)}). ` +
+        'The whole transaction was rolled back; nothing was applied. Schema changes go through db:apply / db:reconcile, not db:exec.',
+    );
+    this.name = 'SchemaChangedError';
+    this.before = before;
+    this.after = after;
+  }
+}
+
+/** Transaction rails — a file can't hang the plane or block behind a lock forever. */
+export interface ExecRails {
+  statementTimeoutMs?: number;
+  lockTimeoutMs?: number;
+  idleInTxnMs?: number;
+}
+
+export const EXEC_RAIL_DEFAULTS: Required<ExecRails> = {
+  statementTimeoutMs: 300_000,
+  lockTimeoutMs: 30_000,
+  idleInTxnMs: 60_000,
+};
+
+/** The SET LOCAL rail statements — SET LOCAL so they die with the transaction, never leak. */
+export function railStatements(rails: ExecRails = {}): string[] {
+  const r = { ...EXEC_RAIL_DEFAULTS, ...rails };
+  return [
+    `SET LOCAL statement_timeout = ${Math.trunc(r.statementTimeoutMs)}`,
+    `SET LOCAL lock_timeout = ${Math.trunc(r.lockTimeoutMs)}`,
+    `SET LOCAL idle_in_transaction_session_timeout = ${Math.trunc(r.idleInTxnMs)}`,
+  ];
+}
+
+export interface RunExecInTxOptions {
+  /** The app schemas the digest guard covers (the deploy's declared set). */
+  schemas: string[];
+  rails?: ExecRails;
+}
+
+/**
+ * Run the file inside the caller's open transaction. Returns per-statement rows-affected on
+ * success; throws SchemaChangedError (→ caller rolls back) if the file changed the catalog.
+ */
+export async function runExecInTx(
+  run: TxRunner,
+  fileSql: string,
+  opts: RunExecInTxOptions,
+): Promise<number[]> {
+  for (const rail of railStatements(opts.rails)) await run(rail);
+
+  const digestQuery = catalogDigestQuery(opts.schemas);
+  const before = readDigest(await run(digestQuery));
+
+  const rowsAffected: number[] = [];
+  for (const stmt of splitSqlStatements(fileSql)) {
+    const res = await run(stmt);
+    rowsAffected.push(res.count);
+  }
+
+  const after = readDigest(await run(digestQuery));
+  if (before !== after) throw new SchemaChangedError(before, after);
+  return rowsAffected;
+}
+
+function readDigest(res: ExecStmtResult): string {
+  const digest = res.rows?.[0]?.digest;
+  if (typeof digest !== 'string') {
+    throw new Error('catalog digest query returned no digest — cannot enforce DML-only; refusing to run.');
+  }
+  return digest;
+}

package/src/cli/index.ts

@@ -21,11 +21,13 @@ import { dbApplyCommand } from './commands/db-apply.js';
 import { dbCheckCommand } from './commands/db-check.js';
 import { dbApproversCommand } from './commands/db-approvers.js';
 import { dbBackfillCommand } from './commands/db-backfill.js';
+import { dbExecCommand } from './commands/db-exec.js';
+import { taskProbeCommand } from './commands/task-probe.js';
 import { pipelineRunCommand, pipelineListCommand } from './commands/pipeline-run.js';
 import { dbTemplateRefreshCommand, dbBranchCommand } from './commands/db-branch.js';
 import { dbForkCommand } from './commands/db-fork.js';
 import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
-import { dbBackupProbeCommand, dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
+import { dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
 import { dbExportCommand } from './commands/db-export.js';
 import { dbSwapCommand } from './commands/db-swap.js';
 import { consoleCommand } from './commands/console.js';
@@ -172,9 +174,6 @@ async function main() {
     case 'db:snapshots':
       await dbSnapshotsCommand(flags);
       break;
-    case 'db:backup:probe':
-      await dbBackupProbeCommand(flags);
-      break;
     case 'db:backup':
       await dbBackupCommand(flags);
       break;
@@ -231,6 +230,15 @@ async function main() {
     case 'db:backfill':
       await dbBackfillCommand(flags);
       break;
+    case 'db:exec': {
+      // The SQL file is a positional (`db:exec <file.sql>`); `-`/absent reads stdin.
+      const file = args[1] && !args[1].startsWith('-') ? args[1] : undefined;
+      await dbExecCommand(flags, file);
+      break;
+    }
+    case 'task:probe':
+      await taskProbeCommand(flags);
+      break;
     case 'pipeline:run':
       await pipelineRunCommand(flags);
       break;
@@ -360,7 +368,6 @@ Usage:
   everystack db:provision      --stage <name> [--direct | --database-url <url>]  Create the least-privilege role chain on an EXISTING database (idempotent; creates no DB). No flag = ops-Lambda venue (auto-falls to direct via the ADMIN_DATABASE_URL secret if the handler has no dbPlugin). --direct = direct connection reading that secret (master never on argv); --database-url = explicit URL. Declared schemas are set as the ROLE default (ALTER ROLE … SET search_path) — the secret stays credentials-only, and the URL is libpq-clean
   everystack db:snapshot       [--stage <name>] [--instance <id>]  Take a physical RDS snapshot (instant DR point; RDS only — use db:backup for portable logical backups)
   everystack db:snapshots      [--stage <name>] [--instance <id>]  List manual RDS snapshots for the instance
-  everystack db:backup:probe   [--stage <name>]  Verify the pg_dump layer is attached + version-compatible with the server
   everystack db:backup         [--stage <name>]  Logical pg_dump of the DB → private S3 backups bucket (prints the backup id)
   everystack db:backups        [--stage <name>]  List logical backups (id, size, created)
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
@@ -380,6 +387,8 @@ Usage:
   everystack db:check          [--models <barrel>] [--schema-out <file.ts>] [--database-url <url>] [--json]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, descriptors compile), every exposed RLS-enabled table must declare a read path (no force-RLS-with-no-read landmine that goes dark on the superuser drop), and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
   everystack db:approvers      --stage <name> [--set "cto,arn:..."] [--remove]  Declare who can DESTROY: the stage's destructive-approver set (SSM parameter, admin-writable). Destructive db:apply runs are then identity-verified (STS) against it; --set '' disables destructive applies; --remove returns the stage to ceremony-only
   everystack db:backfill       [--database-url <url>] [--dir db/backfills] [--apply] [--mark-applied <file.sql>] [--json]  One-shot data jobs in their own lane: plan shows applied (by CONTENT identity — renames/comment edits are no-ops) / pending (in order, unbounded-pass advisories) / blocked (a name that already ran in a different form — one-shot jobs are immutable). --apply runs each pending job as its own transaction, recorded in everystack.backfill_log (a failure rolls back alone, is recorded, stops the run); --mark-applied records without running. Never runs as a schema side effect; direct connection required
+  everystack db:exec           <file.sql> [--stage <name> | --database-url <url>] [--schemas a,b] [--confirm]  Credential-free write SQL — the psql-piped-file replacement (or pipe SQL on stdin). Runs the whole file as ONE transaction, DML-only enforced SEMANTICALLY: a catalog digest before vs after rejects ANY schema change (however caused — a function, a DO block, dynamic SQL) by rolling the txn back. Schema changes go through db:apply / db:reconcile. Every run is recorded crash-truthfully in everystack.exec_log (intent before, per-statement rows-affected after). --stage runs in the ops Lambda (operator holds no URL); prod-tier writes require --confirm; --database-url is local-direct (--schemas scopes the digest, default public)
+  everystack task:probe        --stage <name>  Smoke-test the ephemeral Task lane: dispatch the version-handshake task (credential-free, via the ops Lambda) and poll until it stops. A pass proves the substrate end to end — the task started, resolved its injected operator credential, reached the private DB, and the image's pg tools are compatible with the live server. Recorded in everystack.task_log
   everystack pipeline:run      [--stage <name> | --database-url <url>] [--rebuild | --curate] [--only <substr> | --from <id> | --to <id>] [--resume [--run-id <id>]] [--dry-run] [--continue-on-error] [--json]  Reproduce data from committed source: run the pipeline's stages in topological order, each idempotent and inside its own transaction, recorded in everystack.pipeline_log. Lanes: --rebuild (automated+frozen) / --curate (curated) / neither (all in order). --resume picks up the latest run, skipping applied stages. With --stage the orchestrator runs credential-free IN the ops Lambda (no URL held; heavy stages refused — run those local); --database-url is local-direct and unbounded
   everystack pipeline:list     [--rebuild | --curate] [--only <substr> | --from <id> | --to <id>]  Show the pipeline's stages in run order (topological) with their lane and dependency edges — no execution, no database
   everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy

package/src/cli/task-poll.ts

@@ -0,0 +1,67 @@
+/**
+ * task-poll — poll an ephemeral Task run (dispatched via the ops Lambda) until it stops.
+ *
+ * Shared by task:probe and the pg-binary verbs (db:backup / db:export, later restore/swap): they all
+ * dispatch a task, get back a run id + ARN, then poll `task:status` until STOPPED. The poll is
+ * BOUNDED — a Fargate task can sit in PROVISIONING/PENDING on capacity or ENI trouble, and a naked
+ * loop would hang the CLI. A few consecutive DescribeTasks blips are tolerated (a throttle shouldn't
+ * abort a live task); past that, or the deadline, the caller reconciles via the run id (the task_log
+ * row + ECS both carry it). This owns the loop; the caller owns the success/failure messaging.
+ */
+
+import { invokeAction } from './aws.js';
+import { info } from './output.js';
+
+const POLL_INTERVAL_MS = 5_000;
+/** Backup/export/restore can run minutes on large databases — far longer than the probe's handshake. */
+export const DEFAULT_DEADLINE_MS = 30 * 60_000;
+const MAX_CONSECUTIVE_ERRORS = 3;
+
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+export interface TaskStatus {
+  lastStatus?: string;
+  stopped?: boolean;
+  exitCode?: number | null;
+  stoppedReason?: string | null;
+  /** The task's self-reported result row (db:backup/export write id/key/bytes/fingerprint). */
+  result?: Record<string, unknown> | null;
+  error?: string;
+}
+
+export type TaskPollResult =
+  | { outcome: 'stopped'; status: TaskStatus }
+  | { outcome: 'error'; status: TaskStatus }
+  | { outcome: 'timeout'; lastStatus: string };
+
+/**
+ * Poll until the task stops, printing each lifecycle transition. Returns `stopped` (read exitCode),
+ * `error` (task:status failed repeatedly — the task may still be running), or `timeout`.
+ */
+export async function pollTaskUntilStopped(
+  region: string,
+  fn: string,
+  ids: { runId: string; taskArn: string },
+  opts: { deadlineMs?: number } = {},
+): Promise<TaskPollResult> {
+  const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS);
+  let last = '';
+  let consecutiveErrors = 0;
+  while (Date.now() < deadline) {
+    const status = (await invokeAction(region, fn, 'task:status', { runId: ids.runId, taskArn: ids.taskArn })) as TaskStatus;
+    if (status?.error) {
+      if (++consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) return { outcome: 'error', status };
+      info(`  (status check blipped: ${status.error} — retrying)`);
+      await sleep(POLL_INTERVAL_MS);
+      continue;
+    }
+    consecutiveErrors = 0;
+    if (status.lastStatus && status.lastStatus !== last) {
+      info(`  ${status.lastStatus}`);
+      last = status.lastStatus;
+    }
+    if (status.stopped) return { outcome: 'stopped', status };
+    await sleep(POLL_INTERVAL_MS);
+  }
+  return { outcome: 'timeout', lastStatus: last || 'unknown' };
+}

package/src/exec.ts

@@ -0,0 +1,20 @@
+/**
+ * @everystack/cli/exec — the db:exec core, for the ops-Lambda lane.
+ *
+ * db:exec applies credential-free write SQL as ONE transaction, DML-only by a semantic catalog
+ * digest (no role, no superuser, no regex), bracketed by a crash-truthful ledger. This barrel lets
+ * the ops `db:exec` action (in @everystack/server's dbPlugin) load the same core the CLI uses and
+ * run it on the operator connection db:seed already uses — so `db:exec --stage` needs no raw admin
+ * URL on the operator's machine.
+ */
+
+export { executeExec, assertNoTxnControl, execSha } from './cli/exec-execute.js';
+export type { ExecIntent, ExecOutcome, ExecuteExecOptions, ExecResult } from './cli/exec-execute.js';
+
+export { runExecInTx, SchemaChangedError, railStatements, EXEC_RAIL_DEFAULTS } from './cli/exec-run.js';
+export type { TxRunner, ExecStmtResult, ExecRails, RunExecInTxOptions } from './cli/exec-run.js';
+
+export { catalogDigestQuery } from './cli/exec-digest.js';
+
+export { ENSURE_EXEC_LOG_SQL, renderExecIntentInsert, renderExecOutcomeUpdate } from './cli/exec-log.js';
+export type { ExecIntentRow, ExecOutcomeRow } from './cli/exec-log.js';