@everystack/cli 0.4.89 → 0.4.90

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.89",
+  "version": "0.4.90",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-apply.ts

@@ -266,7 +266,22 @@ async function runCatchUp(
       onStart: () => step('Catching the functions, views and triggers up to the checkout...'),
       onRefusal,
     });
-    if (verbose && outcome.run) for (const text of buildReconcileReport(outcome.run.plan)) info(text);
+    if (outcome.run) {
+      if (verbose) {
+        for (const text of buildReconcileReport(outcome.run.plan)) info(text);
+      } else if (outcome.run.plan.needsBaseline.length > 0) {
+        // NOT gated behind --verbose. A first-contact match (an object that predates Models
+        // tracking) is silently SKIPPED here — the reconciler will not guess whether live
+        // matches source — so a catch-up that only ever prints its full report under --verbose
+        // reads as "nothing to do" while real brownfield objects are left permanently
+        // unverified. This is the exact silence a consumer's derived-layer incident traced back
+        // to: db:apply reported success on every deploy while these objects sat unreconciled.
+        warn(
+          `${outcome.run.plan.needsBaseline.length} derived object(s) skipped (never reconciled, so this catch-up cannot tell whether live matches source): ${outcome.run.plan.needsBaseline.join(', ')} — ` +
+          'run `everystack db:reconcile --baseline` to record trust once you have verified it matches, or `--rebuild` to force it from source. Does not fail this apply.',
+        );
+      }
+    }
     return { ok: true, statements: outcome.statements };
   } catch (err: any) {
     const cause = String(err?.message ?? err);

package/src/cli/commands/db-plan.ts

@@ -64,7 +64,10 @@ import { resolveConfig, opsFunction } from '../config';
 import { resolveOperatorUrlViaStage } from '../direct-venue';
 import { lambdaQueryRunner, lambdaSessionRunner } from '../aws';
 import { loadModels } from './db-generate';
-import { loadDeclaredDerived } from '../declared-derived';
+import { loadDeclaredDerived, type DeclaredDerived } from '../declared-derived';
+import { introspectDerived } from '../derived-introspect';
+import { planReconcile } from '../derived-plan';
+import { checkFails, buildReconcileReport } from './db-reconcile';
 import { governedRoleSet, ungovernedGrants } from '../authz-reconcile';
 import { readBaselineFile, checkAgainstBaseline, baselineRefusal } from '../authz-baseline';
 import { alterTypeStatementTargets, findAlterTypeDependents, renderAlterTypeRefusal } from '../alter-type-dependents';
@@ -269,10 +272,13 @@ export async function dbPlanCommand(flags: Record<string, string>): Promise<void
     // The modules' extensions ride the same read. Without them the plan omits every
     // CREATE EXTENSION the target lacks, and the apply fails on a type the plan promised.
     let declaredExtensions: string[] = [];
+    // Kept for the derived-layer visibility check below — this IS the declared derived layer
+    // (functions/views/matviews/triggers), read once and reused rather than loaded twice.
+    let declaredDerived: DeclaredDerived | null = null;
     try {
-      const declared = await loadDeclaredDerived(flags.models);
-      declaredGovernedRoles = declared?.governedRoles ?? [];
-      declaredExtensions = declared?.extensions ?? [];
+      declaredDerived = await loadDeclaredDerived(flags.models);
+      declaredGovernedRoles = declaredDerived?.governedRoles ?? [];
+      declaredExtensions = declaredDerived?.extensions ?? [];
     } catch {
       // The barrel's own compose errors surface on the paths that need the derived layer;
       // a plan must not fail to mint because a module could not be read for this one field.
@@ -456,6 +462,33 @@ export async function dbPlanCommand(flags: Record<string, string>): Promise<void
       }
     }
 
+    // DERIVED-LAYER VISIBILITY — db:apply ALWAYS reconciles functions/views/matviews/triggers
+    // against the models, whether or not a --plan is passed; the edge this command mints never
+    // covers that layer. Without this, an empty table/column/grant edge reads as "the target
+    // matches the checkout" even when a function body changed underneath it — an operator who
+    // trusts an empty plan never learns db:apply still has work to do. Read-only: the same pure
+    // planReconcile() that backs `db:reconcile --check`, applied to nothing here — advisory
+    // only, like the FK-orphan and backfill checks above, so it can go stale between mint and
+    // apply without corrupting the plan file (see the file header's rationale for why nothing
+    // read-time-only belongs in the artifact).
+    if (declaredDerived) {
+      try {
+        const liveDerived = await introspectDerived(session);
+        const derivedPlan = planReconcile(
+          { objects: declaredDerived.objects, warnings: [] },
+          liveDerived,
+          { renamedTables: declaredDerived.renamedTables },
+        );
+        if (checkFails(derivedPlan)) {
+          warn('derived layer out of sync (functions/views/matviews/triggers) — this plan does not preview these, only tables/columns/grants:');
+          for (const line of buildReconcileReport(derivedPlan)) warn(`  ${line}`);
+          warn('db:apply always catches up the derived layer, --plan or not — run it to reconcile the above.');
+        }
+      } catch (err: any) {
+        warn(`derived-layer check could not run against this target (${err?.message ?? err}) — db:plan cannot say whether functions/views/matviews/triggers are in sync.`);
+      }
+    }
+
     await reportPipelineLastRun(runner);
 
     // THE DEFAULT REPORT — the only thing a reader sees when nothing is wrong.

package/src/cli/index.ts

@@ -331,8 +331,8 @@ Usage:
   everystack db:refresh        [--stage <name> | --database-url <url> | --direct] [--only a,b] [--verify-nonempty] [--list]  Refresh the declared materialized views in dependency order, credential-free. --stage runs the whole refresh in the ops Lambda on the operator connection (no URL on the operator's machine — the data-lane twin of the reconcile lane); --database-url refreshes over a local connection (dev); --stage <name> --direct resolves the stage's operator connection from its ops Lambda (credential never on argv) and refreshes CLI-side on an UNBOUNDED clock, for a matview set that exceeds the ops-Lambda 900s limit; --only refreshes a named subset (full identity or bare name); --verify-nonempty gates on populated-but-zero-rows matviews (the dark-panel check — a scoped EXISTS on just what was refreshed, so a promotion script gets the gate with no read authority) and FAILS naming any empty; --list previews the order without connecting. Plain REFRESH (ACCESS EXCLUSIVE); fail-fast, idempotent to re-run.
   everystack db:sync           [--database-url <url>] [--models <barrel>] [--schema-out <file.ts>] [--allow-drops] [--overwrite-drift] [--baseline] [--json]  Make the database match your checkout — one verb, both layers: apply the state diff (tables+authz, one transaction, verified by re-diff), reconcile the derived layer against the declared descriptors, report the resulting fingerprint vs the models' declared one. Dev databases only (direct connection required); DROPs held back unless --allow-drops; derived drift refuses unless --overwrite-drift; exit 1 when not converged
   everystack db:diff           --from-models <barrel> [--to-models db/models/index.ts] [--allow-drops] [--check] [--json]  The state edge between two declared states, NO database: the SQL db:generate would produce, computed purely — CI plan previews (--check exits 1 on a non-empty edge) and computed rollbacks (swap the flags)
-  everystack db:plan           [--stage <name> [--direct] | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -] [--verbose]  Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only; plans are ephemeral, never committed. VENUES: --stage runs via the ops Lambda, which reads TWICE and refuses on disagreement — agreement there is a DETECTOR, not a verification. --stage --direct resolves the stage's operator connection from its IAM-gated ops Lambda, holds it in memory only, and reads ONCE over one session: the lane for a fingerprint you intend to trust, with the credential never on argv. --database-url is the local-dev venue (same read guarantee, but against a deployed stage it puts a privileged DSN on the command line) DEFAULT OUTPUT is the plain change list: '+' something is now true, '-' something is no longer true, and a '⚠ DESTRUCTIVE' line of its own when real data can be lost. Nothing else. --verbose restores the full internal report (edge summary, kind histogram, adoption and ownership reports, plan ref)
-  everystack db:apply          [--plan <file.plan.json>] [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--state-only] [--snapshot-ref <ref>] [--attest-lineage-lost <sha>] [--verbose]  THE deploy verb — it does the WHOLE job: the reviewed plan (tables, columns, grants), then ALWAYS the functions, views, matviews and triggers, against the same target, reported as one receipt. With NO --plan it does the catch-up alone (this is what replaces running db:reconcile --apply by hand). --state-only skips the catch-up for the narrow case that wants the tables alone. If the plan lands and the catch-up then fails, the exit is non-zero and the message says the table/column changes are committed and safe and gives the rerun, which retries only the catch-up. Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule — it protects MERGED work: a commit on any ref that declares the live state and is not an ancestor of HEAD refuses by name, escaped git-natively by fetch + merge/rebase + re-plan; --attest-lineage-lost <sha> waives it ONLY for a declaring commit proven gone from every ref. When NO commit on ANY ref declares the live state — a database everystack never wrote, or one hand-edited since its last apply — the rule has nothing to protect: over a full, current clone that is PROVEN DRIFT, the reviewed plan is the ORIGIN, it applies with a loud warning, and the origin is recorded in schema_log so every later apply descends from it. A shallow, partial, or behind-remote clone, or a models tree that will not compile, is UNVERIFIABLE and refuses with a fetch/prune runbook), and for DESTRUCTIVE plans require --confirm always + a confirmed rollback point + the stage's approver set when declared (STS identity-verified). The STAGE lane (--stage without --direct) runs every catalog query in its own ops-Lambda invoke, so one read can be assembled from several containers: it reads the target TWICE and REFUSES when the two disagree (inconsistent containers), reports agreement as a NON-DETECTION (it cannot verify read consistency), and REFUSES a DESTRUCTIVE plan outright — destructive applies go over --database-url (direct) with the full ceremony. Every refusal that reaches the ops Lambda is recorded in schema_log. Apply as one transaction (plan_ref stamped), verify it landed exactly on plan.to; idempotent when already there DEFAULT OUTPUT is the receipt: the same change lines back, then 'Applied N changes.'. --verbose restores the endpoint and plan-ref line
+  everystack db:plan           [--stage <name> [--direct] | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -] [--verbose]  Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only; plans are ephemeral, never committed. VENUES: --stage runs via the ops Lambda, which reads TWICE and refuses on disagreement — agreement there is a DETECTOR, not a verification. --stage --direct resolves the stage's operator connection from its IAM-gated ops Lambda, holds it in memory only, and reads ONCE over one session: the lane for a fingerprint you intend to trust, with the credential never on argv. --database-url is the local-dev venue (same read guarantee, but against a deployed stage it puts a privileged DSN on the command line) DEFAULT OUTPUT is the plain change list: '+' something is now true, '-' something is no longer true, and a '⚠ DESTRUCTIVE' line of its own when real data can be lost. Nothing else. --verbose restores the full internal report (edge summary, kind histogram, adoption and ownership reports, plan ref). The edge covers tables/columns/grants ONLY — it also reads (never mints) whether the DERIVED layer (functions/views/matviews/triggers) is out of sync with the target and warns if so, since db:apply always reconciles that layer regardless of --plan: an empty edge here does not mean the target fully matches the checkout
+  everystack db:apply          [--plan <file.plan.json>] [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--state-only] [--snapshot-ref <ref>] [--attest-lineage-lost <sha>] [--verbose]  THE deploy verb — it does the WHOLE job: the reviewed plan (tables, columns, grants), then ALWAYS the functions, views, matviews and triggers, against the same target, reported as one receipt. With NO --plan it does the catch-up alone (this is what replaces running db:reconcile --apply by hand). --state-only skips the catch-up for the narrow case that wants the tables alone. A brownfield derived object the reconciler has never touched before (no provenance row) is SKIPPED, never guessed at, and that skip prints by default (not only under --verbose): 'N derived object(s) skipped ... db:reconcile --baseline' — a plain apply that only showed this under --verbose read as full success while such objects sat permanently unreconciled. If the plan lands and the catch-up then fails, the exit is non-zero and the message says the table/column changes are committed and safe and gives the rerun, which retries only the catch-up. Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule — it protects MERGED work: a commit on any ref that declares the live state and is not an ancestor of HEAD refuses by name, escaped git-natively by fetch + merge/rebase + re-plan; --attest-lineage-lost <sha> waives it ONLY for a declaring commit proven gone from every ref. When NO commit on ANY ref declares the live state — a database everystack never wrote, or one hand-edited since its last apply — the rule has nothing to protect: over a full, current clone that is PROVEN DRIFT, the reviewed plan is the ORIGIN, it applies with a loud warning, and the origin is recorded in schema_log so every later apply descends from it. A shallow, partial, or behind-remote clone, or a models tree that will not compile, is UNVERIFIABLE and refuses with a fetch/prune runbook), and for DESTRUCTIVE plans require --confirm always + a confirmed rollback point + the stage's approver set when declared (STS identity-verified). The STAGE lane (--stage without --direct) runs every catalog query in its own ops-Lambda invoke, so one read can be assembled from several containers: it reads the target TWICE and REFUSES when the two disagree (inconsistent containers), reports agreement as a NON-DETECTION (it cannot verify read consistency), and REFUSES a DESTRUCTIVE plan outright — destructive applies go over --database-url (direct) with the full ceremony. Every refusal that reaches the ops Lambda is recorded in schema_log. Apply as one transaction (plan_ref stamped), verify it landed exactly on plan.to; idempotent when already there DEFAULT OUTPUT is the receipt: the same change lines back, then 'Applied N changes.'. --verbose restores the endpoint and plan-ref line
   everystack db:build          --database-url <url> [--models <barrel>]  Build a database FROM the models and KEEP it — the fresh-database bootstrap a deleted migration folder is replaced BY. Runs the same compose db:check proves buildability with (extensions -> schemas -> contract roles -> declared functions -> state -> derived layer), against a real target, then reports MATCH. Venue is EXPLICIT: --database-url only, never the ambient environment, because this writes a whole schema. REFUSES a database that already holds objects — evolving an existing one is db:sync (dev) or db:plan -> db:apply (a stage). Contract roles are created NOLOGIN if absent, and roles are cluster-level
   everystack db:check          [--models <barrel>] [--schema-out <file.ts>] [--database-url <url>] [--json] [--verbose]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, descriptors compile), every exposed RLS-enabled table must declare a read path (no force-RLS-with-no-read landmine that goes dark on the superuser drop), and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target DEFAULT OUTPUT is ONE line: 'Up to date.' or 'N changes pending. Run db:plan to review.'. --verbose restores every finding and the from-scratch build report
   everystack db:approvers      --stage <name> [--set "cto,arn:..."] [--remove]  Declare who can DESTROY: the stage's destructive-approver set (SSM parameter, admin-writable). Destructive db:apply runs are then identity-verified (STS) against it; --set '' disables destructive applies; --remove returns the stage to ceremony-only