@everystack/cli 0.4.24 → 0.4.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.24",
+  "version": "0.4.27",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-refresh.ts

@@ -10,7 +10,7 @@
  * (the `db:refresh` action), so a least-privileged operator never holds a URL. `--database-url`
  * / `--direct` refresh over a direct connection (dev). `--list` previews the topo-ordered
  * matviews without connecting. Plain REFRESH (ACCESS EXCLUSIVE) — CONCURRENTLY is a follow-on.
- * (GridironDB 2026-07-19.)
+ * (consumer field report 2026-07-19.)
  */
 
 import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
@@ -30,6 +30,9 @@ export async function dbRefreshCommand(flags: Record<string, string>): Promise<v
     process.exit(1);
   }
   const only = flags.only ? flags.only.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
+  // --verify-nonempty: after refreshing, gate on populated-but-zero-rows matviews (the dark-panel
+  // check) — credential-free, a scoped EXISTS on only what was just refreshed.
+  const verifyNonempty = flags['verify-nonempty'] === 'true';
   const matviews = matviewIdentities(declared?.objects ?? [], only);
 
   if (matviews.length === 0) {
@@ -65,14 +68,14 @@ export async function dbRefreshCommand(flags: Record<string, string>): Promise<v
       step(connectingVia(dbSource));
       const conn = await createUrlRunner(dbSource.url);
       end = conn.end;
-      step(`Refreshing ${matviews.length} materialized view(s)...`);
-      run = await executeRefresh(conn.runner, { matviews });
+      step(`Refreshing ${matviews.length} materialized view(s)${verifyNonempty ? ' (verifying non-empty)' : ''}...`);
+      run = await executeRefresh(conn.runner, { matviews, verifyNonempty });
     } else {
       step('Resolving deployed config...');
       const config = await resolveConfig(flags.stage);
       info(`Region: ${config.region}, Function: ${opsFunction(config)}`);
-      step(`Refreshing ${matviews.length} materialized view(s) in the ops Lambda...`);
-      const result: any = await invokeAction(config.region, opsFunction(config), 'db:refresh', { matviews });
+      step(`Refreshing ${matviews.length} materialized view(s) in the ops Lambda${verifyNonempty ? ' (verifying non-empty)' : ''}...`);
+      const result: any = await invokeAction(config.region, opsFunction(config), 'db:refresh', { matviews, verifyNonempty });
       if (result?.error) {
         fail(`db:refresh failed: ${result.error}`);
         if (/Unknown action/i.test(String(result.error))) {
@@ -80,7 +83,7 @@ export async function dbRefreshCommand(flags: Record<string, string>): Promise<v
         }
         process.exit(1);
       }
-      run = { refreshed: result.refreshed ?? [], statements: result.statements ?? [], failed: result.failed ?? undefined };
+      run = { refreshed: result.refreshed ?? [], statements: result.statements ?? [], failed: result.failed ?? undefined, empty: result.empty ?? undefined };
     }
 
     for (const id of run.refreshed) success(`refreshed ${id}`);
@@ -89,7 +92,13 @@ export async function dbRefreshCommand(flags: Record<string, string>): Promise<v
       info(`${run.refreshed.length} of ${matviews.length} refreshed before the failure; fix and re-run (refresh is idempotent).`);
       process.exit(1);
     }
-    success(`db:refresh — ${run.refreshed.length} materialized view(s) refreshed.`);
+    // The dark-panel gate: refreshed cleanly but came back empty (inputs not loaded). Refuse.
+    if (run.empty?.length) {
+      fail(`db:refresh --verify-nonempty: ${run.empty.length} matview(s) refreshed but are EMPTY (populated, zero rows) — a dark panel: ${run.empty.join(', ')}`);
+      info('These refreshed without error but serve no rows — their inputs were likely not loaded. Load the inputs and re-refresh before promoting.');
+      process.exit(1);
+    }
+    success(`db:refresh — ${run.refreshed.length} materialized view(s) refreshed${verifyNonempty ? ', all non-empty' : ''}.`);
     process.exit(0);
   } finally {
     if (end) await end().catch(() => {});

package/src/cli/commands/db.ts

@@ -363,7 +363,7 @@ export function declaredSearchPath(schemas: string[]): string[] {
  * descriptors, NOT the barrel, so a table-only scan structurally misses it — and the
  * authenticator flip then goes dark on every bare-ref matview read ("relation does not
  * exist"). Baking the union into DATABASE_URL's search_path keeps those reads resolving.
- * (GridironDB 2026-07-19: 54 matviews in `stats_view`, absent from the table-only bake.)
+ * (consumer field report 2026-07-19: 54 matviews in `stats_view`, absent from the table-only bake.)
  */
 export function collectDeclaredSchemas(args: {
   models: Array<{ schema?: string }>;
@@ -379,7 +379,7 @@ export function collectDeclaredSchemas(args: {
  * defaults to `"$user",public`, so every bare-ref query 404s until a hand ALTER ROLE pin
  * (recorded nowhere, repeated every stage) — writing it into the minted DATABASE_URL is
  * declarative and survives redeploys. postgres.js forwards unknown URL params as connection
- * startup params (GridironDB proved search_path this way). No-op for an all-public app.
+ * startup params (a consumer proved search_path this way). No-op for an all-public app.
  */
 export function withSearchPath(url: string, schemas: string[]): string {
   const path = declaredSearchPath(schemas);
@@ -389,6 +389,31 @@ export function withSearchPath(url: string, schemas: string[]): string {
   return u.toString();
 }
 
+/**
+ * Carry the CONNECTION params (sslmode, connect_timeout, …) from the URL the app already connects
+ * with onto a freshly minted one. provision mints the new URL from host/port/database components,
+ * which silently drops any `?sslmode=require` on the source — on an `rds.force_ssl=1` instance the
+ * new authenticator/migrator roles then can't connect. `search_path` is provision's OWN concern
+ * (withSearchPath owns it), so it is never copied; existing params on the minted URL win. No-op
+ * when there is no source or it doesn't parse.
+ */
+export function preserveConnParams(mintedUrl: string, sourceUrl?: string): string {
+  if (!sourceUrl) return mintedUrl;
+  let src: URL;
+  let out: URL;
+  try {
+    src = new URL(sourceUrl);
+    out = new URL(mintedUrl);
+  } catch {
+    return mintedUrl;
+  }
+  for (const [key, value] of src.searchParams) {
+    if (key === 'search_path') continue; // provision derives search_path from the declared schemas
+    if (!out.searchParams.has(key)) out.searchParams.set(key, value);
+  }
+  return out.toString();
+}
+
 /** host/port/database from a postgres URL — the direct venue's connection info. */
 export function parseUrlConnection(url: string): { host: string; port: string; database: string } {
   const u = new URL(url);
@@ -575,16 +600,22 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
   const searchPath = declaredSearchPath(declaredSchemas);
   if (searchPath.length) info(`  search_path baked into DATABASE_URL: ${searchPath.join(', ')}`);
 
-  const authUrl = withSearchPath(
-    `postgresql://${result.loginRole}:${authPassword}@${conn.host}:${conn.port ?? 5432}/${conn.database}`,
-    declaredSchemas,
-  );
-  const adminUrl = result.adminRole
-    ? withSearchPath(
-        `postgresql://${result.adminRole}:${adminPassword}@${conn.host}:${conn.port ?? 5432}/${conn.database}`,
-        declaredSchemas,
-      )
-    : undefined;
+  // Preserve the connection params (sslmode, …) from the URL the app already connects with — the
+  // minted URL is built from bare host/port/db components, so without this a `?sslmode=require`
+  // would be dropped and the new roles could fail to connect on an SSL-forced RDS. Prefer the
+  // venue URL provision connected through, else the existing DATABASE_URL secret (either spelling).
+  const sourceUrl = directUrl
+    ?? stageSecrets.ADMIN_DATABASE_URL ?? stageSecrets.AdminDatabaseUrl
+    ?? stageSecrets.DATABASE_URL ?? stageSecrets.DatabaseUrl;
+
+  const mint = (role: string, password: string): string =>
+    withSearchPath(
+      preserveConnParams(`postgresql://${role}:${password}@${conn.host}:${conn.port ?? 5432}/${conn.database}`, sourceUrl),
+      declaredSchemas,
+    );
+
+  const authUrl = mint(result.loginRole, authPassword);
+  const adminUrl = result.adminRole ? mint(result.adminRole, adminPassword) : undefined;
 
   // Write the secrets directly to the SST secret store — never display them. One blob
   // write updates every key together, so there is no window where the API side is the

package/src/cli/discover.ts

@@ -26,6 +26,7 @@ interface DiscoveredConfig {
   updatesBucket: string;
   clientBundlesBucket: string;
   mediaBucket?: string;
+  backupsBucket?: string;
   kvsArn?: string;
   distributionId?: string;
 }
@@ -209,6 +210,7 @@ interface BucketsResult {
   updatesBucket?: string;
   clientBundlesBucket?: string;
   mediaBucket?: string;
+  backupsBucket?: string;
 }
 
 /**
@@ -231,10 +233,15 @@ export async function discoverBuckets(
   // Media bucket resource is named 'Media' (not 'MediaBucket'), so match both patterns
   const mediaNeedle = `${prefix}media-`.toLowerCase();
   const mediaBucketNeedle = `${prefix}mediabucket-`.toLowerCase();
+  // Backups bucket (db:backup/restore/backup:download). Discovered like every other bucket, so the
+  // --stage path carries it — db:backup:download presigns CLIENT-side and needs the name locally
+  // (db:backup/backups run server-side in the ops Lambda, which is why only download failed).
+  const backupsNeedle = `${prefix}backupsbucket-`.toLowerCase();
 
   let updatesBucket: string | undefined;
   let clientBundlesBucket: string | undefined;
   let mediaBucket: string | undefined;
+  let backupsBucket: string | undefined;
 
   for (const b of buckets) {
     if (!b.Name) continue;
@@ -242,10 +249,11 @@ export async function discoverBuckets(
     if (lower.startsWith(updatesNeedle)) updatesBucket = b.Name;
     if (lower.startsWith(clientNeedle)) clientBundlesBucket = b.Name;
     if (!mediaBucket && (lower.startsWith(mediaNeedle) || lower.startsWith(mediaBucketNeedle))) mediaBucket = b.Name;
-    if (updatesBucket && clientBundlesBucket && mediaBucket) break;
+    if (lower.startsWith(backupsNeedle)) backupsBucket = b.Name;
+    if (updatesBucket && clientBundlesBucket && mediaBucket && backupsBucket) break;
   }
 
-  return { updatesBucket, clientBundlesBucket, mediaBucket };
+  return { updatesBucket, clientBundlesBucket, mediaBucket, backupsBucket };
 }
 
 // ---------------------------------------------------------------------------
@@ -304,14 +312,16 @@ export async function discoverConfig(
     );
   }
 
-  // Media bucket fallback: if not found via naming convention (custom/pre-existing bucket),
-  // try reading from .sst/outputs.json where sst.config.ts return values land.
+  // Media/backups bucket fallback: if not found via naming convention (custom/pre-existing
+  // bucket), try reading from .sst/outputs.json where sst.config.ts return values land.
   let mediaBucket = bucketsResult.mediaBucket;
-  if (!mediaBucket) {
+  let backupsBucket = bucketsResult.backupsBucket;
+  if (!mediaBucket || !backupsBucket) {
     try {
       const raw = await fs.readFile(path.resolve('.sst', 'outputs.json'), 'utf8');
       const outputs = JSON.parse(raw);
-      if (outputs.mediaBucket) mediaBucket = outputs.mediaBucket;
+      if (!mediaBucket && outputs.mediaBucket) mediaBucket = outputs.mediaBucket;
+      if (!backupsBucket && outputs.backupsBucket) backupsBucket = outputs.backupsBucket;
     } catch {
       // outputs.json not available — non-fatal
     }
@@ -327,6 +337,7 @@ export async function discoverConfig(
     updatesBucket: bucketsResult.updatesBucket,
     clientBundlesBucket: bucketsResult.clientBundlesBucket,
     mediaBucket,
+    backupsBucket,
     kvsArn: cfResult.kvsArn,
     distributionId,
   };
@@ -337,7 +348,7 @@ export async function discoverConfig(
 // ---------------------------------------------------------------------------
 
 const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
-const CACHE_VERSION = 4; // Bump when CachedOutputs schema changes to invalidate stale caches
+const CACHE_VERSION = 5; // Bump when CachedOutputs schema changes to invalidate stale caches (5: +backupsBucket)
 
 interface CachedOutputs {
   _cachedAt: number;
@@ -350,6 +361,7 @@ interface CachedOutputs {
   updatesBucket?: string;
   clientBundlesBucket?: string;
   mediaBucket?: string;
+  backupsBucket?: string;
   kvsArn?: string;
   distributionId?: string;
 }
@@ -377,6 +389,7 @@ export async function getCachedConfig(stage: string): Promise<DiscoveredConfig |
       updatesBucket: cached.updatesBucket,
       clientBundlesBucket: cached.clientBundlesBucket,
       mediaBucket: cached.mediaBucket,
+      backupsBucket: cached.backupsBucket,
       kvsArn: cached.kvsArn,
       distributionId: cached.distributionId,
     };
@@ -452,6 +465,7 @@ export async function setCachedConfig(stage: string, config: DiscoveredConfig):
     updatesBucket: config.updatesBucket,
     clientBundlesBucket: config.clientBundlesBucket,
     mediaBucket: config.mediaBucket,
+    backupsBucket: config.backupsBucket,
     kvsArn: config.kvsArn,
     distributionId: config.distributionId,
   };

package/src/cli/index.ts

@@ -372,7 +372,7 @@ Usage:
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
   everystack db:fingerprint    [--stage <name> | --database-url <url>] [--models <barrel>] [--json]  Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
   everystack db:reconcile      [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--only a,b] [--json]  Reconcile the derived layer (functions/views/matviews/triggers) against the DECLARED descriptors (defineView/defineMaterializedView/defineFunction/defineSql/trigger() on models, from the barrel) — the single home (db/sql is retired; leftover .sql files fail with the migration path): plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (atomic — DDL + provenance in one transaction) and records provenance + schema_log; --apply --stage runs credential-free in the ops Lambda (no admin URL on the deployer, the db:apply twin), --apply --database-url runs direct. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive. --only <schema.name,…> restricts the run to the named objects (surgical); with --rebuild it FORCES those to rebuild from source even when the hashes show no diff — the recovery exit when a mistaken --rebaseline left a self-consistent-but-wrong provenance row (the dependency cascade rebuilds their live dependents).
-  everystack db:refresh        [--stage <name> | --database-url <url> | --direct] [--only a,b] [--list]  Refresh the declared materialized views in dependency order, credential-free. --stage runs the whole refresh in the ops Lambda on the operator connection (no URL on the operator's machine — the data-lane twin of the reconcile lane); --database-url/--direct refresh over a direct connection (dev); --only refreshes a named subset (full identity or bare name); --list previews the order without connecting. Plain REFRESH (ACCESS EXCLUSIVE); fail-fast, idempotent to re-run.
+  everystack db:refresh        [--stage <name> | --database-url <url> | --direct] [--only a,b] [--verify-nonempty] [--list]  Refresh the declared materialized views in dependency order, credential-free. --stage runs the whole refresh in the ops Lambda on the operator connection (no URL on the operator's machine — the data-lane twin of the reconcile lane); --database-url/--direct refresh over a direct connection (dev); --only refreshes a named subset (full identity or bare name); --verify-nonempty gates on populated-but-zero-rows matviews (the dark-panel check — a scoped EXISTS on just what was refreshed, so a promotion script gets the gate with no read authority) and FAILS naming any empty; --list previews the order without connecting. Plain REFRESH (ACCESS EXCLUSIVE); fail-fast, idempotent to re-run.
   everystack db:sync           [--database-url <url>] [--models <barrel>] [--schema-out <file.ts>] [--allow-drops] [--overwrite-drift] [--baseline] [--json]  Make the database match your checkout — one verb, both layers: apply the state diff (tables+authz, one transaction, verified by re-diff), reconcile the derived layer against the declared descriptors, report the resulting fingerprint vs the models' declared one. Dev databases only (direct connection required); DROPs held back unless --allow-drops; derived drift refuses unless --overwrite-drift; exit 1 when not converged
   everystack db:diff           --from-models <barrel> [--to-models db/models/index.ts] [--allow-drops] [--check] [--json]  The state edge between two declared states, NO database: the SQL db:generate would produce, computed purely — CI plan previews (--check exits 1 on a non-empty edge) and computed rollbacks (swap the flags)
   everystack db:plan           [--stage <name> | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -]  Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only — works via the ops Lambda; plans are ephemeral, never committed

package/src/cli/refresh-execute.ts

@@ -6,7 +6,7 @@
  * path. This is that verb: refresh the declared matviews in dependency order, over whatever
  * runner the venue supplies. The ops `db:refresh` action loads executeRefresh here and runs it
  * on the operator connection, so `db:refresh --stage` needs no raw URL on the operator's machine
- * (GridironDB 2026-07-19, the data-lane twin of the reconcile lane).
+ * (consumer field report 2026-07-19, the data-lane twin of the reconcile lane).
  *
  * Plain `REFRESH MATERIALIZED VIEW` (ACCESS EXCLUSIVE) — the faithful twin of the direct-connect
  * data script it replaces. CONCURRENTLY (needs a unique index; doesn't block reads) is a
@@ -25,6 +25,19 @@ export interface RefreshRun {
   statements: string[];
   /** Fail-fast: the matview whose refresh threw, stopping the run. Earlier ones already committed. */
   failed?: { identity: string; error: string };
+  /**
+   * `--verify-nonempty`: refreshed matviews that came back populated-but-ZERO-rows — the
+   * dark-panel trap (a matview refreshed before its inputs loaded; `ispopulated` is true, so
+   * it's not a refresh error, but it serves nothing). Only present when verifyNonempty was set;
+   * a non-empty list is a promotion-gate failure the venue turns into a refusal.
+   */
+  empty?: string[];
+}
+
+/** Coerce a Postgres boolean (true | 't' | 'true' | 1) to a JS boolean — postgres.js and
+ *  drizzle disagree on the encoding, so accept both. */
+function pgTrue(v: unknown): boolean {
+  return v === true || v === 't' || v === 'true' || v === 1 || v === '1';
 }
 
 /**
@@ -45,8 +58,16 @@ export function matviewIdentities(objects: SourceObject[], only?: string[]): str
  * the run and is reported (earlier refreshes have already committed — each REFRESH is its own
  * statement, no wrapping transaction, so CONCURRENTLY stays possible later). An empty list is a
  * clean no-op.
+ *
+ * `verifyNonempty` (opt-in): after a FULLY-successful refresh, probe each matview it refreshed
+ * with `SELECT EXISTS (SELECT 1 …)` and return the populated-but-zero-rows ones in `empty`. This
+ * is the credential-free dark-panel gate — a scoped read of only what was just refreshed, never
+ * arbitrary db:query authority. Skipped when the refresh itself failed (that failure owns the run).
  */
-export async function executeRefresh(runner: RefreshRunner, opts: { matviews: string[] }): Promise<RefreshRun> {
+export async function executeRefresh(
+  runner: RefreshRunner,
+  opts: { matviews: string[]; verifyNonempty?: boolean },
+): Promise<RefreshRun> {
   const refreshed: string[] = [];
   const statements: string[] = [];
   for (const identity of opts.matviews) {
@@ -59,5 +80,15 @@ export async function executeRefresh(runner: RefreshRunner, opts: { matviews: st
       return { refreshed, statements, failed: { identity, error: err?.message || String(err) } };
     }
   }
+
+  if (opts.verifyNonempty && refreshed.length) {
+    const empty: string[] = [];
+    for (const identity of refreshed) {
+      const rows = await runner(`SELECT EXISTS (SELECT 1 FROM ${quoteQualified(identity)}) AS has_rows`);
+      if (!pgTrue(rows[0]?.has_rows)) empty.push(identity);
+    }
+    if (empty.length) return { refreshed, statements, empty };
+  }
+
   return { refreshed, statements };
 }