@everystack/cli 0.4.61 → 0.4.65

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.61",
+  "version": "0.4.65",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -23,70 +23,87 @@
   ],
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./src/index.ts",
       "default": "./src/index.ts"
     },
     "./model": {
+      "source": "./src/cli/model-api.ts",
       "types": "./src/cli/model-api.ts",
       "default": "./src/cli/model-api.ts"
     },
     "./audit": {
+      "source": "./src/cli/audit-api.ts",
       "types": "./src/cli/audit-api.ts",
       "default": "./src/cli/audit-api.ts"
     },
     "./apply": {
+      "source": "./src/cli/apply-execute.ts",
       "types": "./src/cli/apply-execute.ts",
       "default": "./src/cli/apply-execute.ts"
     },
     "./db-source": {
+      "source": "./src/cli/db-source.ts",
       "types": "./src/cli/db-source.ts",
       "default": "./src/cli/db-source.ts"
     },
     "./reconcile": {
+      "source": "./src/reconcile.ts",
       "types": "./src/reconcile.ts",
       "default": "./src/reconcile.ts"
     },
     "./refresh": {
+      "source": "./src/refresh.ts",
       "types": "./src/refresh.ts",
       "default": "./src/refresh.ts"
     },
     "./backfill": {
+      "source": "./src/backfill.ts",
       "types": "./src/backfill.ts",
       "default": "./src/backfill.ts"
     },
     "./exec": {
+      "source": "./src/exec.ts",
       "types": "./src/exec.ts",
       "default": "./src/exec.ts"
     },
     "./audit/source": {
+      "source": "./src/cli/audit-source-api.ts",
       "types": "./src/cli/audit-source-api.ts",
       "default": "./src/cli/audit-source-api.ts"
     },
     "./runbook": {
+      "source": "./src/cli/runbook/index.ts",
       "types": "./src/cli/runbook/index.ts",
       "default": "./src/cli/runbook/index.ts"
     },
     "./client": {
+      "source": "./src/client/index.ts",
       "types": "./src/client/index.ts",
       "default": "./src/client/index.ts"
     },
     "./schema": {
+      "source": "./src/schema.ts",
       "types": "./src/schema.ts",
       "default": "./src/schema.ts"
     },
     "./handler": {
+      "source": "./src/handler/index.ts",
       "types": "./src/handler/index.ts",
       "default": "./src/handler/index.ts"
     },
     "./plugin": {
+      "source": "./src/plugin.ts",
       "types": "./src/plugin.ts",
       "default": "./src/plugin.ts"
     },
     "./env": {
+      "source": "./src/env.ts",
       "types": "./src/env.ts",
       "default": "./src/env.js"
     },
     "./sst": {
+      "source": "./src/sst.ts",
       "types": "./src/sst.ts",
       "default": "./src/sst.ts"
     }
@@ -160,7 +177,7 @@
     "jest": "29.7.0",
     "react": "19.2.0",
     "ts-jest": "29.4.9",
-    "@everystack/server": "0.4.22"
+    "@everystack/server": "0.4.26"
   },
   "scripts": {
     "test": "jest",

package/src/cli/aws.ts

@@ -734,10 +734,14 @@ export async function describeRdsSnapshots(
  *
  * The call is a no-code-upload configuration change — it completes in ~2s,
  * and Lambda drains in-flight requests before retiring each environment.
+ *
+ * Optional extraEnv merges additional env vars (e.g. rotated credentials).
+ * Existing env vars are preserved (read-modify-write).
  */
 export async function rollingRestart(
   region: string,
   functionName: string,
+  extraEnv?: Record<string, string>,
 ): Promise<{ previousGen: string; newGen: string }> {
   const client = await getLambda(region);
   const {
@@ -755,7 +759,7 @@ export async function rollingRestart(
   await client.send(new UpdateFunctionConfigurationCommand({
     FunctionName: functionName,
     Environment: {
-      Variables: { ...env, EVERYSTACK_GEN: newGen },
+      Variables: { ...env, ...extraEnv, EVERYSTACK_GEN: newGen },
     },
   }));
 

package/src/cli/commands/db.ts

@@ -258,7 +258,7 @@ export function buildProvisionSecretPlan(args: {
  *  same seam as pipeline-loader). */
 export interface ServerProvision {
   runProvision(
-    payload: { authPassword?: string; adminPassword?: string; searchPath?: string[] },
+    payload: { authPassword?: string; adminPassword?: string; searchPath?: string[]; rotationPairs?: boolean },
     deps: {
       execute: (sql: string) => Promise<unknown>;
       connection?: { host: string; port?: number | string; database: string } | null;
@@ -471,12 +471,277 @@ export function parseUrlConnection(url: string): { host: string; port: string; d
   };
 }
 
+/**
+ * db:provision --rotate — swap the active A/B role for one or both credential slots.
+ *
+ * Steps: preflight master → detect active → provision inactive → write SSM →
+ * rolling restart → verify origin → retire old → drain sessions.
+ */
+async function rotateCredentials(flags: Record<string, string>): Promise<void> {
+  const stage = flags.stage!;
+  const graceMs = (flags.grace ? Number(flags.grace) : 60) * 1000;
+
+  step('Resolving deployed config...');
+  let config;
+  try {
+    config = await resolveConfig(stage);
+  } catch (err: any) {
+    fail(err.message);
+    process.exit(1);
+  }
+
+  const { loadSstSecrets, putSstSecrets } = await import('../utils/secrets');
+  const { parseAppName } = await import('../discover');
+  const secretsCtx = { appName: await parseAppName(), stage, region: config.region };
+  let stageSecrets: Record<string, string> = {};
+  try {
+    stageSecrets = await loadSstSecrets(secretsCtx);
+  } catch (err: any) {
+    fail(`Cannot read stage secrets: ${err.message}`);
+    process.exit(1);
+  }
+
+  // Step 0: preflight — verify the operator connection works (break-glass path).
+  step('Preflight: verifying operator connection...');
+  const opsFn = opsFunction(config);
+  try {
+    const result = await invokeAction(config.region, opsFn, 'db:query', { sql: 'SELECT current_user' });
+    if ((result as any)?.error) throw new Error((result as any).error);
+    success('Operator connection verified.');
+  } catch (err: any) {
+    fail(`Operator connection failed: ${err.message}`);
+    info('Fix the operator connection before rotating.');
+    process.exit(1);
+  }
+
+  // All DB operations run through the ops Lambda (inside the VPC).
+  const rotate = async (action: string, payload?: Record<string, unknown>) => {
+    const result = await invokeAction(config.region, opsFn, 'db:rotate', { action, ...payload });
+    if ((result as any)?.error) throw new Error((result as any).error);
+    return result as any;
+  };
+
+  try {
+
+    // Detect which slots to rotate.
+    const slotFlag = flags.slot || 'both';
+    if (!['api', 'ops', 'both'].includes(slotFlag)) {
+      fail(`Invalid --slot: ${slotFlag}. Use api, ops, or both.`);
+      process.exit(1);
+    }
+    const slots: Array<{ base: string; secretKeys: string[] }> = [];
+    if (slotFlag === 'api' || slotFlag === 'both') {
+      slots.push({ base: 'authenticator', secretKeys: ['DatabaseUrl', 'ReadDatabaseUrl'] });
+    }
+    if (slotFlag === 'ops' || slotFlag === 'both') {
+      slots.push({ base: 'migrator', secretKeys: ['AdminDatabaseUrl'] });
+    }
+
+    const { randomBytes } = await import('node:crypto');
+
+    // Resolve host/port/database from the ops Lambda (same as db:provision does).
+    let conn: { host: string; port: string; database: string } | undefined;
+    try {
+      const psqlInfo = await invokeAction(config.region, opsFn, 'db:psql', {});
+      if ((psqlInfo as any)?.host) conn = psqlInfo as any;
+    } catch { /* handled below */ }
+    if (!conn?.host) {
+      fail('Could not resolve database host from the ops Lambda.');
+      process.exit(1);
+    }
+    const sourceUrl = stageSecrets.ADMIN_DATABASE_URL ?? stageSecrets.AdminDatabaseUrl
+      ?? stageSecrets.DATABASE_URL ?? stageSecrets.DatabaseUrl;
+
+    // Track the old active role for each slot so retire uses the RIGHT role (F1 fix).
+    const oldActiveRoles: Array<{ base: string; oldRole: string }> = [];
+    // Track credential env overrides to set during rolling restart.
+    const credEnvOverrides: Record<string, string> = {};
+
+    for (const { base, secretKeys } of slots) {
+      step(`Rotating ${base}...`);
+
+      const detected = await rotate('detect', { baseRole: base });
+      if (!detected || (!detected.activeRole && !detected.inactiveRole)) {
+        fail(`No rotation pairs found for ${base}. Run db:provision --rotation-pairs first.`);
+        process.exit(1);
+      }
+      info(`  Active: ${detected.activeRole}, inactive: ${detected.inactiveRole}`);
+      oldActiveRoles.push({ base, oldRole: detected.activeRole });
+
+      // Provision the inactive role via the ops Lambda.
+      const password = randomBytes(24).toString('hex');
+      const provisioned = await rotate('provision', { baseRole: base, password });
+      success(`  ${detected.inactiveRole} provisioned with new password.`);
+
+      // Build the new URL.
+      const newUrl = preserveConnParams(
+        `postgresql://${detected.inactiveRole}:${password}@${conn.host}:${conn.port}/${conn.database}`,
+        sourceUrl,
+      );
+
+      // Write to the SST secret store (blob-first per Fable ruling — must
+      // complete before UpdateFunctionConfiguration so a crash at any point
+      // is recovered by re-running rotate or by deploying).
+      step(`  Updating secrets (${secretKeys.join(', ')})...`);
+      try {
+        const secrets = { ...stageSecrets };
+        for (const key of secretKeys) {
+          let credUrl = newUrl;
+          if (key === 'ReadDatabaseUrl') {
+            const existingReadUrl = stageSecrets.READ_DATABASE_URL ?? stageSecrets.ReadDatabaseUrl;
+            if (existingReadUrl) {
+              try {
+                const u = new URL(newUrl);
+                u.hostname = new URL(existingReadUrl).hostname;
+                credUrl = u.toString();
+              } catch { /* use primary host */ }
+            }
+          }
+          secrets[key] = credUrl;
+          credEnvOverrides[`EVERYSTACK_CRED_${key}`] = credUrl;
+        }
+        await putSstSecrets(secretsCtx, secrets);
+        stageSecrets = secrets;
+        success(`  Secrets written.`);
+      } catch (err: any) {
+        fail(`  Secret write failed: ${err.message}`);
+        info('  Ensure your IAM role has s3:PutObject + kms:Encrypt on the SST state bucket.');
+        process.exit(1);
+      }
+    }
+
+    // Rolling restart all linked functions with credential env overrides.
+    // Each function gets only the credentials it links — preserving the split.
+    // Track failures on credential-carrying functions — if ANY fail, abort
+    // before retire (both roles keep LOGIN = safe re-runnable state).
+    const { rollingRestart } = await import('../aws');
+    step('Rolling restart...');
+    const apiCreds: Record<string, string> = {};
+    const opsCreds: Record<string, string> = {};
+    for (const [k, v] of Object.entries(credEnvOverrides)) {
+      if (k.includes('Admin')) {
+        opsCreds[k] = v;
+      } else {
+        apiCreds[k] = v;
+      }
+    }
+    const allCreds = { ...apiCreds, ...opsCreds };
+    const functions: Array<{ name: string; creds: Record<string, string>; critical: boolean }> = [
+      { name: config.apiFunctionName, creds: apiCreds, critical: true },
+      ...(config.opsFunctionName ? [{ name: config.opsFunctionName, creds: allCreds, critical: true }] : []),
+      ...(config.workerFunctionName ? [{ name: config.workerFunctionName, creds: {}, critical: false }] : []),
+      ...(config.imageFunctionName ? [{ name: config.imageFunctionName, creds: {}, critical: false }] : []),
+    ];
+
+    let restartFailed = false;
+    for (const { name: fn, creds, critical } of functions) {
+      try {
+        const { newGen } = await rollingRestart(config.region, fn, creds);
+        success(`  ${fn} → gen ${newGen}`);
+      } catch (err: any) {
+        if (critical) {
+          fail(`  ${fn}: ${err.message}`);
+          restartFailed = true;
+        } else {
+          warn(`  ${fn}: ${err.message}`);
+        }
+      }
+    }
+
+    if (restartFailed) {
+      fail('Restart failed on a credential-carrying function. Both roles keep LOGIN (safe state).');
+      info('Fix the issue and re-run db:provision --rotate. No credential was revoked.');
+      process.exit(1);
+    }
+
+    // Verify the new credential can log in — prove it works before revoking
+    // the old one. Runs inside the VPC via the ops Lambda.
+    step('Verifying new credential...');
+    const newApiUrl = credEnvOverrides.EVERYSTACK_CRED_DatabaseUrl;
+    if (newApiUrl) {
+      try {
+        await rotate('probe-login', { url: newApiUrl });
+        success('New credential connects to the database.');
+      } catch (err: any) {
+        fail(`New credential login failed: ${err.message}`);
+        info('Both roles keep LOGIN (safe state). Fix and re-run.');
+        process.exit(1);
+      }
+    }
+
+    // Origin HTTP check (non-blocking — the login probe is the real gate).
+    if (config.baseUrl) {
+      try {
+        const probe = `${config.baseUrl}?_probe=${Date.now()}`;
+        const res = await fetch(probe, { redirect: 'follow' });
+        if (res.ok) success('Origin responding.');
+        else warn(`Origin returned ${res.status}.`);
+      } catch (err: any) {
+        warn(`Origin probe failed: ${err.message}`);
+      }
+    }
+
+    // Retire old roles and drain sessions — using the roles recorded at
+    // detection time, NOT re-detected (F1 fix: both roles have LOGIN now,
+    // re-detection would pick the wrong one via the tiebreak).
+    for (const { base, oldRole } of oldActiveRoles) {
+      step(`Retiring ${oldRole}...`);
+      await rotate('retire', { role: oldRole });
+      success(`  ${oldRole} set to NOLOGIN.`);
+
+      // Also retire the base role if it still has LOGIN (F6 fix).
+      try {
+        const baseDetect = await rotate('detect', { baseRole: base });
+        if (baseDetect === null) {
+          // Base role check — query directly via db:query
+          const baseResult = await invokeAction(config.region, opsFn, 'db:query', {
+            sql: `SELECT rolcanlogin FROM pg_roles WHERE rolname = '${base}'`,
+          });
+          const rows = (baseResult as any)?.rows ?? baseResult;
+          if (Array.isArray(rows) && rows[0]?.rolcanlogin) {
+            await rotate('retire', { role: base });
+            success(`  ${base} (base role) set to NOLOGIN.`);
+          }
+        }
+      } catch { /* best-effort */ }
+
+      step(`Draining ${oldRole} sessions (grace: ${graceMs / 1000}s)...`);
+      const deadline = Date.now() + graceMs;
+      const POLL_MS = 2000;
+      while (Date.now() < deadline) {
+        const { count } = await rotate('count-sessions', { role: oldRole });
+        if (count === 0) { success(`  ${oldRole} drained cleanly.`); break; }
+        await new Promise((r) => setTimeout(r, POLL_MS));
+      }
+      const { count: remaining } = await rotate('count-sessions', { role: oldRole });
+      if (remaining > 0) {
+        const { terminated } = await rotate('terminate-sessions', { role: oldRole });
+        success(`  ${oldRole} drained (${terminated} session(s) terminated after grace period).`);
+      }
+    }
+
+    success('Credential rotation complete.');
+
+  } catch (err: any) {
+    if (err && (err as any).code !== 'EEXIT') {
+      fail(`Rotation error: ${err.message}`);
+      process.exit(1);
+    }
+    throw err;
+  }
+}
+
 export async function dbProvisionCommand(flags: Record<string, string>): Promise<void> {
   if (!flags.stage) {
     fail('--stage is required for db:provision (it writes the stage\'s DATABASE_URL secret — the new credentials are never displayed, so they must land in the secret store)');
     process.exit(1);
   }
 
+  if (flags.rotate) {
+    await rotateCredentials(flags);
+    return;
+  }
+
   step('Resolving deployed config...');
   let config;
   try {
@@ -558,7 +823,7 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
     const { createUrlRunner } = await import('../db-source');
     const { runner, end } = await createUrlRunner(url);
     try {
-      return await runProvision({ authPassword, adminPassword, searchPath }, {
+      return await runProvision({ authPassword, adminPassword, searchPath, rotationPairs: !!flags['rotation-pairs'] }, {
         execute: async (statement) => runner(statement),
         connection: parseUrlConnection(url),
         // Same probe as the ops-Lambda venue: a real login as the new role, DDL-capable.
@@ -591,7 +856,7 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
     conn = parseUrlConnection(directUrl);
   } else {
     try {
-      result = await invokeAction(config.region, opsFunction(config), 'db:provision', { authPassword, adminPassword, searchPath });
+      result = await invokeAction(config.region, opsFunction(config), 'db:provision', { authPassword, adminPassword, searchPath, rotationPairs: !!flags['rotation-pairs'] });
     } catch (err: any) {
       // Auto-fallback: no Ops Lambda (Unknown action), but an ADMIN_DATABASE_URL secret exists —
       // provision directly over it. Master never crosses the CLI; the operator sets nothing new.

package/src/cli/commands/diag.ts

@@ -85,7 +85,7 @@ export async function diagCommand(
       cacheStatus: response.headers.get('x-cache') ?? undefined,
       cacheAge: response.headers.get('age') ?? undefined,
       cacheControl: response.headers.get('cache-control') ?? undefined,
-      serverDebug: response.headers.get('x-server-debug') ?? undefined,
+      serverDebug: response.headers.get('x-everystack-release') ?? response.headers.get('x-everystack-created') ?? undefined,
     };
 
     // Fetch full HTML if hydration analysis requested
@@ -138,7 +138,7 @@ export async function diagCommand(
   if (diag.serverDebug) {
     console.log(`  Serving Release:  ${diag.serverDebug}`);
   } else {
-    warn('  No X-Server-Debug header found. Update @everystack/server to get release timestamp on SSR responses.');
+    warn('  No x-everystack-release header found. Update @everystack/server (>= 0.4.22) to get release headers on SSR responses.');
   }
   console.log('');
 
@@ -168,15 +168,15 @@ export async function diagCommand(
   // 4. Version comparison
   if (!manifest) {
     warn('  Cannot compare: no manifest available.');
-  } else if (diag.serverDebug && manifest.createdAt) {
-    if (diag.serverDebug === manifest.createdAt) {
+  } else if (diag.serverDebug && manifest.updateId) {
+    if (diag.serverDebug === manifest.updateId) {
       success('  Version match: serving the latest release');
     } else {
-      fail(`  Version mismatch: serving ${diag.serverDebug}, latest is ${manifest.createdAt}`);
+      fail(`  Version mismatch: serving ${diag.serverDebug}, latest is ${manifest.updateId}`);
       info('  Run `everystack cache:purge --origin web` to bust the CDN cache.');
     }
   } else {
-    warn('  Cannot compare: missing X-Server-Debug header or manifest timestamp.');
+    warn('  Cannot compare: missing x-everystack-release header or manifest updateId.');
   }
 
   console.log('');

package/src/cli/index.ts

@@ -353,7 +353,8 @@ Usage:
   everystack db:psql           --stage <name>  Interactive ADMIN psql (IAM-gated; resolves the admin URL in-process)
   everystack db:psql           [--stage <name>] -c <command>  Run one query via Lambda (works for private RDS)
   everystack db:doctor         [--stage <name>] [--direct | --database-url <api> [--admin-database-url <ops>]]  Check the DB is least-privilege + RLS-subject (api vs operator connection). No flag = ops-Lambda venue (auto-falls to direct via the stage secrets if the handler has no dbPlugin). --direct = probe both connections from the stage's DATABASE_URL + ADMIN_DATABASE_URL secrets; --database-url = explicit local venue (a lone URL is probed as both)
-  everystack db:provision      --stage <name> [--direct | --database-url <url>]  Create the least-privilege role chain on an EXISTING database (idempotent; creates no DB). No flag = ops-Lambda venue (auto-falls to direct via the ADMIN_DATABASE_URL secret if the handler has no dbPlugin). --direct = direct connection reading that secret (master never on argv); --database-url = explicit URL. Declared schemas are set as the ROLE default (ALTER ROLE … SET search_path) — the secret stays credentials-only, and the URL is libpq-clean
+  everystack db:provision      --stage <name> [--direct | --database-url <url>] [--rotation-pairs]  Create the least-privilege role chain (idempotent). --rotation-pairs creates A/B pairs for live rotation
+  everystack db:provision      --rotate --stage <name> [--slot api|ops|both] [--grace 60]  Rotate credentials live (A/B role swap, no deploy, no downtime)
   everystack db:snapshot       [--stage <name>] [--instance <id>]  Take a physical RDS snapshot (instant DR point; RDS only — use db:backup for portable logical backups)
   everystack db:snapshots      [--stage <name>] [--instance <id>]  List manual RDS snapshots for the instance
   everystack db:backup         [--stage <name>]  Logical pg_dump of the DB → private S3 backups bucket (prints the backup id). Add --database-url <local> --stage <target> to dump a LOCAL database and upload it into the target stage's bucket as a restorable backup — the credential-free break-glass local→stage full replace (then: db:restore --from <id> --stage <target>)