@everystack/server 0.4.6 → 0.4.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/server",
-  "version": "0.4.6",
+  "version": "0.4.9",
   "description": "Server runtime primitives for Lambda — event adapters, routing, SSR, image processing",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -43,6 +43,10 @@
       "types": "./src/migrate.ts",
       "default": "./src/migrate.ts"
     },
+    "./db-doctor": {
+      "types": "./src/db-doctor.ts",
+      "default": "./src/db-doctor.ts"
+    },
     "./worker": {
       "types": "./src/worker.ts",
       "default": "./src/worker.ts"
@@ -155,9 +159,9 @@
     "sst": "4.13.1",
     "ts-jest": "29.4.9",
     "typescript": "5.9.3",
+    "@everystack/cli": "0.4.23",
     "@everystack/auth": "0.4.0",
-    "@everystack/pg-tools": "0.4.0",
-    "@everystack/cli": "0.4.20"
+    "@everystack/pg-tools": "0.4.0"
   },
   "scripts": {
     "test": "jest",

package/src/db-doctor.ts

@@ -593,6 +593,88 @@ export function buildDoctorReport(probe: DoctorProbe, options: DoctorReportOptio
   };
 }
 
+/** A query runner for ONE connection — returns plain result rows. Venue-agnostic: the
+ *  dbPlugin wraps drizzle (`db.execute(sql.raw(q))`), the CLI direct venue wraps a
+ *  postgres.js client. Its errors propagate so the ambient probe can catch a fail-closed. */
+export type DoctorRunner = (query: string) => Promise<any[]>;
+
+/** The two connections db:doctor compares: the api credential under scrutiny and the
+ *  operator credential that carries the catalog introspection. */
+export interface DoctorProbeRunners {
+  /** The api (least-privilege / DATABASE_URL) connection. */
+  api: DoctorRunner;
+  /** The operator (ADMIN_DATABASE_URL) connection — sees the whole catalog. */
+  ops: DoctorRunner;
+}
+
+export interface RunDoctorProbeOptions extends DoctorReportOptions {
+  /** Names the api credential source in the connect-failure message (e.g. 'DATABASE_URL'). */
+  apiSourceLabel?: string;
+  /** The backups lens (pg_dump layer) probe — Lambda-only. Omitted on the direct venue,
+   *  where there is no layer concept, so the lens is simply not shown. */
+  backup?: () => Promise<DoctorProbe['backup']>;
+}
+
+const quoteDoctorIdent = (s: string): string => '"' + String(s).replace(/"/g, '""') + '"';
+
+/**
+ * Gather the full doctor probe over two connection runners and classify it. This is the
+ * single orchestration BOTH venues share:
+ *
+ * - the dbPlugin `db:doctor` action — drizzle runners, inside the VPC, with the pg_dump
+ *   layer backups lens and the app's dbPlugin `forceRlsCarveouts` / `functionSchemas`;
+ * - the CLI `--database-url` / `--direct` path — postgres.js runners from the operator's
+ *   shell, no layer lens (there is none locally).
+ *
+ * The api connection is probed for its identity (role, superuser, BYPASSRLS) and its
+ * ambient read; the operator connection carries every catalog query. A failure to even
+ * identify the api connection throws (the venue reports it) — every other probe error
+ * surfaces as a red check via buildDoctorReport.
+ */
+export async function runDoctorProbe(
+  runners: DoctorProbeRunners,
+  options: RunDoctorProbeOptions = {},
+): Promise<DoctorReport> {
+  const { api: apiRun, ops: opsRun } = runners;
+  const sourceLabel = options.apiSourceLabel ?? 'DATABASE_URL';
+
+  let api: ConnInfo;
+  try {
+    api = rowToConnInfo((await apiRun(CONN_SQL))[0]);
+  } catch (err: any) {
+    throw new Error(`api connection probe failed (${sourceLabel}): ${err?.message || String(err)}`);
+  }
+
+  const admin = rowToConnInfo((await opsRun(CONN_SQL))[0]);
+  const forceRls = (await opsRun(FORCE_RLS_SQL)).map(rowToForceRls);
+  const appGrantedRls = (await opsRun(appGrantsRlsSql())).map(rowToAppGranted);
+  const secdefFunctions = (await opsRun(SECDEF_FUNCTIONS_SQL)).map(rowToSecdefFn);
+  const publicExecutableFunctions = (await opsRun(PUBLIC_EXECUTABLE_FUNCTIONS_SQL)).map(rowToPublicExecutableFn);
+  const darkRelations = (await opsRun(DARK_RELATIONS_SQL)).map(rowToDarkRelation);
+
+  // Ambient-access probe: can the api connection read an RLS table WITHOUT SET ROLE? A
+  // least-privilege role must fail closed here; a role that returns rows carries ambient
+  // privilege (INHERIT + membership in a USING(true) role like the master).
+  let apiAmbientRead: { table: string; accessible: boolean } | undefined;
+  if (forceRls.length > 0) {
+    const t = forceRls[0];
+    const ref = `${t.schema}.${t.table}`;
+    try {
+      await apiRun(`SELECT 1 FROM ${quoteDoctorIdent(t.schema)}.${quoteDoctorIdent(t.table)} LIMIT 1`);
+      apiAmbientRead = { table: ref, accessible: true };
+    } catch {
+      apiAmbientRead = { table: ref, accessible: false };
+    }
+  }
+
+  const backup = options.backup ? await options.backup() : undefined;
+
+  return buildDoctorReport(
+    { api, admin, forceRls, appGrantedRls, apiAmbientRead, secdefFunctions, publicExecutableFunctions, darkRelations, backup },
+    { forceRlsCarveouts: options.forceRlsCarveouts, functionSchemas: options.functionSchemas },
+  );
+}
+
 /** Map a PUBLIC_EXECUTABLE_FUNCTIONS_SQL row to PublicExecutableFn. */
 export function rowToPublicExecutableFn(row: any): PublicExecutableFn {
   return {

package/src/db.ts

@@ -9,27 +9,19 @@
 
 import { drizzle } from 'drizzle-orm/postgres-js';
 import postgres from 'postgres';
+import { loadSstResources } from './sst-resources';
 
 /**
- * Read an SST-linked resource WITHOUT importing the `sst` package. SST injects
- * every link as `process.env.SST_RESOURCE_<Name>` (a JSON blob) — SST's own
- * Resource proxy just parses that env var (see sst/dist/resource/shared.js:
- * `raw[name] = JSON.parse(process.env['SST_RESOURCE_' + name])`). Reading it
- * directly means this module never imports `sst`, which is a deploy-time
- * construct: Metro/esbuild cannot bundle it, and SST's own build-time
- * version-check plugin mis-reads a node_module's `sst` import and fails the
- * build. So the API/SSR helpers stayed unbundleable in the exact runtime half
- * the framework targets. Returns undefined when the link is absent — every
+ * Read an SST-linked resource WITHOUT importing the `sst` package. `sst` is a
+ * deploy-time construct: importing it makes this module unbundleable (Metro) and
+ * trips SST's own build-time version-check plugin. `loadSstResources` reads every
+ * mechanism SST uses to deliver links — per-name env, the bundled JSON, and the
+ * encrypted key file (SST >= ~4.13's Lambda default) — so this resolves the same
+ * links `Resource.<Name>` would. Returns undefined when the link is absent; every
  * getter then falls through to process.env, unchanged.
  */
 function sstLink<T = Record<string, any>>(name: string): T | undefined {
-  const raw = process.env[`SST_RESOURCE_${name}`];
-  if (!raw) return undefined;
-  try {
-    return JSON.parse(raw) as T;
-  } catch {
-    return undefined;
-  }
+  return loadSstResources()[name] as T | undefined;
 }
 
 /**

package/src/plugin.ts

@@ -420,6 +420,16 @@ export function ssrPlugin(options: SsrPluginOptions = {}): (ctx: PluginContext)
     if (options.stripQueryParams) ssrOptions.stripQueryParams = options.stripQueryParams;
     if (options.injectHydrateFlag !== undefined) ssrOptions.injectHydrateFlag = options.injectHydrateFlag;
 
+    // Fail LOUD on the misconfiguration, not silent 404. Without ctx.updatesStorage the SSR
+    // fallback can never serve `/` (getWebHandler has no bundle source) — but /api still works,
+    // so it reads like a broken bundle, not a missing link. A consumer lost hours to exactly
+    // this silent null. Log once at init with the fix.
+    if (!(ctx as SsrContext).updatesStorage) {
+      log('error', 'ssrPlugin: no updatesStorage in the plugin context — every non-API route will 404. '
+        + 'Wire it in createAppContext: updatesStorage = await createStorage({ type: "s3", '
+        + 'bucket: sstResource<{name:string}>("Updates")?.name }) and LINK the Updates bucket to this function.');
+    }
+
     return async (request: Request): Promise<Response> => {
       const webHandler = await getWebHandler(
         (ctx as SsrContext).updatesStorage,
@@ -684,82 +694,63 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
     // (api still on the master), BYPASSRLS, and un-FORCEd RLS. See db-doctor.ts.
     actions['db:doctor'] = async (_payload, ctx) => {
       const { sql } = await import('drizzle-orm');
-      const {
-        buildDoctorReport, rowToConnInfo, rowToForceRls, rowToAppGranted, rowToSecdefFn,
-        rowToPublicExecutableFn, rowToDarkRelation,
-        CONN_SQL, FORCE_RLS_SQL, appGrantsRlsSql, SECDEF_FUNCTIONS_SQL,
-        PUBLIC_EXECUTABLE_FUNCTIONS_SQL, DARK_RELATIONS_SQL,
-      } = await import('./db-doctor');
+      const { runDoctorProbe } = await import('./db-doctor');
       const rowsOf = (result: any): any[] => (Array.isArray(result) ? result : result?.rows || []);
 
-      const quoteIdent = (s: string) => '"' + String(s).replace(/"/g, '""') + '"';
-
+      // Two drizzle-backed runners for the shared orchestrator: the api credential under
+      // scrutiny (createDb / DATABASE_URL) and the operator connection that carries the
+      // catalog introspection (opsDb / ADMIN_DATABASE_URL).
       let apiDb: any;
-      let api;
       try {
         const { createDb } = await import('./db');
         apiDb = createDb(ctx.schema).db;
-        api = rowToConnInfo(rowsOf(await apiDb.execute(sql.raw(CONN_SQL)))[0]);
       } catch (err: any) {
         return { error: `api connection probe failed (DATABASE_URL): ${err?.message || String(err)}` };
       }
+      const apiRun = async (q: string): Promise<any[]> => rowsOf(await apiDb.execute(sql.raw(q)));
+      const opsRun = async (q: string): Promise<any[]> => rowsOf(await opsDb.execute(sql.raw(q)));
 
-      const admin = rowToConnInfo(rowsOf(await opsDb.execute(sql.raw(CONN_SQL)))[0]);
-      const forceRls = rowsOf(await opsDb.execute(sql.raw(FORCE_RLS_SQL))).map(rowToForceRls);
-      // The "we require RLS" set — every app-reachable table + its RLS posture, so a table
-      // granted to a request role with RLS off (the PostgREST fail-open shape) is caught.
-      const appGrantedRls = rowsOf(await opsDb.execute(sql.raw(appGrantsRlsSql()))).map(rowToAppGranted);
-      // SECDEF functions (schema, name, search_path pinned?) — the `functions` teeth read these
-      // to assert every SECDEF function in a functions-written schema pins its search_path.
-      const secdefFunctions = rowsOf(await opsDb.execute(sql.raw(SECDEF_FUNCTIONS_SQL))).map(rowToSecdefFn);
-      // B4 — the derived layer's runtime echoes: PUBLIC-executable functions (NULL proacl
-      // included via acldefault) and superuser-only views/matviews.
-      const publicExecutableFunctions = rowsOf(await opsDb.execute(sql.raw(PUBLIC_EXECUTABLE_FUNCTIONS_SQL))).map(rowToPublicExecutableFn);
-      const darkRelations = rowsOf(await opsDb.execute(sql.raw(DARK_RELATIONS_SQL))).map(rowToDarkRelation);
-
-      // Ambient-access probe: can the api connection read an RLS table WITHOUT SET ROLE?
-      let apiAmbientRead: { table: string; accessible: boolean } | undefined;
-      if (forceRls.length > 0) {
-        const t = forceRls[0];
-        const ref = `${t.schema}.${t.table}`;
+      // Backups lens — is the pg_dump layer attached AND version-compatible? Lambda-only:
+      // a missing layer leaves pgDump null (the lens warns), never throws. Absent on the
+      // CLI direct venue, where there is no layer.
+      const backup = async (): Promise<import('./db-doctor').DoctorProbe['backup']> => {
         try {
-          await apiDb.execute(sql.raw(`SELECT 1 FROM ${quoteIdent(t.schema)}.${quoteIdent(t.table)} LIMIT 1`));
-          apiAmbientRead = { table: ref, accessible: true };
+          const { pgClientVersion, parsePgDumpMajor, serverMajorFromNum, isDumpCompatible } = await import('./backup');
+          let pgDump: string | null = null;
+          let pgDumpMajor: number | null = null;
+          try {
+            pgDump = await pgClientVersion('pg_dump');
+            pgDumpMajor = parsePgDumpMajor(pgDump);
+          } catch {
+            // layer absent — pgDump stays null, the lens reports "not attached"
+          }
+          const svnRow = rowsOf(await opsDb.execute(sql.raw('SHOW server_version_num')))[0];
+          const svn = Number(svnRow?.server_version_num);
+          const serverMajor = Number.isFinite(svn) ? serverMajorFromNum(svn) : null;
+          return {
+            pgDump,
+            pgDumpMajor,
+            serverMajor,
+            compatible: pgDumpMajor != null && Number.isFinite(svn) ? isDumpCompatible(pgDumpMajor, svn) : false,
+          };
         } catch {
-          apiAmbientRead = { table: ref, accessible: false };
+          return undefined; // leave backup out → the lens is skipped, like the other optional probes
         }
-      }
+      };
 
-      // Backups lens — is the pg_dump layer attached AND version-compatible? Reuse the
-      // db:backup:probe internals; a missing layer leaves pgDump null (the lens warns), never throws.
-      let backup: import('./db-doctor').DoctorProbe['backup'];
       try {
-        const { pgClientVersion, parsePgDumpMajor, serverMajorFromNum, isDumpCompatible } = await import('./backup');
-        let pgDump: string | null = null;
-        let pgDumpMajor: number | null = null;
-        try {
-          pgDump = await pgClientVersion('pg_dump');
-          pgDumpMajor = parsePgDumpMajor(pgDump);
-        } catch {
-          // layer absent — pgDump stays null, the lens reports "not attached"
-        }
-        const svnRow = rowsOf(await opsDb.execute(sql.raw('SHOW server_version_num')))[0];
-        const svn = Number(svnRow?.server_version_num);
-        const serverMajor = Number.isFinite(svn) ? serverMajorFromNum(svn) : null;
-        backup = {
-          pgDump,
-          pgDumpMajor,
-          serverMajor,
-          compatible: pgDumpMajor != null && Number.isFinite(svn) ? isDumpCompatible(pgDumpMajor, svn) : false,
-        };
-      } catch {
-        // leave backup undefined → the lens is skipped, exactly like the other optional probes
+        return await runDoctorProbe(
+          { api: apiRun, ops: opsRun },
+          {
+            apiSourceLabel: 'DATABASE_URL',
+            forceRlsCarveouts: options.forceRlsCarveouts,
+            functionSchemas: options.functionSchemas,
+            backup,
+          },
+        );
+      } catch (err: any) {
+        return { error: err?.message || String(err) };
       }
-
-      return buildDoctorReport(
-        { api, admin, forceRls, appGrantedRls, apiAmbientRead, secdefFunctions, publicExecutableFunctions, darkRelations, backup },
-        { forceRlsCarveouts: options.forceRlsCarveouts, functionSchemas: options.functionSchemas },
-      );
     };
 
     // --- db:provision — BOTH sides of the credential split, in the safe order ---
@@ -978,6 +969,71 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       }
     };
 
+    // --- db:reconcile — the credential-free DERIVED-layer apply (the db:apply twin) ---
+    // The STATE edge (db:apply) runs in the ops Lambda; this is the DERIVED edge's lane so a
+    // least-privileged operator never needs the raw admin URL to deploy functions/views/matviews.
+    // Unlike db:apply's portable plan, a reconcile plan needs LIVE introspection (source hash vs
+    // live def hash), so the whole executeReconcile runs HERE on the operator connection: the CLI
+    // ships only the declared derived objects (bundle-independent — they come from the models
+    // barrel), and this introspects → plans → applies, emitting its own SET search_path. The DDL
+    // batch and its provenance/schema_log land in one transaction, same as the direct path.
+    actions['db:reconcile'] = async (payload, _ctx) => {
+      const p = (payload ?? {}) as {
+        declared?: any[];
+        renamedTables?: any;
+        apply?: boolean;
+        baseline?: boolean;
+        rebaseline?: boolean;
+        rebuild?: boolean;
+        overwriteDrift?: boolean;
+        actor?: string | null;
+        gitRef?: string | null;
+      };
+      try {
+        const { executeReconcile } = await import('@everystack/cli/reconcile');
+        const { sql } = await import('drizzle-orm');
+        const runner = async (query: string): Promise<any[]> => {
+          const result: any = await opsDb.execute(sql.raw(query));
+          return Array.isArray(result) ? result : result?.rows ?? [];
+        };
+        const run = await executeReconcile(runner, {
+          declared: p.declared,
+          renamedTables: p.renamedTables,
+          apply: p.apply,
+          baseline: p.baseline,
+          rebaseline: p.rebaseline,
+          rebuild: p.rebuild,
+          overwriteDrift: p.overwriteDrift,
+          actor: p.actor ?? null,
+          gitRef: p.gitRef ?? null,
+        } as any);
+        return { plan: run.plan, applied: run.applied, statements: run.statements, refusal: run.refusal ?? null };
+      } catch (err: any) {
+        return { error: err?.message || String(err) };
+      }
+    };
+
+    // --- db:refresh — the credential-free DATA lane's refresh half (the reconcile-lane twin) ---
+    // db:reconcile deploys the derived DEFINITION but no-ops on an unchanged def, so a pure
+    // DATA-changed matview refresh had no credential-free path. This runs the refresh HERE on
+    // the operator connection: the CLI ships the declared matview identities in dependency order,
+    // this REFRESHes each. A least-privileged operator never holds a URL for the routine refresh.
+    actions['db:refresh'] = async (payload, _ctx) => {
+      const p = (payload ?? {}) as { matviews?: string[] };
+      try {
+        const { executeRefresh } = await import('@everystack/cli/refresh');
+        const { sql } = await import('drizzle-orm');
+        const runner = async (query: string): Promise<any[]> => {
+          const result: any = await opsDb.execute(sql.raw(query));
+          return Array.isArray(result) ? result : result?.rows ?? [];
+        };
+        const run = await executeRefresh(runner, { matviews: p.matviews ?? [] });
+        return { refreshed: run.refreshed, statements: run.statements, failed: run.failed ?? null };
+      } catch (err: any) {
+        return { error: err?.message || String(err) };
+      }
+    };
+
     // --- pipeline:run — the credential-free ingest reproduction (Phase 3, Brick 5) ---
     // The orchestrator runs HERE, in the ops Lambda, over the operator connection: a
     // deployed stage's small curated overlays reproduce via IAM, never a held URL. The

package/src/sst-resources.ts

@@ -0,0 +1,80 @@
+/**
+ * @everystack/server/sst-resources — read SST-linked resources WITHOUT importing `sst`.
+ *
+ * `sst` is a deploy-time construct: importing it makes @everystack/server unbundleable
+ * (Metro) and trips SST's own build-time version-check plugin. So we read the resources the
+ * way SST delivers them, mirroring `sst/dist/resource/node.js` exactly:
+ *
+ *   1. per-name env — `process.env.SST_RESOURCE_<Name>` (JSON)
+ *   2. the bundled all-resources JSON — `process.env.SST_RESOURCES_JSON`
+ *   3. the ENCRYPTED key file — `SST_KEY_FILE` + `SST_KEY` (SST >= ~4.13's Lambda default,
+ *      aes-256-gcm, 12-byte zero nonce, 16-byte auth tag appended)
+ *
+ * (1) alone — the 0.4.5 sst-removal's mistake — misses everything on a modern SST Lambda,
+ * where only `SST_RESOURCE_App` is plain env and every real link rides the encrypted blob.
+ * `node:crypto`/`node:fs` are builtins already shipped in this package's server-bundle path
+ * (ssr.ts, backup-run.ts) — never the `sst` package, so nothing here breaks a bundle.
+ */
+
+import crypto from 'node:crypto';
+import { readFileSync } from 'node:fs';
+
+let cache: Record<string, any> | undefined;
+
+/** Decrypt the SST key file exactly as sst/dist/resource/node.js does. Throws on any format
+ *  mismatch (bad key, truncated file) — the caller swallows it and uses the other sources. */
+function decryptKeyFile(keyFile: string, keyB64: string): Record<string, any> {
+  const key = Buffer.from(keyB64, 'base64');
+  const encrypted = readFileSync(keyFile);
+  const nonce = Buffer.alloc(12, 0);
+  const decipher = crypto.createDecipheriv('aes-256-gcm', key, nonce);
+  decipher.setAuthTag(encrypted.subarray(-16));
+  const decrypted = Buffer.concat([decipher.update(encrypted.subarray(0, -16)), decipher.final()]);
+  return JSON.parse(decrypted.toString());
+}
+
+/**
+ * The merged SST resource map, loaded once and cached (resources are container-stable on a
+ * Lambda). Every source is best-effort — a malformed one is skipped, never fatal — so an app
+ * that only ever sets plain env still resolves what it has, and local/Metro (no SST env) gets
+ * an empty map. Later sources win, matching SST's own precedence.
+ */
+export function loadSstResources(): Record<string, any> {
+  if (cache) return cache;
+  const raw: Record<string, any> = {};
+  const env = process.env;
+
+  for (const [k, v] of Object.entries(env)) {
+    if (v && k.startsWith('SST_RESOURCE_')) {
+      try {
+        raw[k.slice('SST_RESOURCE_'.length)] = JSON.parse(v);
+      } catch {
+        /* not JSON — skip this key */
+      }
+    }
+  }
+
+  if (env.SST_RESOURCES_JSON) {
+    try {
+      Object.assign(raw, JSON.parse(env.SST_RESOURCES_JSON));
+    } catch {
+      /* malformed — skip */
+    }
+  }
+
+  if (env.SST_KEY_FILE && env.SST_KEY) {
+    try {
+      Object.assign(raw, decryptKeyFile(env.SST_KEY_FILE, env.SST_KEY));
+    } catch {
+      /* undecryptable — fall through to what the env already gave us */
+    }
+  }
+
+  cache = raw;
+  return raw;
+}
+
+/** Test seam: drop the cache so a fresh env is re-read. */
+export function resetSstResourceCache(): void {
+  cache = undefined;
+}