@everystack/server 0.3.3 → 0.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/server",
-  "version": "0.3.3",
+  "version": "0.3.6",
   "description": "Server runtime primitives for Lambda — event adapters, routing, SSR, image processing",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -72,6 +72,10 @@
     "./media": {
       "types": "./src/media.ts",
       "default": "./src/media.ts"
+    },
+    "./pipeline": {
+      "types": "./src/pipeline.ts",
+      "default": "./src/pipeline.ts"
     }
   },
   "peerDependencies": {
@@ -139,8 +143,8 @@
     "sst": "4.13.1",
     "ts-jest": "29.4.9",
     "typescript": "5.9.3",
-    "@everystack/auth": "0.3.1",
-    "@everystack/cli": "0.3.27"
+    "@everystack/auth": "0.4.0",
+    "@everystack/cli": "0.4.3"
   },
   "scripts": {
     "test": "jest",

package/src/backup-run.ts

@@ -37,8 +37,42 @@ export interface BackupSummary {
   lastModified?: string;
 }
 
+/**
+ * Lazily load an optional-peer AWS SDK client, turning a missing dependency into a one-line
+ * install remedy instead of a raw `ERR_MODULE_NOT_FOUND` deep in the ops Lambda. The `@aws-sdk/*`
+ * clients are optional peers of @everystack/server (a V1 app needs none of them), so a consumer
+ * whose ops function runs db:backup must install them — this names which, not "Cannot find
+ * package". Takes a thunk so the `import()` stays a literal — statically analyzable and typed.
+ */
+export async function importOptionalAws<T>(
+  load: () => Promise<T>,
+  pkg: string,
+  feature: string,
+): Promise<T> {
+  try {
+    return await load();
+  } catch (err: unknown) {
+    const code = (err as { code?: string })?.code;
+    const message = String((err as { message?: string })?.message ?? err);
+    if (
+      code === 'ERR_MODULE_NOT_FOUND' ||
+      code === 'MODULE_NOT_FOUND' ||
+      /Cannot find (package|module)/.test(message)
+    ) {
+      throw new Error(
+        `${feature} needs ${pkg}, which isn't installed (it's an optional peer dependency of @everystack/server). Install it:\n  pnpm add ${pkg}`,
+      );
+    }
+    throw err;
+  }
+}
+
 async function getS3(region?: string): Promise<any> {
-  const { S3Client } = await import('@aws-sdk/client-s3');
+  const { S3Client } = await importOptionalAws(
+    () => import('@aws-sdk/client-s3'),
+    '@aws-sdk/client-s3',
+    'db:backup',
+  );
   return new S3Client(region ? { region } : {});
 }
 
@@ -57,8 +91,16 @@ export async function runBackup(deps: BackupRunDeps, stage: string): Promise<Bac
   const id = backupId(stage, stamp, shortId);
 
   const s3 = await getS3(region);
-  const { Upload } = await import('@aws-sdk/lib-storage');
-  const { DeleteObjectCommand, PutObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3');
+  const { Upload } = await importOptionalAws(
+    () => import('@aws-sdk/lib-storage'),
+    '@aws-sdk/lib-storage',
+    'db:backup',
+  );
+  const { DeleteObjectCommand, PutObjectCommand, HeadObjectCommand } = await importOptionalAws(
+    () => import('@aws-sdk/client-s3'),
+    '@aws-sdk/client-s3',
+    'db:backup',
+  );
 
   const child = spawn('pg_dump', pgDumpArgs(), { env, stdio: ['ignore', 'pipe', 'pipe'] });
   let stderr = '';

package/src/cdn/index.ts

@@ -38,6 +38,7 @@ export {
 export {
   stripSstDeadCode,
   applyWafExclusions,
+  applyWafRateLimit429,
   WAF_JSON_API_OVERRIDES,
 } from './transforms';
 

package/src/cdn/transforms.ts

@@ -155,3 +155,60 @@ export function applyWafExclusions(
     return rule;
   });
 }
+
+/**
+ * Rewrite rate-based-rule block actions to answer 429 + Retry-After instead
+ * of WAF's default 403.
+ *
+ * Google is explicit that the status code matters for SEO ("Don't use 403s
+ * or 404s for rate limiting"): every 4xx except 429 signals the content no
+ * longer exists and deindexes the URL over time, while 429/503 mean "back
+ * off and retry" — crawling slows almost immediately and Google retries for
+ * ~2 days before dropping anything. A per-IP rate rule clears within
+ * minutes of the burst ending, so a legitimate crawler never sees multi-day
+ * 429s. Getting this wrong turns a load shield into a deindexing engine.
+ *
+ * Only rules with a rateBasedStatement AND a block action are touched;
+ * managed rule groups (overrideAction) and count-mode rules pass through.
+ *
+ * @param rules  Resolved WAF rules array from Pulumi.
+ * @param retryAfterSeconds  Retry-After header value. Default 120.
+ *
+ * @example
+ * ```ts
+ * // In sst.config.ts — composes with applyWafExclusions in one transform.
+ * // MUST be registered BEFORE `new sst.aws.Router` (a $transform only
+ * // applies to resources created after it, and the Router creates the WebAcl).
+ * import { applyWafExclusions, applyWafRateLimit429 } from '@everystack/server/cdn';
+ *
+ * $transform(aws.wafv2.WebAcl, (args: any) => {
+ *   if (!args.rules) return;
+ *   args.rules = $resolve(args.rules).apply(
+ *     (rules) => applyWafRateLimit429(applyWafExclusions(rules)),
+ *   );
+ * });
+ * ```
+ */
+export function applyWafRateLimit429(
+  rules: any[],
+  retryAfterSeconds = 120,
+): any[] {
+  return rules.map((rule: any) => {
+    if (!rule.statement?.rateBasedStatement || !rule.action?.block) return rule;
+    return {
+      ...rule,
+      action: {
+        ...rule.action,
+        block: {
+          ...rule.action.block,
+          customResponse: {
+            responseCode: 429,
+            responseHeaders: [
+              { name: 'Retry-After', value: String(retryAfterSeconds) },
+            ],
+          },
+        },
+      },
+    };
+  });
+}

package/src/db-doctor.ts

@@ -67,8 +67,40 @@ export interface DoctorProbe {
    * `functions` teeth read this: for each schema that owns a `writtenBy: 'functions'` table
    * (passed as `functionSchemas`), the package's SECDEF functions must exist and each must
    * pin its search_path. Absent on an older probe (the check is then skipped).
+   *
+   * B4 also reads it globally: an unpinned SECDEF ANYWHERE is the runtime echo of the
+   * compile gate (defineFunction refuses to compile one) — warn-level outside the
+   * functions-teeth schemas, which keep their fail.
    */
   secdefFunctions?: SecdefFn[];
+  /**
+   * Functions PUBLIC can EXECUTE — PostgreSQL's silent-open default (a NULL proacl MEANS
+   * EXECUTE to PUBLIC; the query expands acldefault to see it). The declared derived layer
+   * revokes it unconditionally; anything here was created outside it or hand-granted back.
+   * Absent on an older probe (the check is then skipped). B4.
+   */
+  publicExecutableFunctions?: PublicExecutableFn[];
+  /**
+   * Views/matviews NO app role can read (no non-owner ACL entries) — superuser-only
+   * relations, the runtime echo of the derived read-gap gate. Declared-dark descriptors
+   * (`private: true`) also land here by design; the warn names both resolutions. Absent on
+   * an older probe (the check is then skipped). B4.
+   */
+  darkRelations?: DarkRelation[];
+}
+
+/** A function PUBLIC can EXECUTE, and whether it runs as its owner (the dangerous half). */
+export interface PublicExecutableFn {
+  schema: string;
+  name: string;
+  secdef: boolean;
+}
+
+/** A view ('v') or matview ('m') with no grants to any non-owner role. */
+export interface DarkRelation {
+  schema: string;
+  name: string;
+  kind: string;
 }
 
 /** A SECURITY DEFINER function and whether its `search_path` is pinned. */
@@ -143,6 +175,59 @@ WHERE p.prosecdef
 ORDER BY n.nspname, p.proname
 `.trim();
 
+/**
+ * Functions PUBLIC can EXECUTE. The trap this exists for: a NULL `proacl` is not "no
+ * grants" — it is PostgreSQL's BUILT-IN DEFAULT, which includes EXECUTE to PUBLIC, so the
+ * query expands `acldefault('f', proowner)` before exploding. Trigger-returning functions
+ * are excluded (not directly callable — PUBLIC EXECUTE on them is inert), as are
+ * extension-owned functions (not ours to judge) and system schemas. `secdef` marks the
+ * dangerous half: PUBLIC-callable code that runs as its OWNER.
+ */
+export const PUBLIC_EXECUTABLE_FUNCTIONS_SQL = `
+SELECT
+  n.nspname AS schema,
+  p.proname AS name,
+  p.prosecdef AS secdef
+FROM pg_proc p
+JOIN pg_namespace n ON n.oid = p.pronamespace
+CROSS JOIN LATERAL aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a
+WHERE p.prokind IN ('f', 'p')
+  AND a.grantee = 0
+  AND a.privilege_type = 'EXECUTE'
+  AND p.prorettype <> 'trigger'::regtype
+  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND n.nspname NOT LIKE 'pg_%'
+  AND NOT EXISTS (
+    SELECT 1 FROM pg_depend dep WHERE dep.objid = p.oid AND dep.deptype = 'e'
+  )
+GROUP BY n.nspname, p.proname, p.prosecdef
+ORDER BY n.nspname, p.proname
+`.trim();
+
+/**
+ * Views/matviews with no ACL entry for any non-owner role — readable by the superuser
+ * and nobody else. The runtime echo of the derived read-gap gate: on a modeled project
+ * this is either an object created outside the read model or a declared-dark descriptor.
+ */
+export const DARK_RELATIONS_SQL = `
+SELECT
+  n.nspname AS schema,
+  c.relname AS name,
+  c.relkind AS kind
+FROM pg_class c
+JOIN pg_namespace n ON n.oid = c.relnamespace
+WHERE c.relkind IN ('v', 'm')
+  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND n.nspname NOT LIKE 'pg_%'
+  AND NOT EXISTS (
+    SELECT 1 FROM aclexplode(c.relacl) a WHERE a.grantee <> c.relowner
+  )
+  AND NOT EXISTS (
+    SELECT 1 FROM pg_depend dep WHERE dep.objid = c.oid AND dep.deptype = 'e'
+  )
+ORDER BY n.nspname, c.relname
+`.trim();
+
 /** Coerce a Postgres boolean (true | 't' | 'true' | 1) to a JS boolean. */
 export function pgBool(v: unknown): boolean {
   return v === true || v === 't' || v === 'true' || v === 1 || v === '1';
@@ -385,6 +470,75 @@ export function buildDoctorReport(probe: DoctorProbe, options: DoctorReportOptio
     }
   }
 
+  // B4 — the derived layer's runtime echoes (read-model-everywhere). Each check echoes a
+  // compile/check gate on the LIVE database: the gates cover what the models declare;
+  // these cover what actually exists, however it got there.
+
+  // PUBLIC-executable functions: fail when any runs as its owner (SECDEF — PUBLIC-callable
+  // privileged code), warn otherwise (invoker functions still run with the caller's
+  // rights, but PUBLIC EXECUTE is never a decision someone reviewed).
+  if (probe.publicExecutableFunctions) {
+    const offenders = probe.publicExecutableFunctions;
+    if (offenders.length > 0) {
+      const label = (f: PublicExecutableFn): string => `${f.schema}.${f.name}${f.secdef ? ' (SECURITY DEFINER)' : ''}`;
+      checks.push({
+        name: 'Functions are not PUBLIC-executable',
+        status: offenders.some((f) => f.secdef) ? 'fail' : 'warn',
+        detail: `PUBLIC can EXECUTE (PostgreSQL's silent-open default — a NULL proacl grants it): ${offenders.map(label).join(', ')}`,
+        remediation: offenders.map((f) => `REVOKE ALL ON FUNCTION ${f.schema}.${f.name} FROM PUBLIC;`).join('  ')
+          + ` Then grant the intended callers — or declare the function with defineFunction + can('execute', { role }) so the compiler emits both.`,
+      });
+    } else {
+      checks.push({
+        name: 'Functions are not PUBLIC-executable',
+        status: 'pass',
+        detail: 'no function is executable by PUBLIC',
+      });
+    }
+  }
+
+  // Dark relations: readable by the superuser and nobody else — either created outside
+  // the read model, or declared dark on purpose. Warn, because only the author knows.
+  if (probe.darkRelations) {
+    if (probe.darkRelations.length > 0) {
+      checks.push({
+        name: 'Derived relations are readable or declared dark',
+        status: 'warn',
+        detail: `no app role can read: ${probe.darkRelations.map((r) => `${r.schema}.${r.name}`).join(', ')} — superuser-only relation(s)`,
+        remediation: `Declare can('read') on the descriptor (the compiler emits the GRANT), or private: true if it is intentionally dark — either way the decision lands in source.`,
+      });
+    } else {
+      checks.push({
+        name: 'Derived relations are readable or declared dark',
+        status: 'pass',
+        detail: 'every view/matview is granted to at least one role',
+      });
+    }
+  }
+
+  // The global SECDEF-pin echo: the compile gate refuses to DECLARE an unpinned SECDEF;
+  // this catches the ones that exist anyway (hand-created, pre-descriptor). The
+  // functions-teeth schemas keep their fail above — excluded here, never double-reported.
+  if (probe.secdefFunctions) {
+    const teeth = new Set(options.functionSchemas ?? []);
+    const unpinned = probe.secdefFunctions.filter((f) => !f.hasSearchPath && !teeth.has(f.schema));
+    if (unpinned.length > 0) {
+      checks.push({
+        name: 'Every SECURITY DEFINER function pins search_path',
+        status: 'warn',
+        detail: `unpinned SECDEF function(s) — owner-privilege hijack vector: ${unpinned.map((f) => `${f.schema}.${f.name}`).join(', ')}`,
+        remediation: unpinned.map((f) => `ALTER FUNCTION ${f.schema}.${f.name} SET search_path = pg_catalog;`).join('  ')
+          + ` Better: declare it with defineFunction({ security: 'definer', searchPath: [...] }) — the compiler refuses an unpinned one.`,
+      });
+    } else {
+      checks.push({
+        name: 'Every SECURITY DEFINER function pins search_path',
+        status: 'pass',
+        detail: `all ${probe.secdefFunctions.length} SECURITY DEFINER function(s) pin their search_path`,
+      });
+    }
+  }
+
   return {
     api,
     admin,
@@ -393,6 +547,24 @@ export function buildDoctorReport(probe: DoctorProbe, options: DoctorReportOptio
   };
 }
 
+/** Map a PUBLIC_EXECUTABLE_FUNCTIONS_SQL row to PublicExecutableFn. */
+export function rowToPublicExecutableFn(row: any): PublicExecutableFn {
+  return {
+    schema: String(row.schema),
+    name: String(row.name),
+    secdef: pgBool(row.secdef),
+  };
+}
+
+/** Map a DARK_RELATIONS_SQL row to DarkRelation. */
+export function rowToDarkRelation(row: any): DarkRelation {
+  return {
+    schema: String(row.schema),
+    name: String(row.name),
+    kind: String(row.kind),
+  };
+}
+
 /** Map a SECDEF_FUNCTIONS_SQL row to SecdefFn. */
 export function rowToSecdefFn(row: any): SecdefFn {
   return {

package/src/pipeline.ts

@@ -0,0 +1,423 @@
+/**
+ * @everystack/server/pipeline — the ingest pipeline framework (Phase 3).
+ *
+ * A pipeline reproduces data from committed source: an ordered set of STAGES,
+ * each a function that reads source and writes rows idempotently. You never
+ * copy data between environments — you re-run the same committed source + the
+ * same stage code and get the same rows ("sync by reproduction").
+ *
+ * everystack owns the HARNESS — the stage contract, the topological order, the
+ * provenance-lane wall (automated/curated/frozen), resume, and the memoir
+ * (everystack.pipeline_log). The consumer owns the STAGES: their
+ * fetch/parse/transform/load code, their source files, their table and column
+ * names, their provenance values. Ingest logic never enters the framework,
+ * exactly as schema and migrations never do.
+ *
+ * A stage is a FUNCTION, not a spawned script, so the same stage runs unchanged
+ * in three places: a local process (`pipeline:run --database-url`), the ops
+ * Lambda (`pipeline:run --stage`, credential-free), or — later — a worker.
+ *
+ * See docs/plans/ingest-pipeline-framework.md.
+ */
+
+/** Read/write SQL against the target, inside whatever transaction the caller holds. */
+export type PipelineQuery = (sql: string) => Promise<any[]>;
+
+/**
+ * The connection the orchestrator drives. `transaction` runs a stage's writes
+ * and its memoir row as one atomic unit; `query` is for the pre-flight ensure
+ * and the best-effort failure record (which must land OUTSIDE the rolled-back
+ * stage transaction).
+ */
+export interface PipelineRunner {
+  query: PipelineQuery;
+  transaction: <T>(fn: (tx: PipelineQuery) => Promise<T>) => Promise<T>;
+}
+
+/** The provenance lane — the wall between the two reproduction audiences. */
+export type StageLane = 'automated' | 'curated' | 'frozen';
+
+/** What a stage reports back for the memoir (both optional — a stage may write nothing). */
+export interface StageResult {
+  /** Rows this stage wrote, for the memoir. */
+  rows?: number;
+  /** Content hash of the source this stage reproduced from — Phase 4's verify reads it. */
+  sourceHash?: string;
+}
+
+/** What a stage receives. The framework builds it; the consumer's run() uses it. */
+export interface StageContext {
+  /** The transactional runner for this stage's writes. */
+  runner: PipelineQuery;
+  /** This stage's provenance tag (`provenance` override, else `${lane}:${id}`). */
+  provenance: string;
+  /** True → compute/preview, write nothing. */
+  dryRun: boolean;
+  /** Emit a progress line. */
+  log: (message: string) => void;
+  /**
+   * The idempotency helper: delete the rows this stage owns before re-inserting.
+   * `DELETE FROM <table> WHERE <column> = <this stage's provenance>`. The stage
+   * owns the actual insert; the framework owns only the provenance-replace shape.
+   */
+  replaceProvenance: (table: string, column: string) => Promise<void>;
+}
+
+export interface Stage {
+  id: string;
+  lane: StageLane;
+  /** Data/topo edges — a dependency runs before its dependent. */
+  dependsOn?: string[];
+  /** Ordering-only edges (run after, but no data dependency). */
+  after?: string[];
+  /** Refuses the ops-Lambda executor — this stage is too big, run it local-direct. */
+  heavy?: boolean;
+  /** Provenance tag for the memoir + the replace helper. Default `${lane}:${id}`. */
+  provenance?: string;
+  run: (ctx: StageContext) => Promise<StageResult | void>;
+}
+
+export interface Pipeline {
+  stages: Stage[];
+}
+
+/** Identity for defineStage — a stage is a plain descriptor. */
+export function defineStage(stage: Stage): Stage {
+  return stage;
+}
+
+/** Compose stages into a pipeline, validating unique ids and resolvable edges. */
+export function definePipeline(stages: Stage[]): Pipeline {
+  const ids = new Set<string>();
+  for (const s of stages) {
+    if (ids.has(s.id)) throw new Error(`pipeline has a duplicate stage id: "${s.id}"`);
+    ids.add(s.id);
+  }
+  for (const s of stages) {
+    for (const dep of [...(s.dependsOn ?? []), ...(s.after ?? [])]) {
+      if (!ids.has(dep)) {
+        throw new Error(`stage "${s.id}" depends on "${dep}", which is not a stage in this pipeline`);
+      }
+    }
+  }
+  return { stages };
+}
+
+/** The provenance tag a stage writes under. */
+export function stageProvenance(stage: Stage): string {
+  return stage.provenance ?? `${stage.lane}:${stage.id}`;
+}
+
+/**
+ * Deterministic topological sort over dependsOn ∪ after edges. Stable: with no
+ * edge between two stages, declared order wins (so a re-run is byte-stable).
+ * Throws on an unknown edge or a cycle (naming a stage on the cycle).
+ */
+export function topoSortStages(stages: Stage[]): Stage[] {
+  const byId = new Map(stages.map((s) => [s.id, s]));
+  const indegree = new Map<string, number>(stages.map((s) => [s.id, 0]));
+  const dependents = new Map<string, string[]>(stages.map((s) => [s.id, []]));
+
+  for (const s of stages) {
+    for (const dep of [...(s.dependsOn ?? []), ...(s.after ?? [])]) {
+      if (!byId.has(dep)) {
+        throw new Error(`stage "${s.id}" depends on unknown stage "${dep}"`);
+      }
+      indegree.set(s.id, (indegree.get(s.id) ?? 0) + 1);
+      dependents.get(dep)!.push(s.id);
+    }
+  }
+
+  // Kahn's algorithm; the ready queue is kept in declared order for stability.
+  const declaredIndex = new Map(stages.map((s, i) => [s.id, i]));
+  const ready = stages.filter((s) => (indegree.get(s.id) ?? 0) === 0).map((s) => s.id);
+  const sorted: Stage[] = [];
+  while (ready.length > 0) {
+    ready.sort((a, b) => declaredIndex.get(a)! - declaredIndex.get(b)!);
+    const id = ready.shift()!;
+    sorted.push(byId.get(id)!);
+    for (const next of dependents.get(id)!) {
+      indegree.set(next, (indegree.get(next) ?? 0) - 1);
+      if (indegree.get(next) === 0) ready.push(next);
+    }
+  }
+
+  if (sorted.length !== stages.length) {
+    const remaining = stages.filter((s) => !sorted.includes(s)).map((s) => s.id);
+    throw new Error(`pipeline has a dependency cycle involving: ${remaining.join(', ')}`);
+  }
+  return sorted;
+}
+
+export interface SelectOptions {
+  /** rebuild → automated + frozen; curate → curated; else all lanes. */
+  lane?: 'rebuild' | 'curate' | 'all';
+  /** Keep stages whose id contains this substring. */
+  only?: string;
+  /** Slice the ordered list from this stage id (inclusive). */
+  from?: string;
+  /** Slice the ordered list to this stage id (inclusive). */
+  to?: string;
+}
+
+const laneKeeps = (lane: SelectOptions['lane'], stageLane: StageLane): boolean => {
+  if (lane === 'rebuild') return stageLane === 'automated' || stageLane === 'frozen';
+  if (lane === 'curate') return stageLane === 'curated';
+  return true;
+};
+
+/** Filter an already-ordered stage list by lane, --only, and --from/--to. */
+export function selectStages(ordered: Stage[], opts: SelectOptions): Stage[] {
+  let selected = ordered;
+  if (opts.from || opts.to) {
+    const start = opts.from ? selected.findIndex((s) => s.id === opts.from) : 0;
+    const endIdx = opts.to ? selected.findIndex((s) => s.id === opts.to) : selected.length - 1;
+    if (start === -1) throw new Error(`--from names no stage: "${opts.from}"`);
+    if (endIdx === -1) throw new Error(`--to names no stage: "${opts.to}"`);
+    selected = selected.slice(start, endIdx + 1);
+  }
+  selected = selected.filter((s) => laneKeeps(opts.lane, s.lane));
+  if (opts.only) selected = selected.filter((s) => s.id.includes(opts.only!));
+  return selected;
+}
+
+// ---------------------------------------------------------------------------
+// The memoir — everystack.pipeline_log (append-only, sibling to backfill_log).
+// ---------------------------------------------------------------------------
+
+export const ENSURE_PIPELINE_SQL: string[] = [
+  'CREATE SCHEMA IF NOT EXISTS everystack',
+  `CREATE TABLE IF NOT EXISTS everystack.pipeline_log (
+    id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    applied_at timestamptz NOT NULL DEFAULT now(),
+    run_id text NOT NULL,
+    stage_id text NOT NULL,
+    lane text NOT NULL,
+    source_hash text,
+    rows integer,
+    actor text,
+    git_ref text,
+    duration_ms integer,
+    outcome text NOT NULL
+  )`,
+];
+
+const sqlStr = (v: string | null | undefined): string =>
+  v === null || v === undefined ? 'NULL' : `'${String(v).replace(/'/g, "''")}'`;
+const sqlNum = (v: number | null | undefined): string =>
+  v === null || v === undefined || !Number.isFinite(v) ? 'NULL' : String(Math.trunc(v));
+
+export interface PipelineLogEntry {
+  runId: string;
+  stageId: string;
+  lane: StageLane;
+  sourceHash?: string | null;
+  rows?: number | null;
+  actor?: string | null;
+  gitRef?: string | null;
+  durationMs?: number | null;
+  outcome: 'applied' | 'failed';
+}
+
+export function renderPipelineLogInsert(e: PipelineLogEntry): string {
+  return `INSERT INTO everystack.pipeline_log (run_id, stage_id, lane, source_hash, rows, actor, git_ref, duration_ms, outcome)
+    VALUES (${sqlStr(e.runId)}, ${sqlStr(e.stageId)}, ${sqlStr(e.lane)}, ${sqlStr(e.sourceHash)}, ${sqlNum(e.rows)}, ${sqlStr(e.actor)}, ${sqlStr(e.gitRef)}, ${sqlNum(e.durationMs)}, ${sqlStr(e.outcome)})`;
+}
+
+/**
+ * The most recently touched run_id in the memoir — what `--resume` picks up
+ * with no explicit run id. Null when the pipeline has never run here.
+ */
+export async function latestRunId(query: PipelineQuery): Promise<string | null> {
+  try {
+    const rows = await query('SELECT run_id FROM everystack.pipeline_log ORDER BY id DESC LIMIT 1');
+    return rows.length > 0 ? String(rows[0].run_id) : null;
+  } catch {
+    // No memoir yet (table absent) — nothing to resume.
+    return null;
+  }
+}
+
+/** A one-line summary of a pipeline's most recent run — the visibility surface. */
+export interface PipelineRunSummary {
+  runId: string;
+  /** applied_at of the run's last row (a string as the driver returns it). */
+  at: string | null;
+  /** Stages whose LATEST attempt under this run succeeded. */
+  applied: number;
+  /** Stages whose LATEST attempt failed (an incomplete run — resume). */
+  failed: number;
+  failedStages: string[];
+}
+
+/**
+ * Summarize the most recent run for db:sync / db:plan to surface — the analog
+ * of the pending-backfill nudge. Read-only; null when the pipeline has never
+ * run here (or the memoir table is absent). Counts the LATEST attempt per stage
+ * so a resumed-and-fixed stage reads as applied, not failed.
+ */
+export async function readLastRun(query: PipelineQuery): Promise<PipelineRunSummary | null> {
+  try {
+    const latest = await query('SELECT run_id FROM everystack.pipeline_log ORDER BY id DESC LIMIT 1');
+    if (latest.length === 0) return null;
+    const runId = String(latest[0].run_id);
+    const rows = await query(
+      `SELECT stage_id, outcome, applied_at FROM everystack.pipeline_log WHERE run_id = ${sqlStr(runId)} ORDER BY id`,
+    );
+    const latestByStage = new Map<string, string>();
+    let at: string | null = null;
+    for (const r of rows) {
+      latestByStage.set(String(r.stage_id), String(r.outcome));
+      if (r.applied_at != null) at = String(r.applied_at);
+    }
+    const failedStages = [...latestByStage].filter(([, o]) => o === 'failed').map(([s]) => s);
+    const applied = [...latestByStage.values()].filter((o) => o === 'applied').length;
+    return { runId, at, applied, failed: failedStages.length, failedStages };
+  } catch {
+    return null;
+  }
+}
+
+/** The stage ids that already succeeded under a given run — the resume set. */
+export async function readAppliedStages(query: PipelineQuery, runId: string): Promise<Set<string>> {
+  const rows = await query(
+    `SELECT stage_id FROM everystack.pipeline_log WHERE run_id = ${sqlStr(runId)} AND outcome = 'applied'`,
+  );
+  return new Set(rows.map((r: any) => String(r.stage_id)));
+}
+
+// ---------------------------------------------------------------------------
+// The orchestrator — pure over an injected runner, identical local and ops-routed.
+// ---------------------------------------------------------------------------
+
+export interface RunPipelineOptions extends SelectOptions {
+  /** Groups a multi-stage run for resume; caller supplies it (a uuid). */
+  runId: string;
+  /** Skip stages already applied under runId. */
+  resume?: boolean;
+  /** Print the plan, run nothing. */
+  dryRun?: boolean;
+  /** Keep going past a failed stage (default: fail-fast). */
+  continueOnError?: boolean;
+  actor?: string | null;
+  gitRef?: string | null;
+  /** Injectable clock for tests. */
+  now?: () => number;
+  /** Emit a progress line. */
+  log?: (message: string) => void;
+}
+
+export type StageOutcome = 'applied' | 'skipped' | 'failed' | 'dry-run';
+
+export interface StageRunResult {
+  stageId: string;
+  outcome: StageOutcome;
+  rows?: number;
+  durationMs?: number;
+  error?: string;
+}
+
+export interface PipelineRunOutcome {
+  runId: string;
+  /** The ordered, selected stage ids — the plan. */
+  plan: string[];
+  results: StageRunResult[];
+  failed: boolean;
+}
+
+/**
+ * Reproduce the pipeline against the target: topo-order, filter to the selected
+ * lane/range, then run each stage inside its own transaction — the stage's
+ * writes AND its memoir row commit together. A throw rolls the stage back,
+ * records a best-effort `failed` row outside the transaction, and (unless
+ * continueOnError) stops, marking the rest skipped. dry-run returns the plan and
+ * runs nothing.
+ */
+export async function runPipeline(
+  runner: PipelineRunner,
+  pipeline: Pipeline,
+  options: RunPipelineOptions,
+): Promise<PipelineRunOutcome> {
+  const now = options.now ?? Date.now;
+  const log = options.log ?? (() => {});
+  const ordered = topoSortStages(pipeline.stages);
+  const selected = selectStages(ordered, options);
+  const plan = selected.map((s) => s.id);
+
+  if (options.dryRun) {
+    return {
+      runId: options.runId,
+      plan,
+      results: selected.map((s) => ({ stageId: s.id, outcome: 'dry-run' as const })),
+      failed: false,
+    };
+  }
+
+  await runner.query(ENSURE_PIPELINE_SQL.join(';\n'));
+  const alreadyApplied = options.resume
+    ? await readAppliedStages(runner.query, options.runId)
+    : new Set<string>();
+
+  const results: StageRunResult[] = [];
+  let stopped = false;
+
+  for (const stage of selected) {
+    if (stopped) {
+      results.push({ stageId: stage.id, outcome: 'skipped' });
+      continue;
+    }
+    if (options.resume && alreadyApplied.has(stage.id)) {
+      results.push({ stageId: stage.id, outcome: 'skipped' });
+      continue;
+    }
+
+    const provenance = stageProvenance(stage);
+    const started = now();
+    try {
+      const result = await runner.transaction(async (tx) => {
+        const ctx: StageContext = {
+          runner: tx,
+          provenance,
+          dryRun: false,
+          log,
+          replaceProvenance: async (table, column) => {
+            await tx(`DELETE FROM ${table} WHERE ${column} = ${sqlStr(provenance)}`);
+          },
+        };
+        const r = (await stage.run(ctx)) ?? {};
+        await tx(renderPipelineLogInsert({
+          runId: options.runId,
+          stageId: stage.id,
+          lane: stage.lane,
+          sourceHash: r.sourceHash ?? null,
+          rows: r.rows ?? null,
+          actor: options.actor ?? null,
+          gitRef: options.gitRef ?? null,
+          durationMs: now() - started,
+          outcome: 'applied',
+        }));
+        return r;
+      });
+      results.push({ stageId: stage.id, outcome: 'applied', rows: result.rows, durationMs: now() - started });
+    } catch (err: any) {
+      const message = String(err?.message ?? err);
+      // Best-effort failure memoir, OUTSIDE the rolled-back stage transaction.
+      try {
+        await runner.query(ENSURE_PIPELINE_SQL.join(';\n'));
+        await runner.query(renderPipelineLogInsert({
+          runId: options.runId,
+          stageId: stage.id,
+          lane: stage.lane,
+          durationMs: now() - started,
+          outcome: 'failed',
+        }));
+      } catch {
+        // the memoir is best-effort on failure
+      }
+      results.push({ stageId: stage.id, outcome: 'failed', durationMs: now() - started, error: message });
+      if (!options.continueOnError) stopped = true;
+    }
+  }
+
+  return { runId: options.runId, plan, results, failed: results.some((r) => r.outcome === 'failed') };
+}

package/src/plugin.ts

@@ -7,6 +7,7 @@
  */
 
 import type { Handler, Route, LogSink, ServerCacheConfig } from './index';
+import type { Pipeline } from './pipeline';
 import type {
   APIGatewayProxyEventV2,
   APIGatewayProxyStructuredResultV2,
@@ -185,6 +186,18 @@ export function createPluginLambdaHandler(
   ): Promise<APIGatewayProxyStructuredResultV2 | unknown> => {
     // Direct CLI invoke via Lambda Invoke — IAM is the auth layer.
     if ('_action' in event && typeof event._action === 'string') {
+      // Built-in health probe — `everystack deploy` invokes it right after a deploy so a bundle
+      // that can't initialize is caught at deploy time, not at the next db verb. Handled before
+      // the action lookup (it must answer even when initialization itself is the failure) and
+      // reserved: an app action named `_health` is never reached.
+      if (event._action === '_health') {
+        try {
+          const { actions } = await ensureInitialized();
+          return { ok: true, mode: httpEnabled ? 'http' : 'ops', actions: Object.keys(actions).sort() };
+        } catch (error) {
+          return { ok: false, error: String(error) };
+        }
+      }
       const { ctx, actions } = await ensureInitialized();
       const actionName = event._action;
       const actionHandler = actions[actionName];
@@ -490,6 +503,13 @@ export interface DbPluginOptions {
   backupBucket?: string;
   /** AWS region for the backups bucket (defaults to the Lambda's AWS_REGION). */
   backupRegion?: string;
+  /**
+   * The app's ingest pipeline (built with definePipeline). When provided, registers the
+   * credential-free `pipeline:run` action: the orchestrator runs IN this Lambda over the
+   * operator connection, so a deployed stage's small curated overlays reproduce via IAM,
+   * never a held database URL. Stages marked `heavy` are refused here (run them local-direct).
+   */
+  pipeline?: Pipeline;
 }
 
 /**
@@ -497,7 +517,14 @@ export interface DbPluginOptions {
  * Contributes: migrate, seed, db:reset, db:psql, db:query, console.
  *
  * Usage:
- *   dbPlugin({ migrationsFolder: join(__dirname, '..', 'drizzle'), seed: runSeed })
+ *   dbPlugin({
+ *     migrationsFolder: join(__dirname, '..', 'drizzle'),
+ *     // Load the seed module LAZILY. Seed scripts are often also runnable directly
+ *     // (`node db/seed.ts`) and may connect or process.exit at import time — a static
+ *     // `import { runSeed }` runs that at Lambda INIT and crashes the ops plane before
+ *     // any handler exists. The dynamic import defers it to the moment db:seed runs.
+ *     seed: async (db, schema) => (await import('../db/seed.js')).runSeed(db, schema),
+ *   })
  */
 export function dbPlugin(options: DbPluginOptions): Plugin {
   return async (ctx: PluginContext): Promise<PluginContribution> => {
@@ -508,6 +535,24 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
     // connection can be least-privilege and RLS-gated.
     const opsDb = ctx.adminDb ?? ctx.db;
 
+    // Resolve the logical-backup dependencies (bucket + operator URL) once —
+    // shared by db:backup/db:backups/db:restore and by db:apply's auto-snapshot
+    // before a destructive staged apply. Returns `{ error }` when unconfigured.
+    const resolveBackupDeps = async (): Promise<
+      { deps: { bucket: string; region?: string; adminUrl: string } } | { error: string }
+    > => {
+      const bucket = options.backupBucket;
+      if (!bucket) {
+        return { error: 'backup storage not configured — pass backupBucket to dbPlugin (e.g. Resource.Backups.name)' };
+      }
+      const { getAdminDatabaseUrl, getMasterDatabaseUrl } = await import('./db');
+      const adminUrl = getAdminDatabaseUrl() ?? getMasterDatabaseUrl();
+      if (!adminUrl) {
+        return { error: 'no operator connection available (ADMIN_DATABASE_URL / Resource.Database)' };
+      }
+      return { deps: { bucket, region: options.backupRegion ?? process.env.AWS_REGION, adminUrl } };
+    };
+
     // --- migrate ---
     actions.migrate = async (_payload, _ctx) => {
       const { runMigrations } = await import('./migrate');
@@ -636,7 +681,9 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       const { sql } = await import('drizzle-orm');
       const {
         buildDoctorReport, rowToConnInfo, rowToForceRls, rowToAppGranted, rowToSecdefFn,
+        rowToPublicExecutableFn, rowToDarkRelation,
         CONN_SQL, FORCE_RLS_SQL, appGrantsRlsSql, SECDEF_FUNCTIONS_SQL,
+        PUBLIC_EXECUTABLE_FUNCTIONS_SQL, DARK_RELATIONS_SQL,
       } = await import('./db-doctor');
       const rowsOf = (result: any): any[] => (Array.isArray(result) ? result : result?.rows || []);
 
@@ -660,6 +707,10 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       // SECDEF functions (schema, name, search_path pinned?) — the `functions` teeth read these
       // to assert every SECDEF function in a functions-written schema pins its search_path.
       const secdefFunctions = rowsOf(await opsDb.execute(sql.raw(SECDEF_FUNCTIONS_SQL))).map(rowToSecdefFn);
+      // B4 — the derived layer's runtime echoes: PUBLIC-executable functions (NULL proacl
+      // included via acldefault) and superuser-only views/matviews.
+      const publicExecutableFunctions = rowsOf(await opsDb.execute(sql.raw(PUBLIC_EXECUTABLE_FUNCTIONS_SQL))).map(rowToPublicExecutableFn);
+      const darkRelations = rowsOf(await opsDb.execute(sql.raw(DARK_RELATIONS_SQL))).map(rowToDarkRelation);
 
       // Ambient-access probe: can the api connection read an RLS table WITHOUT SET ROLE?
       let apiAmbientRead: { table: string; accessible: boolean } | undefined;
@@ -675,7 +726,7 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       }
 
       return buildDoctorReport(
-        { api, admin, forceRls, appGrantedRls, apiAmbientRead, secdefFunctions },
+        { api, admin, forceRls, appGrantedRls, apiAmbientRead, secdefFunctions, publicExecutableFunctions, darkRelations },
         { forceRlsCarveouts: options.forceRlsCarveouts, functionSchemas: options.functionSchemas },
       );
     };
@@ -759,19 +810,6 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
     // dedicated private backups bucket. Restore is destructive and requires an explicit confirm in
     // the payload (the CLI sets it after the user confirms). See backup-run.ts.
     {
-      const resolveBackupDeps = async () => {
-        const bucket = options.backupBucket;
-        if (!bucket) {
-          return { error: 'backup storage not configured — pass backupBucket to dbPlugin (e.g. Resource.Backups.name)' as string };
-        }
-        const { getAdminDatabaseUrl, getMasterDatabaseUrl } = await import('./db');
-        const adminUrl = getAdminDatabaseUrl() ?? getMasterDatabaseUrl();
-        if (!adminUrl) {
-          return { error: 'no operator connection available (ADMIN_DATABASE_URL / Resource.Database)' as string };
-        }
-        return { deps: { bucket, region: options.backupRegion ?? process.env.AWS_REGION, adminUrl } };
-      };
-
       actions['db:backup'] = async (payload, ctx) => {
         const r = await resolveBackupDeps();
         if ('error' in r) return { error: r.error };
@@ -814,6 +852,123 @@ export function dbPlugin(options: DbPluginOptions): Plugin {
       };
     }
 
+    // --- db:apply — the credential-free staged write (Phase 2) ---
+    // The staged half of db:apply: the CLI computes the plan and the
+    // checkout-bound verdicts (the fast-forward/descent rule, the destructive
+    // approver check — neither can be recomputed here: one needs the git
+    // checkout, the other the caller's real IAM identity); THIS action does the
+    // write against the operator connection. Verify the concurrency lock, apply
+    // the DDL in one transaction, stamp everystack.schema_log with the plan_ref,
+    // and verify the target landed on plan.to — the same executeApplyPlan the
+    // direct --database-url path runs, only here the connection never leaves the
+    // Lambda. A destructive plan snapshots first (db:backup), in order.
+    actions['db:apply'] = async (payload, ctx) => {
+      const { plan, actor, gitRef, descentVerdict, authorityVerdict, stage } =
+        (payload ?? {}) as {
+          plan?: any;
+          actor?: string | null;
+          gitRef?: string | null;
+          descentVerdict?: { ok: boolean; reason?: string };
+          authorityVerdict?: { ok: boolean; reason?: string };
+          stage?: string;
+        };
+      if (!plan || typeof plan !== 'object' || !Array.isArray(plan.statements)) {
+        return { error: 'db:apply requires { plan } — a reviewed edge minted by db:plan' };
+      }
+      const { executeApplyPlan, PLAN_VERSION } = await import('@everystack/cli/apply');
+      if (plan.v !== PLAN_VERSION) {
+        return { error: `plan version ${plan.v} is not this server's ${PLAN_VERSION} — re-mint with a matching db:plan` };
+      }
+
+      const { sql } = await import('drizzle-orm');
+      const runner = async (query: string): Promise<any[]> => {
+        const result: any = await opsDb.execute(sql.raw(query));
+        return Array.isArray(result) ? result : result?.rows ?? [];
+      };
+
+      const destructive = Number(plan.destructive) > 0;
+      let snapshotId: string | undefined;
+      try {
+        const result = await executeApplyPlan(runner, plan, {
+          actor: actor ?? null,
+          gitRef: gitRef ?? null,
+          // Descent/authority are decided by the CLI (checkout + caller identity)
+          // and enforced + recorded here, in order, atomic with the lock.
+          ...(descentVerdict
+            ? { verifyDescent: async () => (descentVerdict.ok ? { ok: true as const } : { ok: false as const, reason: descentVerdict.reason ?? 'descent refused' }) }
+            : {}),
+          ...(destructive && authorityVerdict
+            ? { verifyAuthority: async () => (authorityVerdict.ok ? { ok: true as const } : { ok: false as const, reason: authorityVerdict.reason ?? 'authority refused' }) }
+            : {}),
+          // Losing data is hard: the snapshot runs HERE, after the lock/descent/
+          // authority gates pass and before the DDL. It uses the same logical
+          // backup as db:backup, on the operator connection.
+          ...(destructive
+            ? {
+                beforeDestructive: async () => {
+                  const r = await resolveBackupDeps();
+                  if ('error' in r) {
+                    throw new Error(`snapshot before a destructive apply is impossible — ${r.error}`);
+                  }
+                  const { runBackup } = await import('./backup-run');
+                  const backup: any = await runBackup(r.deps, stage ?? ctx.environment ?? 'default');
+                  snapshotId = backup?.id;
+                },
+              }
+            : {}),
+        });
+        return snapshotId ? { ...result, snapshotId } : result;
+      } catch (err: any) {
+        return { error: err?.message || String(err) };
+      }
+    };
+
+    // --- pipeline:run — the credential-free ingest reproduction (Phase 3, Brick 5) ---
+    // The orchestrator runs HERE, in the ops Lambda, over the operator connection: a
+    // deployed stage's small curated overlays reproduce via IAM, never a held URL. The
+    // pipeline definition is the one the app bundled into this Lambda (like `seed`); the
+    // CLI passes only the run options. A `heavy` stage is refused — it belongs local-direct.
+    if (options.pipeline) {
+      const pipeline = options.pipeline;
+      actions['pipeline:run'] = async (payload, _ctx) => {
+        const runOpts: any = { ...((payload ?? {}) as Record<string, unknown>) };
+        const { runPipeline, topoSortStages, selectStages, latestRunId } = await import('./pipeline');
+
+        let selected;
+        try {
+          selected = selectStages(topoSortStages(pipeline.stages), runOpts);
+        } catch (err: any) {
+          return { error: err?.message || String(err) };
+        }
+        const heavy = selected.filter((s) => s.heavy).map((s) => s.id);
+        if (heavy.length > 0) {
+          return { error: `stage(s) ${heavy.join(', ')} are marked heavy — too big for the ops Lambda. Run them local-direct: pipeline:run --database-url <url>.` };
+        }
+
+        const { sql } = await import('drizzle-orm');
+        const asRows = (r: any): any[] => (Array.isArray(r) ? r : r?.rows ?? []);
+        const runner = {
+          query: async (q: string) => asRows(await opsDb.execute(sql.raw(q))),
+          transaction: async <T>(fn: (tx: (q: string) => Promise<any[]>) => Promise<T>): Promise<T> =>
+            opsDb.transaction(async (tx: any) => fn(async (q: string) => asRows(await tx.execute(sql.raw(q))))),
+        };
+
+        // Resume with no explicit run id picks up the latest run on THIS stage.
+        if (runOpts.resume && !runOpts.runId) {
+          const latest = await latestRunId(runner.query);
+          if (!latest) return { error: 'nothing to resume — this pipeline has not run on this stage yet.' };
+          runOpts.runId = latest;
+        }
+        if (!runOpts.runId) return { error: 'pipeline:run requires a runId' };
+
+        try {
+          return await runPipeline(runner, pipeline, runOpts);
+        } catch (err: any) {
+          return { error: err?.message || String(err) };
+        }
+      };
+    }
+
     // --- console ---
     if (options.allowConsole !== false) {
       actions['console:meta'] = async (_payload, ctx) => {