@everystack/cli 0.3.27 → 0.3.31

package/src/cli/derived-plan.ts

@@ -8,7 +8,8 @@
  * overwrite explicit, and the drift stays in the report so the operator sees
  * what they overwrote). Never-reconciled matches are `needsBaseline`: the
  * reconciler refuses to guess whether live matches source — `baseline: true`
- * records the trust explicitly, once.
+ * records the trust explicitly, once, or `rebuild: true` rebuilds them from
+ * source (drop+create) so the match is guaranteed rather than asserted.
  *
  * Graph work: replacing (or dropping) a relation rebuilds its live transitive
  * dependents — dropped in reverse-topological order, recreated in source
@@ -72,6 +73,11 @@ export interface ReconcileOptions {
   baseline?: boolean;
   /** Rebuild drifted objects from source (the drift finding stays in the report). */
   overwriteDrift?: boolean;
+  /**
+   * Rebuild never-reconciled source↔live matches from source (drop+create) instead of trusting
+   * them via baseline — the guarantee that live == db/sql when first contact can't verify it.
+   */
+  rebuild?: boolean;
 }
 
 const isRelation = (kind: DerivedKind): boolean => kind !== 'function';
@@ -117,8 +123,18 @@ export function planReconcile(
       continue;
     }
     if (!prov) {
-      if (options.baseline) baseline.push(src.identity);
-      else needsBaseline.push(src.identity);
+      // First contact: no provenance, and src-hash (source text) can't be compared to def-hash
+      // (live deparse), so the reconciler can't verify live == source. --baseline trusts it,
+      // --rebuild guarantees it by rebuilding from source (relations through the dependency graph
+      // below; functions replace in place), and the default flags it for a decision.
+      if (options.rebuild) {
+        if (isRelation(src.kind)) rebuild.set(src.identity, 'rebuilt from source (first contact, --rebuild)');
+        else fnReplace.set(src.identity, 'replaced from source (first contact, --rebuild)');
+      } else if (options.baseline) {
+        baseline.push(src.identity);
+      } else {
+        needsBaseline.push(src.identity);
+      }
       continue;
     }
 

package/src/cli/discover.ts

@@ -423,6 +423,22 @@ export async function invalidateCacheByFunctionName(staleName: string): Promise<
   }
 }
 
+/**
+ * Delete the per-stage config cache for one stage, forcing the next stage command to
+ * re-discover live. Called after `everystack deploy` (a deploy can add a function the old
+ * cache never saw — the Ops function is the canonical case: its name stays absent from a
+ * config discovered before it existed, and the 24h TTL keeps that stale config valid, so
+ * db ops misroute to the Api function) and by `--refresh`. Best-effort; only touches the
+ * `outputs.{stage}.json` cache, never the deploy-written `outputs.json`.
+ */
+export async function invalidateCachedConfig(stage: string): Promise<void> {
+  try {
+    await fs.unlink(cachePath(stage));
+  } catch {
+    // No cache for this stage — nothing to invalidate.
+  }
+}
+
 export async function setCachedConfig(stage: string, config: DiscoveredConfig): Promise<void> {
   const data: CachedOutputs = {
     _cachedAt: Date.now(),

package/src/cli/index.ts

@@ -20,6 +20,7 @@ import { dbApplyCommand } from './commands/db-apply.js';
 import { dbCheckCommand } from './commands/db-check.js';
 import { dbApproversCommand } from './commands/db-approvers.js';
 import { dbBackfillCommand } from './commands/db-backfill.js';
+import { pipelineRunCommand, pipelineListCommand } from './commands/pipeline-run.js';
 import { dbTemplateRefreshCommand, dbBranchCommand } from './commands/db-branch.js';
 import { dbForkCommand } from './commands/db-fork.js';
 import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
@@ -114,6 +115,14 @@ async function main() {
     process.exit(0);
   }
 
+  // `--refresh` forces re-discovery: drop the stale per-stage cache before anything reads it
+  // (loadSstOutputs and resolveConfig both consult it). Use after a deploy the CLI didn't run,
+  // or whenever a stage command resolves to the wrong function.
+  if (flags.refresh && flags.stage) {
+    const { invalidateCachedConfig } = await import('./discover.js');
+    await invalidateCachedConfig(flags.stage);
+  }
+
   // Auto-detect deployed URL from SST outputs
   await loadSstOutputs(flags.stage);
 
@@ -210,6 +219,12 @@ async function main() {
     case 'db:backfill':
       await dbBackfillCommand(flags);
       break;
+    case 'pipeline:run':
+      await pipelineRunCommand(flags);
+      break;
+    case 'pipeline:list':
+      await pipelineListCommand(flags);
+      break;
     case 'db:template:refresh':
       await dbTemplateRefreshCommand(flags);
       break;
@@ -342,14 +357,16 @@ Usage:
   everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
   everystack db:fingerprint    [--stage <name> | --database-url <url>] [--models <barrel>] [--json]  Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
-  everystack db:reconcile      [--sql-dir db/sql] [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews) against db/sql sources: plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only) and records provenance + schema_log. Hand-edits are drift (never overwritten silently); first contact with existing objects wants --baseline, once.
+  everystack db:reconcile      [--sql-dir db/sql] [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews) against db/sql sources: plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only, atomic — DDL + provenance in one transaction) and records provenance + schema_log. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive.
   everystack db:sync           [--database-url <url>] [--models <barrel>] [--sql-dir db/sql] [--schema-out <file.ts>] [--allow-drops] [--overwrite-drift] [--baseline] [--json]  Make the database match your checkout — one verb, both layers: apply the state diff (tables+authz, one transaction, verified by re-diff), reconcile the derived layer against db/sql, report the resulting fingerprint vs the models' declared one. Dev databases only (direct connection required); DROPs held back unless --allow-drops; derived drift refuses unless --overwrite-drift; exit 1 when not converged
   everystack db:diff           --from-models <barrel> [--to-models db/models/index.ts] [--allow-drops] [--check] [--json]  The state edge between two declared states, NO database: the SQL db:generate would produce, computed purely — CI plan previews (--check exits 1 on a non-empty edge) and computed rollbacks (swap the flags)
   everystack db:plan           [--stage <name> | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -]  Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only — works via the ops Lambda; plans are ephemeral, never committed
   everystack db:apply          --plan <file.plan.json> [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--snapshot-ref <ref>] [--force-descent <snapshot-ref> --confirm]  Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule, else refuse — "rebase first"), and for DESTRUCTIVE plans require --confirm always + a snapshot (automatic via db:backup with --stage, else --snapshot-ref) + the stage's approver set when declared (STS identity-verified). Every refusal is recorded in schema_log. Apply as one transaction (plan_ref stamped), verify it landed exactly on plan.to; idempotent when already there; direct connection required
-  everystack db:check          [--models <barrel>] [--sql-dir db/sql] [--schema-out <file.ts>] [--database-url <url>] [--json]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, db/sql parses) and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
+  everystack db:check          [--models <barrel>] [--sql-dir db/sql] [--schema-out <file.ts>] [--database-url <url>] [--json]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, db/sql parses), every exposed RLS-enabled table must declare a read path (no force-RLS-with-no-read landmine that goes dark on the superuser drop), and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
   everystack db:approvers      --stage <name> [--set "cto,arn:..."] [--remove]  Declare who can DESTROY: the stage's destructive-approver set (SSM parameter, admin-writable). Destructive db:apply runs are then identity-verified (STS) against it; --set '' disables destructive applies; --remove returns the stage to ceremony-only
   everystack db:backfill       [--database-url <url>] [--dir db/backfills] [--apply] [--mark-applied <file.sql>] [--json]  One-shot data jobs in their own lane: plan shows applied (by CONTENT identity — renames/comment edits are no-ops) / pending (in order, unbounded-pass advisories) / blocked (a name that already ran in a different form — one-shot jobs are immutable). --apply runs each pending job as its own transaction, recorded in everystack.backfill_log (a failure rolls back alone, is recorded, stops the run); --mark-applied records without running. Never runs as a schema side effect; direct connection required
+  everystack pipeline:run      [--stage <name> | --database-url <url>] [--rebuild | --curate] [--only <substr> | --from <id> | --to <id>] [--resume [--run-id <id>]] [--dry-run] [--continue-on-error] [--json]  Reproduce data from committed source: run the pipeline's stages in topological order, each idempotent and inside its own transaction, recorded in everystack.pipeline_log. Lanes: --rebuild (automated+frozen) / --curate (curated) / neither (all in order). --resume picks up the latest run, skipping applied stages. With --stage the orchestrator runs credential-free IN the ops Lambda (no URL held; heavy stages refused — run those local); --database-url is local-direct and unbounded
+  everystack pipeline:list     [--rebuild | --curate] [--only <substr> | --from <id> | --to <id>]  Show the pipeline's stages in run order (topological) with their lane and dependency edges — no execution, no database
   everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--sql-dir db/sql] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
   everystack db:branch         [--list | --drop --confirm | --prune --confirm] [--database-url <url>]  Mint (or find) the current git branch's database from the template (CREATE DATABASE … TEMPLATE — schema, authz, derived layer, and seed rows inherited), print its DATABASE_URL, then db:sync evolves it with the checkout. --list maps every branch DB to its branch; --prune drops the ones whose branch is gone (never guesses: unknown mappings are kept)
   everystack db:fork           --from-stage <src> --stage <target> --confirm [--backup <id>]  Fork one DEPLOYED stage's database into another: back up the source (or reuse --backup <id>), presign the dump (the operator's credentials ARE the cross-stage authorization; expires in 1h), restore into the target via its ops Lambda. Production is never a target (that's db:restore); forking FROM production warns about PII; the branch's schema edge then lands via db:plan → db:apply (descent composes). Teardown: sst remove --stage <target>
@@ -398,6 +415,9 @@ Stage resolution:
   --stage <name>  Discover AWS resources for the given stage by querying AWS APIs.
                   Uses SST naming convention: {app}-{stage}-{Resource}-{hash}.
                   Results are cached in .sst/outputs.{stage}.json for 24 hours.
+  --refresh       Drop the cached stage config and re-discover live. Use after a
+                  deploy the CLI didn't run, or if a stage command resolves to the
+                  wrong function. \`everystack deploy\` refreshes this cache for you.
 
   Without --stage, reads .sst/outputs.json (written by the last \`sst deploy\`).
 

package/src/cli/ops-diagnostics.ts

@@ -0,0 +1,71 @@
+/**
+ * Diagnostics for a crashed ops-Lambda invoke.
+ *
+ * A consumer's ops handler can fail to *boot* — a broken top-level import (a dependency missing
+ * from the deployed bundle, a version skew, a bad path). Lambda then returns a FunctionError with
+ * the opaque `Runtime exited with error: exit status 1` for EVERY action, giving the operator no
+ * signal that it's the handler failing to start (not their command) and no path to the cause. The
+ * CLI turns that into a report naming the real cause by reading the handler's CloudWatch logs; the
+ * pure detection/extraction/formatting lives here, the AWS fetch in aws.ts.
+ */
+
+/**
+ * Whether a Lambda FunctionError message is the handler failing to boot / crashing the runtime — an
+ * uncaught init error worth chasing into CloudWatch — rather than an already-descriptive error. Only
+ * these opaque cases trigger the log fetch; a clear message passes through untouched.
+ */
+export function isOpsHandlerCrash(message: string): boolean {
+  return /Runtime exited|exited with error|Runtime\.(ImportModuleError|UserCodeSyntaxError|HandlerNotFound)|Cannot find (module|package)|ERR_MODULE_NOT_FOUND|\bMODULE_NOT_FOUND\b/i.test(
+    message,
+  );
+}
+
+/** Log lines worth surfacing: the import error, a runtime error class, or a thrown Error/stack. */
+const SIGNAL = /Cannot find (module|package)|ERR_MODULE_NOT_FOUND|\bMODULE_NOT_FOUND\b|Runtime\.\w+|(?:Reference|Type|Syntax|Range)?Error:/;
+
+/**
+ * Pull the diagnostic lines from raw CloudWatch log text for a crashed ops invoke — the import
+ * error / stack frame that names the missing dependency — so an opaque "exit status 1" becomes the
+ * real cause. De-duplicates and caps output. Returns null when nothing diagnostic is present.
+ */
+export function extractCrashDiagnosis(logText: string): string | null {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const raw of logText.split('\n')) {
+    const line = raw.trim();
+    if (!line || !SIGNAL.test(line) || seen.has(line)) continue;
+    seen.add(line);
+    out.push(line);
+    if (out.length >= 6) break;
+  }
+  return out.length ? out.join('\n') : null;
+}
+
+/**
+ * Assemble the operator-facing report for an ops-handler crash: the raw message, the "it's the
+ * handler, not your command" framing, and either the real cause from CloudWatch or a where-to-look
+ * hint when the logs couldn't be read.
+ */
+export function formatOpsCrashReport(
+  baseMessage: string,
+  logGroup?: string,
+  diagnosis?: string | null,
+): string {
+  const lines = [
+    baseMessage,
+    '',
+    'The ops handler crashed at startup — usually a missing dependency in the deployed bundle, not a problem with this command.',
+  ];
+  if (diagnosis) {
+    lines.push(
+      '',
+      `From CloudWatch${logGroup ? ` (${logGroup})` : ''}:`,
+      diagnosis,
+      '',
+      'Fix the import/dependency and redeploy.',
+    );
+  } else {
+    lines.push('Check its CloudWatch logs for the failing top-level import, then redeploy.');
+  }
+  return lines.join('\n');
+}

package/src/cli/pipeline-loader.ts

@@ -0,0 +1,68 @@
+/**
+ * pipeline-loader — the CLI's window onto @everystack/server/pipeline.
+ *
+ * The CLI keeps NO compile-time dependency on @everystack/server (the two are
+ * mutual peers; hard-importing would be a project-reference cycle — the same
+ * reason plugin.ts inlines its types). So the orchestrator's types are mirrored
+ * structurally here and the runtime module is loaded lazily via a variable
+ * specifier. Every CLI verb that reaches the pipeline framework (pipeline:run,
+ * and the db:sync / db:plan visibility) goes through this one door.
+ */
+
+export type PipelineLane = 'rebuild' | 'curate' | 'all';
+
+export interface PipelineStage {
+  id: string;
+  lane: 'automated' | 'curated' | 'frozen';
+  dependsOn?: string[];
+  after?: string[];
+  heavy?: boolean;
+}
+export interface Pipeline {
+  stages: PipelineStage[];
+}
+export interface RunPipelineOptions {
+  runId: string;
+  lane?: PipelineLane;
+  only?: string;
+  from?: string;
+  to?: string;
+  resume?: boolean;
+  dryRun?: boolean;
+  continueOnError?: boolean;
+  actor?: string | null;
+  gitRef?: string | null;
+}
+export interface StageRunResult {
+  stageId: string;
+  outcome: 'applied' | 'skipped' | 'failed' | 'dry-run';
+  rows?: number;
+  error?: string;
+}
+export interface PipelineRunOutcome {
+  runId: string;
+  plan: string[];
+  results: StageRunResult[];
+  failed: boolean;
+}
+export interface PipelineRunSummary {
+  runId: string;
+  at: string | null;
+  applied: number;
+  failed: number;
+  failedStages: string[];
+}
+
+export interface ServerPipeline {
+  runPipeline(runner: unknown, pipeline: Pipeline, options: RunPipelineOptions): Promise<PipelineRunOutcome>;
+  latestRunId(query: (sql: string) => Promise<any[]>): Promise<string | null>;
+  readLastRun(query: (sql: string) => Promise<any[]>): Promise<PipelineRunSummary | null>;
+  topoSortStages(stages: PipelineStage[]): PipelineStage[];
+  selectStages(ordered: PipelineStage[], opts: Record<string, unknown>): PipelineStage[];
+}
+
+/** Lazily load the orchestrator from the @everystack/server peer (runtime only). */
+export async function loadServerPipeline(): Promise<ServerPipeline> {
+  const spec = '@everystack/server/pipeline';
+  return (await import(spec)) as ServerPipeline;
+}

package/src/cli/pipeline-path.ts

@@ -0,0 +1,25 @@
+/**
+ * pipeline-path — where the consumer's pipeline definition lives.
+ *
+ * Beside the schema homes (`db/models/`, `db/sql/`, `db/backfills/`), the
+ * pipeline is authored at `db/pipeline.ts` (or `db/pipeline/index.ts`),
+ * exporting a `pipeline` built with definePipeline. An explicit `--pipeline`
+ * always wins; when neither home exists the primary is returned so error
+ * messages point at the current convention.
+ */
+
+import { existsSync } from 'node:fs';
+
+/** Candidate pipeline entrypoints, in preference order. */
+export const PIPELINE_HOMES = ['db/pipeline.ts', 'db/pipeline/index.ts'] as const;
+
+export function resolvePipelinePath(
+  flag?: string,
+  exists: (p: string) => boolean = existsSync,
+): string {
+  if (flag) return flag;
+  for (const home of PIPELINE_HOMES) {
+    if (exists(home)) return home;
+  }
+  return PIPELINE_HOMES[0];
+}

package/src/cli/schema-fingerprint.ts

@@ -27,6 +27,13 @@
  *  - FUNCTIONS are excluded — they belong to the derived layer, whose own
  *    content hashes (derived-introspect) cover them. Base fingerprint + derived
  *    manifest together are the full schema state.
+ *  - Expressions (column defaults, check predicates, partial-index WHERE) hash
+ *    through the SAME normalizers the diff uses (`normalizeDefault` /
+ *    `normalizeCheck`), applied to both sides in `canonicalizeState`. Without
+ *    this the model side hashes its source form (`'primary'`, `scope = ANY(...)`)
+ *    while the live side hashes pg's deparse (`'primary'::text`,
+ *    `(scope = ANY(...))`) — the identity below breaks on any schema with a text
+ *    default, a check, or a partial index even when the diff is a no-op.
  *  - Deparse stability caveat, stated once: policy/check predicates hash as
  *    Postgres deparses them; a major-version deparser change can shift
  *    fingerprints. The version prefix plus the re-diff path keep that honest.
@@ -43,8 +50,12 @@ import type { SchemaSnapshot, TableSchema } from './schema-introspect.js';
 import type { AuthzContract, TableContract } from './authz-contract.js';
 import { compileTableSchema, compileEnums } from './schema-compile.js';
 import { compileTableContract } from './authz-compile.js';
+import { normalizeDefault, normalizeCheck } from './schema-diff.js';
 
-export const FINGERPRINT_VERSION = 1;
+// v2: expression fields (defaults, checks, partial-index WHERE) normalize through
+// the diff's pg-deparse normalizers before hashing, so the model form and the live
+// deparsed form agree — the identity `fingerprintModels === fingerprintLive` holds.
+export const FINGERPRINT_VERSION = 2;
 
 // ---------------------------------------------------------------------------
 // Canonical form.
@@ -70,13 +81,17 @@ function canonicalTable(table: TableSchema): Record<string, unknown> {
   return {
     table: table.table,
     columns: byKey(
-      table.columns.map((c) => ({ name: c.name, type: c.type, notNull: c.notNull, default: c.default })),
+      // Defaults hash SEMANTICALLY, through the same normalizer the diff uses: the Model
+      // emits `'primary'`, pg deparses it to `'primary'::text` — the same state, one hash.
+      table.columns.map((c) => ({ name: c.name, type: c.type, notNull: c.notNull, default: normalizeDefault(c.default) })),
       (c) => c.name,
     ),
     primaryKey: table.primaryKey,
     // Content, not names: a pg-default `_key` and a model-named unique are the same state.
     uniques: byKey(table.uniques.map((u) => ({ columns: u.columns })), (u) => u.columns.join(',')),
-    checks: byKey(table.checks.map((c) => ({ expr: c.expr })), (c) => c.expr),
+    // Predicates hash as the diff compares them — one layer of pg's wrapping parens dropped
+    // (`(scope = ANY (...))` == `scope = ANY (...)`), whitespace collapsed.
+    checks: byKey(table.checks.map((c) => ({ expr: normalizeCheck(c.expr) })), (c) => c.expr),
     foreignKeys: byKey(
       table.foreignKeys.map((f) => ({
         columns: f.columns, refTable: f.refTable, refColumns: f.refColumns,
@@ -86,7 +101,8 @@ function canonicalTable(table: TableSchema): Record<string, unknown> {
       (f) => stableStringify(f),
     ),
     indexes: byKey(
-      table.indexes.map((i) => ({ columns: i.columns, unique: i.unique, ...(i.where ? { where: i.where } : {}) })),
+      // Partial-index WHERE is a predicate too — normalize it the same way as a check.
+      table.indexes.map((i) => ({ columns: i.columns, unique: i.unique, ...(i.where ? { where: normalizeCheck(i.where) } : {}) })),
       (i) => stableStringify(i),
     ),
   };