@everystack/cli 0.3.28 → 0.4.0

package/src/cli/model-render.ts

@@ -16,6 +16,7 @@
  */
 
 import type { SchemaSnapshot, TableSchema, ColumnSchema, CheckConstraint } from './schema-introspect.js';
+import type { DerivedRenderResult } from './derived-render.js';
 import { normalizeDefault, normalizeCheck } from './schema-diff.js';
 
 /**
@@ -59,6 +60,47 @@ export function checkToValidate(expr: string): { column: string; zod: string } |
 export interface RenderOptions {
   /** Only pull tables in this Postgres schema (others are framework-managed). Default: `public`. */
   schema?: string;
+  /**
+   * The read-model scaffold. Default `'commented'`: every model carries the authz decision
+   * as a commented stanza (same guidance as the db:check gate — the model fails the gate
+   * until it is authored). A preset name (`'public-read'`) stamps that stanza uncommented
+   * into every rendered model — an explicit, one-shot, operator-invoked choice that lands
+   * as reviewable code. Grants are authored, never inherited; there is no runtime default.
+   */
+  abilities?: string;
+  /** The rendered derived layer (B5) — rides the barrel: block after the models,
+   *  sequences/derived arrays on the module wrapper, symbols on the import header. */
+  derived?: DerivedRenderResult;
+}
+
+/**
+ * The `--abilities` presets db:pull can stamp. Deliberately few: `public-read` is the
+ * dominant brownfield shape (public reference data, admin-managed). Anything else is a
+ * per-model edit — the decision belongs in the file, not in a flag grammar.
+ */
+export const ABILITY_PRESETS: Record<string, string> = {
+  'public-read': `abilities: [can('read'), can('manage', { role: 'admin' })],`,
+};
+
+/**
+ * The scaffold stanza for one model: the commented decision surface (guide text says the
+ * same thing the db:check gate failure says — one voice, two doorways), or a preset
+ * stamped uncommented. An unknown preset throws — grants are authored, never guessed.
+ */
+function abilitiesStanza(mode: string): string {
+  if (mode === 'commented') {
+    return [
+      '  // Declare the read model — db:check fails this model until its authz is authored:',
+      `  // abilities: [can('read')],                          // public data`,
+      `  // abilities: [can('read', { owner: '<column>' })],   // rows owned by a user`,
+      '  // private: true,                                     // not part of the data API',
+    ].join('\n');
+  }
+  const preset = ABILITY_PRESETS[mode];
+  if (!preset) {
+    throw new Error(`Unknown --abilities preset '${mode}' — known: ${Object.keys(ABILITY_PRESETS).join(', ')} (or omit the flag for the commented scaffold).`);
+  }
+  return `  ${preset}`;
 }
 
 /** `format_type` → the `field.*()` factory that produces it (the non-parameterized types). */
@@ -210,8 +252,11 @@ function renderField(table: TableSchema, col: ColumnSchema, known: Set<string>,
 
   if (col.default != null && !serial) {
     const d = renderDefault(col.default);
-    if (d) expr += d;
-    else comment += ` // TODO: unmapped default ${col.default}`;
+    // No builder maps it → declare it verbatim with the escape hatch. RAW, not the
+    // normalized form — stripping a trailing cast here could change semantics
+    // (`'7 days'::interval`); the diff normalizes both sides for comparison instead.
+    // Drift closed by construction: pull output converges instead of shedding a TODO.
+    expr += d ?? `.defaultSql(sql\`${col.default}\`)`;
   }
 
   const validate = validates.get(col.name);
@@ -236,8 +281,14 @@ function renderConstraintsBlock(table: TableSchema, checks: CheckConstraint[], k
     items.push(`check(sql\`${normalizeCheck(ck.expr)}\`)`);
   }
   for (const ix of table.indexes) {
-    let s = `index(${ix.columns.map((c) => tsLiteral(toCamelCase(c))).join(', ')})`;
+    // Plain entries render as field keys; canonical-text entries (expressions, DESC,
+    // opclass) render as raw sql`` args — the same form the builder declares (Brick E).
+    const entry = (c: string): string =>
+      /^[a-z_][a-z0-9_$]*$/i.test(c) ? tsLiteral(toCamelCase(c)) : `sql\`${c}\``;
+    let s = `index(${ix.columns.map(entry).join(', ')})`;
+    if (ix.using) s += `.using(${tsLiteral(ix.using)})`;
     if (ix.unique) s += '.unique()';
+    if (ix.include?.length) s += `.include(${ix.include.map((c) => tsLiteral(toCamelCase(c))).join(', ')})`;
     if (ix.where) s += `.where(sql\`${normalizeCheck(ix.where)}\`)`;
     items.push(s);
   }
@@ -261,7 +312,7 @@ function renderConstraintsBlock(table: TableSchema, checks: CheckConstraint[], k
 }
 
 /** One `export const X = defineModel(...)` block for a table. */
-export function renderModelBlock(table: TableSchema, known: Set<string>, enums: Map<string, string[]> = new Map()): string {
+export function renderModelBlock(table: TableSchema, known: Set<string>, enums: Map<string, string[]> = new Map(), abilities = 'commented'): string {
   // A CHECK that reverses to a single field's .validate() is rendered on the field (ergonomic);
   // the rest stay table-level check(). Both round-trip — this only chooses the nicer form.
   const validates = new Map<string, string>();
@@ -275,23 +326,52 @@ export function renderModelBlock(table: TableSchema, known: Set<string>, enums:
 
   const fields = table.columns.map((c) => renderField(table, c, known, enums, validates)).join('\n');
   const constraints = renderConstraintsBlock(table, tableChecks, known);
+  // The authz decision renders FIRST — before fields — because it is the first thing a
+  // reviewer must resolve about a model (and where the field-report consumer's codemod
+  // put it, proving the position is mechanical-edit-friendly).
+  const stanza = abilitiesStanza(abilities);
 
-  return `export const ${modelVarName(table.table)} = defineModel('${bareName(table.table)}', {\n  fields: {\n${fields}\n  },${constraints}\n});`;
+  return `export const ${modelVarName(table.table)} = defineModel('${bareName(table.table)}', {\n${stanza}\n  fields: {\n${fields}\n  },${constraints}\n});`;
 }
 
 /** The `import` lines a rendered body needs — only what it actually uses, so the file reads clean. */
-function importHeader(body: string): string {
+function importHeader(body: string, extra: string[] = []): string {
   const symbols = ['defineModel', 'field'];
+  // `can` only when a stanza is stamped UNCOMMENTED (line starts with spaces, not `//`) —
+  // the commented scaffold must not leave an unused import behind.
+  if (/^\s*abilities: \[can\(/m.test(body)) symbols.push('can');
   if (/\bunique\(/.test(body)) symbols.push('unique');
   if (/\bcheck\(/.test(body)) symbols.push('check');
   if (/\bindex\(/.test(body)) symbols.push('index');
   if (/\bforeignKey\(/.test(body)) symbols.push('foreignKey');
+  if (/\bdefineModule\(/.test(body)) symbols.push('defineModule');
   if (/\bsql`/.test(body)) symbols.push('sql');
+  for (const s of extra) if (!symbols.includes(s)) symbols.push(s);
   const imports = [`import { ${symbols.join(', ')} } from '@everystack/model';`];
   if (/\.validate\(/.test(body)) imports.push(`import { z } from 'zod';`);
   return imports.join('\n');
 }
 
+/** The barrel's module wrapper: models always; sequences/derived when the pull found any (B5). */
+function moduleFooter(modelNames: string[], derived?: DerivedRenderResult, multiline = false): string {
+  const models = multiline
+    ? `export const models = [\n${modelNames.map((n) => `  ${n},`).join('\n')}\n];`
+    : `export const models = [${modelNames.join(', ')}];`;
+  const parts = [models];
+  const keys = ['models'];
+  if (derived?.sequenceNames.length) {
+    parts.push(`export const sequences = [${derived.sequenceNames.join(', ')}];`);
+    keys.push('sequences');
+  }
+  if (derived?.names.length) {
+    parts.push(`export const derived = [${derived.names.join(', ')}];`);
+    keys.push('derived');
+  }
+  parts.push(`export const appModule = defineModule({ ${keys.join(', ')} });`);
+  parts.push(`export const modules = [appModule];`);
+  return parts.join('\n\n');
+}
+
 /**
  * Render a whole snapshot as a single Models module: the import, one block per table (scoped
  * to `opts.schema`), and the `models` array the rest of the framework consumes. The tables
@@ -303,11 +383,16 @@ export function renderModelSource(snapshot: SchemaSnapshot, opts: RenderOptions
   const known = new Set(tables.map((t) => bareName(t.table)));
   const enums = new Map((snapshot.enums ?? []).map((e) => [e.name, e.values]));
 
-  const blocks = tables.map((t) => renderModelBlock(t, known, enums));
-  const body = blocks.join('\n\n');
+  const blocks = tables.map((t) => renderModelBlock(t, known, enums, opts.abilities ?? 'commented'));
+  // The derived layer (B5): sequences + views/matviews/functions, after the models they
+  // reference, before the module wrapper that composes all three.
+  if (opts.derived?.block) blocks.push(opts.derived.block);
+
+  // Module composition is the norm: the pulled barrel exports `modules` alongside `models`,
+  // so a fresh brownfield project lands on the same shape the framework composes.
+  const footer = moduleFooter(tables.map((t) => modelVarName(t.table)), opts.derived);
 
-  const header = importHeader(body);
-  const footer = `export const models = [${tables.map((t) => modelVarName(t.table)).join(', ')}];`;
+  const header = importHeader([...blocks, footer].join('\n\n'), opts.derived?.imports ?? []);
 
   return [header, ...blocks, footer].join('\n\n') + '\n';
 }
@@ -349,7 +434,7 @@ export function renderModelFiles(snapshot: SchemaSnapshot, opts: RenderOptions =
   const enums = new Map((snapshot.enums ?? []).map((e) => [e.name, e.values]));
 
   const files: RenderedModelFile[] = tables.map((t) => {
-    const block = renderModelBlock(t, known, enums);
+    const block = renderModelBlock(t, known, enums, opts.abilities ?? 'commented');
     const crossImports = referencedTables(t, known).map(
       (target) => `import { ${modelVarName(target)} } from './${bareName(target).replace(/_/g, '-')}';`,
     );
@@ -357,11 +442,19 @@ export function renderModelFiles(snapshot: SchemaSnapshot, opts: RenderOptions =
     return { file: modelFileName(t.table), source: `${header}\n\n${block}\n` };
   });
 
+  // The index carries the module wrapper (composition is the norm) and the derived layer
+  // (B5 — it references the model variables the index imports); model files carry only
+  // their own model.
   const names = tables.map((t) => modelVarName(t.table));
+  const modelSymbols = ['defineModule', ...(opts.derived?.imports ?? [])];
   const index = [
-    names.map((n, i) => `import { ${n} } from './${bareName(tables[i].table).replace(/_/g, '-')}';`).join('\n'),
+    [
+      `import { ${modelSymbols.join(', ')} } from '@everystack/model';`,
+      ...names.map((n, i) => `import { ${n} } from './${bareName(tables[i].table).replace(/_/g, '-')}';`),
+    ].join('\n'),
     `export {\n${names.map((n) => `  ${n},`).join('\n')}\n};`,
-    `export const models = [\n${names.map((n) => `  ${n},`).join('\n')}\n];`,
+    ...(opts.derived?.block ? [opts.derived.block] : []),
+    moduleFooter(names, opts.derived, true),
   ].join('\n\n');
   files.push({ file: 'index.ts', source: index + '\n' });
 

package/src/cli/models-path.ts

@@ -1,7 +1,7 @@
 /**
  * models-path — where the Models barrel lives.
  *
- * One home for schema: `db/models/` (state) beside `db/sql/` (compute). The
+ * One home for schema: `db/models/` — state (tables) and compute (descriptors). The
  * CLI looks there first, falling back to the legacy top-level `models/` so
  * existing consumers keep working without a flag. An explicit `--models`
  * always wins. When neither home exists, the NEW home is returned so error

package/src/cli/ops-diagnostics.ts

@@ -0,0 +1,71 @@
+/**
+ * Diagnostics for a crashed ops-Lambda invoke.
+ *
+ * A consumer's ops handler can fail to *boot* — a broken top-level import (a dependency missing
+ * from the deployed bundle, a version skew, a bad path). Lambda then returns a FunctionError with
+ * the opaque `Runtime exited with error: exit status 1` for EVERY action, giving the operator no
+ * signal that it's the handler failing to start (not their command) and no path to the cause. The
+ * CLI turns that into a report naming the real cause by reading the handler's CloudWatch logs; the
+ * pure detection/extraction/formatting lives here, the AWS fetch in aws.ts.
+ */
+
+/**
+ * Whether a Lambda FunctionError message is the handler failing to boot / crashing the runtime — an
+ * uncaught init error worth chasing into CloudWatch — rather than an already-descriptive error. Only
+ * these opaque cases trigger the log fetch; a clear message passes through untouched.
+ */
+export function isOpsHandlerCrash(message: string): boolean {
+  return /Runtime exited|exited with error|Runtime\.(ImportModuleError|UserCodeSyntaxError|HandlerNotFound)|Cannot find (module|package)|ERR_MODULE_NOT_FOUND|\bMODULE_NOT_FOUND\b/i.test(
+    message,
+  );
+}
+
+/** Log lines worth surfacing: the import error, a runtime error class, or a thrown Error/stack. */
+const SIGNAL = /Cannot find (module|package)|ERR_MODULE_NOT_FOUND|\bMODULE_NOT_FOUND\b|Runtime\.\w+|(?:Reference|Type|Syntax|Range)?Error:/;
+
+/**
+ * Pull the diagnostic lines from raw CloudWatch log text for a crashed ops invoke — the import
+ * error / stack frame that names the missing dependency — so an opaque "exit status 1" becomes the
+ * real cause. De-duplicates and caps output. Returns null when nothing diagnostic is present.
+ */
+export function extractCrashDiagnosis(logText: string): string | null {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const raw of logText.split('\n')) {
+    const line = raw.trim();
+    if (!line || !SIGNAL.test(line) || seen.has(line)) continue;
+    seen.add(line);
+    out.push(line);
+    if (out.length >= 6) break;
+  }
+  return out.length ? out.join('\n') : null;
+}
+
+/**
+ * Assemble the operator-facing report for an ops-handler crash: the raw message, the "it's the
+ * handler, not your command" framing, and either the real cause from CloudWatch or a where-to-look
+ * hint when the logs couldn't be read.
+ */
+export function formatOpsCrashReport(
+  baseMessage: string,
+  logGroup?: string,
+  diagnosis?: string | null,
+): string {
+  const lines = [
+    baseMessage,
+    '',
+    'The ops handler crashed at startup — usually a missing dependency in the deployed bundle, not a problem with this command.',
+  ];
+  if (diagnosis) {
+    lines.push(
+      '',
+      `From CloudWatch${logGroup ? ` (${logGroup})` : ''}:`,
+      diagnosis,
+      '',
+      'Fix the import/dependency and redeploy.',
+    );
+  } else {
+    lines.push('Check its CloudWatch logs for the failing top-level import, then redeploy.');
+  }
+  return lines.join('\n');
+}

package/src/cli/pipeline-loader.ts

@@ -0,0 +1,68 @@
+/**
+ * pipeline-loader — the CLI's window onto @everystack/server/pipeline.
+ *
+ * The CLI keeps NO compile-time dependency on @everystack/server (the two are
+ * mutual peers; hard-importing would be a project-reference cycle — the same
+ * reason plugin.ts inlines its types). So the orchestrator's types are mirrored
+ * structurally here and the runtime module is loaded lazily via a variable
+ * specifier. Every CLI verb that reaches the pipeline framework (pipeline:run,
+ * and the db:sync / db:plan visibility) goes through this one door.
+ */
+
+export type PipelineLane = 'rebuild' | 'curate' | 'all';
+
+export interface PipelineStage {
+  id: string;
+  lane: 'automated' | 'curated' | 'frozen';
+  dependsOn?: string[];
+  after?: string[];
+  heavy?: boolean;
+}
+export interface Pipeline {
+  stages: PipelineStage[];
+}
+export interface RunPipelineOptions {
+  runId: string;
+  lane?: PipelineLane;
+  only?: string;
+  from?: string;
+  to?: string;
+  resume?: boolean;
+  dryRun?: boolean;
+  continueOnError?: boolean;
+  actor?: string | null;
+  gitRef?: string | null;
+}
+export interface StageRunResult {
+  stageId: string;
+  outcome: 'applied' | 'skipped' | 'failed' | 'dry-run';
+  rows?: number;
+  error?: string;
+}
+export interface PipelineRunOutcome {
+  runId: string;
+  plan: string[];
+  results: StageRunResult[];
+  failed: boolean;
+}
+export interface PipelineRunSummary {
+  runId: string;
+  at: string | null;
+  applied: number;
+  failed: number;
+  failedStages: string[];
+}
+
+export interface ServerPipeline {
+  runPipeline(runner: unknown, pipeline: Pipeline, options: RunPipelineOptions): Promise<PipelineRunOutcome>;
+  latestRunId(query: (sql: string) => Promise<any[]>): Promise<string | null>;
+  readLastRun(query: (sql: string) => Promise<any[]>): Promise<PipelineRunSummary | null>;
+  topoSortStages(stages: PipelineStage[]): PipelineStage[];
+  selectStages(ordered: PipelineStage[], opts: Record<string, unknown>): PipelineStage[];
+}
+
+/** Lazily load the orchestrator from the @everystack/server peer (runtime only). */
+export async function loadServerPipeline(): Promise<ServerPipeline> {
+  const spec = '@everystack/server/pipeline';
+  return (await import(spec)) as ServerPipeline;
+}

package/src/cli/pipeline-path.ts

@@ -0,0 +1,25 @@
+/**
+ * pipeline-path — where the consumer's pipeline definition lives.
+ *
+ * Beside the schema homes (`db/models/`, `db/backfills/`), the
+ * pipeline is authored at `db/pipeline.ts` (or `db/pipeline/index.ts`),
+ * exporting a `pipeline` built with definePipeline. An explicit `--pipeline`
+ * always wins; when neither home exists the primary is returned so error
+ * messages point at the current convention.
+ */
+
+import { existsSync } from 'node:fs';
+
+/** Candidate pipeline entrypoints, in preference order. */
+export const PIPELINE_HOMES = ['db/pipeline.ts', 'db/pipeline/index.ts'] as const;
+
+export function resolvePipelinePath(
+  flag?: string,
+  exists: (p: string) => boolean = existsSync,
+): string {
+  if (flag) return flag;
+  for (const home of PIPELINE_HOMES) {
+    if (exists(home)) return home;
+  }
+  return PIPELINE_HOMES[0];
+}

package/src/cli/schema-compile.ts

@@ -19,12 +19,12 @@
  * Input is a `@everystack/model` ModelDescriptor (type-only — no runtime dep).
  */
 
-import type { ModelDescriptor, FieldSpec } from '@everystack/model';
-import type { TableSchema, EnumType, UniqueConstraint, CheckConstraint, IndexSchema, ForeignKey } from './schema-introspect.js';
-import type { RenameMap } from './schema-diff.js';
+import type { ModelDescriptor, FieldSpec, SequenceDescriptor } from '@everystack/model';
+import type { TableSchema, EnumType, SequenceSchema, UniqueConstraint, CheckConstraint, IndexSchema, ForeignKey } from './schema-introspect.js';
+import { nextvalSequence, type RenameMap } from './schema-diff.js';
 
 /** `authorId` -> `author_id`. SQL identifiers are snake_case. */
-function toSnakeCase(name: string): string {
+export function toSnakeCase(name: string): string {
   return name.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
 }
 
@@ -48,20 +48,25 @@ export function modelConstraintSpecs(model: ModelDescriptor): { uniques: UniqueC
     } else if (con.kind === 'check') {
       checks.push({ name: `${model.table}_check_${checkIndex++}`, expr: con.predicate });
     } else if (con.kind === 'index') {
-      const cols = con.columns.map(toSnakeCase);
+      // Plain entries are field keys → snake_cased; raw sql`` entries (expressions,
+      // DESC, opclass) pass verbatim — they are already SQL (Brick E).
+      const cols = con.columns.map((c, i) => (con.rawPositions?.[i] ? c : toSnakeCase(c)));
       // Disambiguate multiple indexes over the same columns — e.g. a full index plus a
       // partial one (`familyId` and `familyId WHERE status='active'`): the name must be
       // unique within the table for CREATE to succeed. The diff matches indexes by content
-      // (columns + uniqueness + predicate), never by name, so the suffix never affects the
-      // round-trip. Declaration order is deterministic, so the name is stable across calls.
-      let name = `${model.table}_${cols.join('_')}_index`;
-      for (let n = 2; usedIndexNames.has(name); n++) name = `${model.table}_${cols.join('_')}_index_${n}`;
+      // (entries + uniqueness + predicate + method + INCLUDE), never by name, so the suffix
+      // never affects the round-trip. Raw entries sanitize to identifier-safe name parts.
+      const nameParts = cols.map((c) => c.replace(/\W+/g, '_').replace(/^_+|_+$/g, ''));
+      let name = `${model.table}_${nameParts.join('_')}_index`;
+      for (let n = 2; usedIndexNames.has(name); n++) name = `${model.table}_${nameParts.join('_')}_index_${n}`;
       usedIndexNames.add(name);
       indexes.push({
         name,
         columns: cols,
         unique: con.isUnique,
         ...(con.predicate ? { where: con.predicate } : {}),
+        ...(con.method ? { using: con.method } : {}),
+        ...(con.includeColumns?.length ? { include: con.includeColumns.map(toSnakeCase) } : {}),
       });
     }
   }
@@ -209,6 +214,7 @@ function defaultExpr(spec: FieldSpec): string | null {
   switch (spec.defaultKind) {
     case 'now': return 'now()';
     case 'random': return 'gen_random_uuid()';
+    case 'sql': return String(spec.default); // .defaultSql() — the author's SQL, verbatim
     case 'value':
       return spec.isArray && Array.isArray(spec.default) ? arrayLiteral(spec.default) : literal(spec.default);
     default: return null;
@@ -251,7 +257,12 @@ export function fieldColumnSql(sqlName: string, spec: FieldSpec, inlinePrimaryKe
   let s = `"${sqlName}" ${serial ?? pgType(spec)}`;
   if (inlinePrimaryKey) s += ' PRIMARY KEY';
   const def = serial ? null : defaultExpr(spec);
-  if (def) s += ` DEFAULT ${def}`;
+  // A `defaultSql` nextval draws from an EXISTING sequence, possibly owned by a table
+  // created later in the same migration — held out of the column line (compileMigration
+  // re-attaches it as a trailing SET DEFAULT). Never the serial shorthand: that would
+  // mint a NEW sequence and silently detach the column from the one it declared.
+  const heldSequenceDraw = spec.defaultKind === 'sql' && nextvalSequence(def) != null;
+  if (def && !heldSequenceDraw) s += ` DEFAULT ${def}`;
   if (spec.isNotNull) s += ' NOT NULL';
   return s;
 }
@@ -485,6 +496,26 @@ export function compileEnums(models: ModelDescriptor[]): EnumType[] {
   return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name));
 }
 
+/**
+ * Standalone-sequence descriptors → the snapshot shape the diff and fingerprint consume.
+ * PostgreSQL's defaults (start/increment of 1) are omitted, so a declaration that omits
+ * them reaches the same canonical form a live default-configured sequence introspects to.
+ */
+export function compileSequences(sequences: readonly SequenceDescriptor[]): SequenceSchema[] {
+  return sequences
+    .map((s) => ({
+      name: s.name,
+      as: s.as,
+      ...(s.start != null && s.start !== 1 ? { start: s.start } : {}),
+      ...(s.increment != null && s.increment !== 1 ? { increment: s.increment } : {}),
+    }))
+    .sort((a, b) => a.name.localeCompare(b.name));
+}
+
+// The DDL renderer lives in schema-diff (emitOne needs it; this file already imports
+// schema-diff, so defining it there avoids a cycle) — re-exported here beside its compiler.
+export { sequenceCreateSql } from './schema-diff.js';
+
 /**
  * Foreign-key constraints for a model — its `.references()` fields and its table-level
  * `foreignKey(...)` constraints, via `modelForeignKeys` — as `ALTER TABLE` statements. These