@everystack/cli 0.4.49 → 0.4.50

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.49",
+  "version": "0.4.50",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/authz-derive.ts

@@ -64,8 +64,10 @@ const DML: Record<string, 'read' | 'create' | 'update' | 'delete'> = {
   DELETE: 'delete',
 };
 
-/** The roles the compiler itself emits policies for; anything else is app-specific. */
-const KNOWN_ROLES = new Set(['anon', 'authenticated', 'admin']);
+/** The roles the compiler itself emits policies for; anything else is app-specific.
+ *  Exported so the DERIVED renderer applies the identical rule — the views path once had
+ *  its own idea of which grantees to render, and the two disagreed inside one pull. */
+export const KNOWN_ROLES = new Set(['anon', 'authenticated', 'admin']);
 
 /**
  * The grantees a rendered model GOVERNS — the three vocabulary roles plus PUBLIC, which is
@@ -75,7 +77,7 @@ const KNOWN_ROLES = new Set(['anon', 'authenticated', 'admin']);
  * an ungoverned grantee alone, so there is no REVOKE to prevent, and naming the role in a model
  * would GOVERN it — turning a rendering decision into an access decision for every table.
  */
-const GOVERNED_VOCABULARY = new Set([...KNOWN_ROLES, 'PUBLIC', 'public']);
+export const GOVERNED_VOCABULARY = new Set([...KNOWN_ROLES, 'PUBLIC', 'public']);
 
 /** The beyond-CRUD privileges a governed role holds live — what `can()` cannot say. */
 function deriveExtraPrivileges(contract: TableContract): Record<string, string[]> {

package/src/cli/commands/db-pull.ts

@@ -2,7 +2,7 @@
  * `everystack db:pull` — generate `field()` Models from a live database (the brownfield on-ramp).
  *
  *   db:pull [--stage <name>] [--database-url <url>] [--schema public] [--out <dir | file.ts>]
- *           [--derived-out <file.ts>] [--abilities public-read]
+ *           [--derived-out <file.ts>] [--abilities live|public-read]
  *
  * `--stage` and `--database-url` COMPOSE: the URL picks the connection, the stage names the
  * baseline entry (`db/authz-baseline.json` is per-stage — a local adoption pulls with
@@ -358,7 +358,21 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
   const flagged = (source.match(/\/\/ (FIXME|TODO|composite|CHECK|FK →|verbatim:)/g) ?? []).length;
   if (flagged) caution(`${flagged} inline comment(s) flag things to review (verbatim types, checks, cross-schema FKs).`);
   if (abilities === 'commented') {
-    note(`Each model scaffolds its authz decision as comments — author them (db:check fails until every model declares), or stamp the common case: db:pull --abilities public-read.`);
+    // `live` FIRST, and named. It is the mode brownfield adoption needs — derive the authz
+    // from grants and policies the database already has — and it was advertised nowhere:
+    // not in the usage line, not in --help, not here. The only place an operator met the
+    // word was the error text for an unknown preset, so the discovery path for the right
+    // flag was to guess a wrong one. A consumer and their agent both walked past it in one
+    // session, and it cost a misdiagnosis plus a needless overwrite of 30 model files.
+    //
+    // `public-read` is deliberately no longer the headline: on an existing database it
+    // stamps can('read') on EVERY table, which on a real schema means public read of
+    // credit_cards, transactions, emails and users.
+    note(
+      `Each model scaffolds its authz decision as comments — db:check fails until every model declares. `
+      + `For an EXISTING database, re-pull with --abilities live to derive them from the grants and policies `
+      + `already there. --abilities public-read stamps public-read/admin-write on every table — greenfield only.`,
+    );
   } else {
     note(`Stamped '${abilities}' abilities into every model — review the generated stanzas; they are code, not defaults.`);
   }

package/src/cli/derived-lint.ts

@@ -32,6 +32,7 @@ import type {
   FunctionDescriptor, DependsOnRef,
 } from '@everystack/model';
 import { parseQualified } from './derived-source.js';
+import { GOVERNED_VOCABULARY } from './authz-derive.js';
 
 export interface DerivedAuthzGap {
   /** `schema.name` of the descriptor that fails the gate. */
@@ -60,6 +61,10 @@ function isModelRef(ref: DependsOnRef): ref is ModelDescriptor {
  * as zero-reach, which bricked every invoker view over a private table.
  */
 export function tableReaches(m: ModelDescriptor, role: string): boolean {
+  // A recorded grant is reach too. PUBLIC SELECT is held by every role; a named grantee
+  // reaches itself. See grantsPublic — this is the half the gate kept forgetting.
+  if (grantsPublic(m.privileges, 'SELECT')) return true;
+  if (privilegeRoles(m.privileges, 'SELECT').includes(role)) return true;
   for (const a of m.abilities) {
     if (isColumnAbility(a)) {
       // Only the READ half reaches: a column-scoped update compiles to `UPDATE (cols)`
@@ -79,6 +84,30 @@ export function tableReaches(m: ModelDescriptor, role: string): boolean {
   return false;
 }
 
+/**
+ * REACH comes from abilities AND from recorded privileges. Forgetting the second half is a
+ * bug this repo has now made three times — the 63-byte ACL truncation, the SECDEF caller
+ * gate, and this one — always the same shape: a fact recorded as `privileges` falls out of
+ * a downstream read and lands in neither set.
+ *
+ * `PUBLIC` is the case that matters. `GRANT … TO PUBLIC` is held by EVERY role, so a
+ * dependency carrying one is reachable by all of them, and a live PUBLIC grant is exactly
+ * what `db:pull` RECORDS rather than inventing an audience for. Reading abilities alone
+ * made a faithful pull fail to COMPILE, with no hand-edit available to fix it.
+ */
+function grantsPublic(privileges: Record<string, string[]> | undefined, privilege: string): boolean {
+  return Object.entries(privileges ?? {}).some(
+    ([grantee, privs]) => grantee.toUpperCase() === 'PUBLIC' && privs.some((p) => p.toUpperCase() === privilege),
+  );
+}
+
+/** Grantees a recorded privilege names directly — reach for that role, nobody else. */
+function privilegeRoles(privileges: Record<string, string[]> | undefined, privilege: string): string[] {
+  return Object.entries(privileges ?? {})
+    .filter(([, privs]) => privs.some((p) => p.toUpperCase() === privilege))
+    .map(([grantee]) => grantee);
+}
+
 /** The roles a relation's grants reach — bare read is anon + authenticated (the table precedent). */
 function relationRoles(d: ViewDescriptor | MaterializedViewDescriptor): Set<string> {
   const roles = new Set<string>();
@@ -89,12 +118,15 @@ function relationRoles(d: ViewDescriptor | MaterializedViewDescriptor): Set<stri
       roles.add('authenticated');
     }
   }
+  for (const r of privilegeRoles(d.privileges, 'SELECT')) roles.add(r);
   return roles;
 }
 
 function functionRoles(fn: FunctionDescriptor): Set<string> {
   // define-time already rejected bare can('execute') — every ability carries a role.
-  return new Set(fn.abilities.map((a) => a.condition.role!));
+  const roles = new Set(fn.abilities.map((a) => a.condition.role!));
+  for (const r of privilegeRoles(fn.privileges, 'EXECUTE')) roles.add(r);
+  return roles;
 }
 
 // ---------------------------------------------------------------------------
@@ -262,7 +294,7 @@ function findUnreachable(role: string, refs: readonly DependsOnRef[], seen: Set<
     }
     switch (ref.kind) {
       case 'view': {
-        if (ref.private || !relationRoles(ref).has(role)) return identityOf(ref);
+        if (ref.private || (!relationRoles(ref).has(role) && !grantsPublic(ref.privileges, 'SELECT'))) return identityOf(ref);
         // An invoker dep re-checks the caller one level down; a definer dep reads as its owner.
         if (ref.securityInvoker) {
           const deeper = findUnreachable(role, ref.dependsOn, seen);
@@ -272,11 +304,11 @@ function findUnreachable(role: string, refs: readonly DependsOnRef[], seen: Set<
       }
       case 'materialized view':
         // A matview is its own snapshot: SELECT on the matview is the whole requirement.
-        if (ref.private || !relationRoles(ref).has(role)) return identityOf(ref);
+        if (ref.private || (!relationRoles(ref).has(role) && !grantsPublic(ref.privileges, 'SELECT'))) return identityOf(ref);
         break;
       case 'function':
         // The body calls it with the caller's rights — EXECUTE is part of reach.
-        if (!functionRoles(ref).has(role)) return identityOf(ref);
+        if (!functionRoles(ref).has(role) && !grantsPublic(ref.privileges, 'EXECUTE')) return identityOf(ref);
         break;
       case 'sql':
         break; // opaque — never a false positive
@@ -296,7 +328,12 @@ export function findInvokerReachabilityGaps(derived: readonly DerivedDescriptor[
   for (const d of derived) {
     if (d.kind !== 'view' || !d.securityInvoker || d.private || d.abilities.length === 0) continue;
     const identity = identityOf(d);
-    for (const role of [...relationRoles(d)].sort()) {
+    // Only roles the declared world can SATISFY. A grantee outside the compiler's
+    // vocabulary is recorded fact on the view AND on the table (the tables path never
+    // renders one as an ability), so table-side reach for it is not expressible — checking
+    // it is a demand no model can ever meet. Their reach is adopted via authz-baseline,
+    // outside this gate. PUBLIC is governed and stays checked.
+    for (const role of [...relationRoles(d)].filter((r) => GOVERNED_VOCABULARY.has(r)).sort()) {
       const unreachable = findUnreachable(role, d.dependsOn, new Set());
       if (unreachable) {
         gaps.push({

package/src/cli/derived-render.ts

@@ -18,6 +18,7 @@ import type { ColumnSchema, SequenceSchema, TableSchema } from './schema-introsp
 import { parseIndexDefinition } from './schema-introspect.js';
 import { modelFileName, renderFieldLines } from './model-render.js';
 import { splitFunctionIdentity } from './pg-argtypes.js';
+import { GOVERNED_VOCABULARY, KNOWN_ROLES } from './authz-derive.js';
 
 export interface DerivedRenderResult {
   /** The source block: `export const … = defineView(…)` etc., dependency-ordered. */
@@ -100,12 +101,28 @@ function tsString(s: string): string {
  */
 function splitRelationGrants(
   grants: Record<string, string[]>,
-): { abilities: string[]; privileges: Record<string, string[]> } {
+): { abilities: string[]; privileges: Record<string, string[]>; foreign: string[] } {
   const readRoles = new Set<string>();
   const privileges: Record<string, string[]> = {};
+  const foreign: string[] = [];
   for (const [grantee, privs] of Object.entries(grants)) {
     const selectOnly = privs.length === 1 && privs[0] === 'SELECT';
-    if (selectOnly && grantee.toUpperCase() !== 'PUBLIC') readRoles.add(grantee);
+    // ABILITIES are limited to the compiler's own vocabulary (authz-derive's KNOWN_ROLES).
+    // A foreign grantee is FACT, not opinion, so it is RECORDED — never declared as an
+    // intended audience. The views path used to render any select-only grantee as an
+    // ability, so one pull could declare a role as an ABILITY on a view while the table
+    // recorded it as a foreign grantee, and the reachability gate then demanded table-side
+    // reach the tables path will never render. The contract contradicted itself.
+    //
+    // RECORDED, not dropped — and that is the part the obvious fix gets wrong. On a TABLE,
+    // "not rendered" is safe because the table reconciler leaves an ungoverned grantee
+    // alone. On a DERIVED object there is no such carve-out: diffObjectGrants unions the
+    // declared and live grantees, so a grantee we omit is a grantee we REVOKE. Measured:
+    // declared {anon,authenticated} against live +outbound_migrator plans
+    // `REVOKE SELECT, UPDATE ON public.v FROM outbound_migrator`. Dropping them would
+    // reintroduce the silent revoke this whole branch exists to prevent.
+    if (!GOVERNED_VOCABULARY.has(grantee)) foreign.push(grantee);
+    if (selectOnly && KNOWN_ROLES.has(grantee)) readRoles.add(grantee);
     else privileges[grantee] = [...privs].sort();
   }
   const abilities: string[] = [];
@@ -115,7 +132,7 @@ function splitRelationGrants(
     readRoles.delete('authenticated');
   }
   for (const role of [...readRoles].sort()) abilities.push(`can('read', { role: ${tsString(role)} })`);
-  return { abilities, privileges };
+  return { abilities, privileges, foreign: foreign.sort() };
 }
 
 /**
@@ -363,7 +380,15 @@ export function renderDerivedSource(catalog: DerivedCatalog, opts: DerivedRender
       // the abilities we can express would leave the write grants undeclared, so the next
       // generate would plan a REVOKE for each — a silent skip traded for a silent revoke.
       // Declared == live means nothing is planned at all.
-      const { abilities, privileges } = splitRelationGrants(o.grants ?? {});
+      const { abilities, privileges, foreign } = splitRelationGrants(o.grants ?? {});
+      if (foreign.length) {
+        warnings.push(
+          `${o.identity}: grants exist for ${foreign.join(', ')} — recorded as \`privileges\`, not declared as ` +
+          'abilities. Abilities name the roles the model governs (anon/authenticated/admin); a grantee outside ' +
+          'that vocabulary is fact, not an intended audience. Recorded rather than dropped because a derived ' +
+          'object REVOKES any live grantee the declaration omits.',
+        )
+      }
       const writeGrants = Object.values(privileges).some((ps) =>
         ps.some((p) => p === 'INSERT' || p === 'UPDATE' || p === 'DELETE'),
       );

package/src/cli/index.ts

@@ -359,7 +359,7 @@ Usage:
   everystack db:export         --schema <name> [--stage <name> | --database-url <url> [--out <file.dump>]] [--models <barrel>]  Schema-scoped pg_dump artifact, stamped with the DECLARED schema fingerprint (the canonical-sync export; db:swap gates on that stamp). --stage dumps the stage's private DB via the ops Lambda → S3; --database-url (explicit flag, never the env) dumps a reachable DB to a local .dump + .meta.json — the build-locally → publish → swap on-ramp
   everystack db:swap           --schema <name> --from <artifact.dump | artifact-id> [--stage <name> --direct | --database-url <url>] --confirm [--fingerprint <hash>] [--snapshot physical|logical|none] [--snapshot-ref <id>] [--rebuild-derived] [--dump-build <file.json>]  Land a schema artifact atomically: fingerprint gate → pre-flight refusals → CONFIRMED snapshot → restore into <schema>_incoming (COPY-safe rewrite) → build the paired derived layer → one txn (drop+rename+recreate app→schema FKs, re-apply authz + schema USAGE) → assertions → drop retiring. Refresh-free; app.* untouched; the derived layer is never absent. DESTRUCTIVE. The venue is EXPLICIT — DATABASE_URL in the env is refused, and --stage requires --direct (a multi-GB restore exceeds the ops-Lambda 900s clock). The rollback point defaults to a PHYSICAL RDS snapshot on a stage that exposes databaseInstanceId (no locks, no pg_dump contending with the restore) and a WAITED logical db:backup otherwise; a bare --database-url refuses without --snapshot-ref <id> or --snapshot none. docs/schema-swap.md
   everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run]  Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
-  everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review; add --suggest-keys to probe the LIVE rows for functionally-unique columns (one scan per matview) and surface each as a commented .primaryKey() suggestion. docs/derived-objects.md#flipping-a-matview-to-a-materialized-table---matviews-as-tables
+  everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities live|public-read]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored). **--abilities live is the brownfield mode**: derive each model's authz from the grants and policies the database ALREADY has, and write the foreign-grantee baseline (db/authz-baseline.json) — this is what you want when adopting an existing schema. --abilities public-read stamps the common stanza (public read, admin write) uncommented — greenfield only, since on an existing database it declares public read of every table. Both are explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review; add --suggest-keys to probe the LIVE rows for functionally-unique columns (one scan per matview) and surface each as a commented .primaryKey() suggestion. docs/derived-objects.md#flipping-a-matview-to-a-materialized-table---matviews-as-tables
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
   everystack db:fingerprint    [--stage <name> | --database-url <url>] [--models <barrel>] [--json]  Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
   everystack db:reconcile      [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--only a,b] [--json]  Reconcile the derived layer (functions/views/matviews/triggers) against the DECLARED descriptors (defineView/defineMaterializedView/defineFunction/defineSql/trigger() on models, from the barrel) — the single home (db/sql is retired; leftover .sql files fail with the migration path): plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (atomic — DDL + provenance in one transaction) and records provenance + schema_log; --apply --stage runs credential-free in the ops Lambda (no admin URL on the deployer, the db:apply twin), --apply --database-url runs direct. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive. --only <schema.name,…> restricts the run to the named objects (surgical); with --rebuild it FORCES those to rebuild from source even when the hashes show no diff — the recovery exit when a mistaken --rebaseline left a self-consistent-but-wrong provenance row (the dependency cascade rebuilds their live dependents).

package/src/plugin.ts

@@ -9,17 +9,35 @@
 
 import type { StorageAdapter } from './storage/index';
 
-/** Minimal plugin context (compatible with @everystack/server/plugin PluginContext) */
+/**
+ * These types MIRROR @everystack/server/plugin. They are not imported from it, and that
+ * is deliberate: server imports `@everystack/cli/apply`, `/reconcile`, `/exec` and more,
+ * so a type import in the other direction closes a package cycle. Both packages ship
+ * TypeScript source, so a type-only import still drags the whole module graph in.
+ *
+ * The copy going stale is what broke a consumer: server made `publishJob` optional and
+ * gave it an options argument, nothing here compared the two declarations, and `Plugin`
+ * is contravariant in `ctx` — so cli's `Plugin` silently stopped being assignable to
+ * server's and the TS2322 landed in their build instead of ours.
+ *
+ * **The link is `__tests__/cli/plugin-type-compat.test.ts`**, which imports BOTH and
+ * asserts assignability at compile time. The test can cross the boundary because it is
+ * not part of either package's published graph. Drift now fails there, in this repo, on
+ * the commit that causes it. If you change a shape below, that test is the gate.
+ */
+
+/** Mirrors @everystack/server/plugin PluginContext. `publishJob` is OPTIONAL (a lean app
+ *  with no @everystack/jobs never sets it) and takes an options argument. */
 interface PluginContext {
   db: any;
   schema: Record<string, any>;
   verifyToken: (token: string) => Promise<Record<string, unknown> | null>;
   environment: string;
-  publishJob: (type: string, payload: unknown) => Promise<string>;
+  publishJob?: (type: string, payload: unknown, options?: { schedulable?: boolean; runAt?: Date }) => Promise<string>;
   [key: string]: unknown;
 }
 
-/** Minimal route type (compatible with @everystack/server Route) */
+/** Mirrors @everystack/server Route. */
 interface Route {
   path: string;
   method?: string;
@@ -27,10 +45,10 @@ interface Route {
   handler: (req: Request) => Promise<Response>;
 }
 
-/** Action handler type (compatible with @everystack/server/plugin ActionHandler) */
+/** Mirrors @everystack/server/plugin ActionHandler. */
 type ActionHandler = (payload: unknown, ctx: PluginContext) => Promise<unknown>;
 
-/** Plugin factory function */
+/** Mirrors @everystack/server/plugin Plugin. */
 type Plugin = (ctx: PluginContext) => Promise<{
   routes?: Route[];
   actions?: Record<string, ActionHandler>;