@proteinjs/db 1.30.0 → 1.32.0

package/index.ts

@@ -10,6 +10,7 @@ export * from './src/reference/Reference';
 export * from './src/UpdatePreserving';
 export * from './src/reference/ReferenceCache';
 export * from './src/RecordIterator';
+export * from './src/CursorWindowPager';
 export * from './src/source/SourceRecord';
 export * from './src/source/SourceRecordRepo';
 export * from './src/MigrationRunner';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.30.0",
+  "version": "1.32.0",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -41,7 +41,7 @@
     "test": "jest --passWithNoTests"
   },
   "dependencies": {
-    "@proteinjs/db-query": "^1.6.3",
+    "@proteinjs/db-query": "^1.7.0",
     "@proteinjs/logger": "^1.0.21",
     "@proteinjs/reflection": "^1.1.12",
     "@proteinjs/serializer": "^1.1.10",
@@ -66,5 +66,5 @@
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
   },
-  "gitHead": "fd3dd48fae93c6da8c0b3decd6dd30a506a9109a"
+  "gitHead": "edfed2d7826c7e42ddc0ae84bf97cda11614c1f0"
 }

package/src/CursorWindowPager.ts

@@ -0,0 +1,153 @@
+import { Serializer } from '@proteinjs/serializer';
+import { Condition, LogicalGroup, QueryBuilder, SortCriteria } from '@proteinjs/db-query';
+import { Db, getDb } from './Db';
+import { QueryOptions } from './services/DbService';
+import { Table } from './Table';
+import { Record } from './Record';
+
+/**
+ * A cursor anchor names the last row a window served: the row's value for each effective sort
+ * criterion, in criteria order. The final criterion is always the unique `id` axis, so an
+ * anchor always identifies exactly one row — ties on the other axes can never blur where the
+ * next window starts.
+ */
+export type CursorAnchor = {
+  values: unknown[];
+};
+
+export type AnchoredWindow<T> = {
+  rows: T[];
+  /** Anchor for the window after this one; null = the data set is exhausted. */
+  nextAnchor: CursorAnchor | null;
+};
+
+/**
+ * THE owner of cursor-window paging over a table query — every windowed consumer (the
+ * server-side `RecordIterator`, the UI's `QueryCursorLoader`) frames its windows here.
+ *
+ * Each window builds a FRESH query (builders are mutable) carrying a lexicographic
+ * continuation past the anchor row, the effective sort, and `paginate(0, windowSize)` — so
+ * rows inserted or deleted while paging can never shift the window frame, the offset-paging
+ * drift class (`OFFSET n` frames windows by position; concurrent writes slide unvisited rows
+ * into the consumed range or consumed rows back into the next window).
+ *
+ * The effective sort is the consumer's criteria with a unique `id` tiebreak appended (an
+ * explicit `id` criterion ends the list — axes after it can never influence order). Ties on
+ * consumer axes therefore continue across window boundaries instead of being skipped by a
+ * bare `field < cursor`. Sort axes should be non-null columns: a window whose tail carries a
+ * NULL sort value ends paging (a null can't anchor a lexicographic continuation — under a
+ * descending sort, null rows collect at the tail). `byValues` sorts cannot be paged by cursor
+ * and are rejected.
+ *
+ * Anchors are plain in-memory values; `encodeAnchor`/`decodeAnchor` (house `Serializer`, so
+ * dates/moments survive) turn them into strings for consumers whose cursors must serialize —
+ * e.g. react-query page params.
+ */
+export class CursorWindowPager<T extends Record> {
+  private effectiveSort: SortCriteria<T>[];
+
+  /**
+   * @param table the table to load windows from
+   * @param createQuery factory producing a FRESH conditions-only QueryBuilder per call —
+   *   every window builds its own to hang the cursor conditions, sort, and pagination on
+   * @param sort the consumer's sort criteria; the effective sort appends the `id` tiebreak.
+   *   Omitted/empty = pure `id` order (the complete-iteration default).
+   * @param options `db` to ride a specific Db (scoped/system/transactional); defaults to
+   *   `getDb()` per window. `queryOptions` pass through to every window query.
+   */
+  constructor(
+    private table: Table<T>,
+    private createQuery: () => QueryBuilder<T>,
+    sort?: SortCriteria<T>[],
+    private options?: { db?: Db; queryOptions?: QueryOptions<T> }
+  ) {
+    this.effectiveSort = this.normalizeSort(sort ?? []);
+  }
+
+  /** Load one window: the first when `anchor` is null, otherwise the rows past the anchor. */
+  async loadWindow(anchor: CursorAnchor | null, windowSize: number): Promise<AnchoredWindow<T>> {
+    if (windowSize < 1) {
+      throw new Error(`Cursor window size must be at least 1 (got ${windowSize})`);
+    }
+
+    const qb = this.buildWindowQuery(anchor, windowSize);
+    const db = this.options?.db ?? getDb();
+    const rows = await db.query(this.table, qb, this.options?.queryOptions);
+    return { rows, nextAnchor: this.deriveNextAnchor(rows, windowSize) };
+  }
+
+  static encodeAnchor(anchor: CursorAnchor): string {
+    return Serializer.serialize(anchor);
+  }
+
+  static decodeAnchor(encoded: string): CursorAnchor {
+    return Serializer.deserialize(encoded);
+  }
+
+  private buildWindowQuery(anchor: CursorAnchor | null, windowSize: number): QueryBuilder<T> {
+    const qb = this.createQuery();
+    if (anchor) {
+      qb.or(this.anchorConditions(anchor));
+    }
+    return qb.sort(this.effectiveSort).paginate({ start: 0, end: windowSize });
+  }
+
+  /**
+   * Lexicographic continuation past the anchor row: one disjunct per sort criterion — equal on
+   * every earlier criterion, strictly past on this one (`<` descending, `>` ascending). The
+   * final `id` disjunct is what walks through ties on the consumer's axes.
+   */
+  private anchorConditions(anchor: CursorAnchor): Array<Condition<T> | LogicalGroup<T>> {
+    return this.effectiveSort.map((criterion, index) => {
+      const past: Condition<T> = {
+        field: criterion.field,
+        operator: criterion.desc ? '<' : '>',
+        value: anchor.values[index] as T[keyof T],
+      };
+      if (index === 0) {
+        return past;
+      }
+      const equals: Condition<T>[] = this.effectiveSort.slice(0, index).map((earlier, i) => ({
+        field: earlier.field,
+        operator: '=',
+        value: anchor.values[i] as T[keyof T],
+      }));
+      return { operator: 'AND', children: [...equals, past] };
+    });
+  }
+
+  private deriveNextAnchor(rows: T[], windowSize: number): CursorAnchor | null {
+    // A short window means the data set is exhausted; a full one implies more may exist —
+    // the next (short or empty) window settles it.
+    if (rows.length < windowSize) {
+      return null;
+    }
+
+    const tail = rows[rows.length - 1] as unknown as { [field: string]: unknown };
+    const values = this.effectiveSort.map((criterion) => tail[criterion.field as string]);
+    // A null sort value can't anchor a lexicographic continuation; under a descending sort
+    // null rows collect at the tail, so the data set's sortable rows are exhausted here.
+    if (values.some((value) => value == null)) {
+      return null;
+    }
+
+    return { values };
+  }
+
+  private normalizeSort(sort: SortCriteria<T>[]): SortCriteria<T>[] {
+    for (const criterion of sort) {
+      if (criterion.byValues && criterion.byValues.length > 0) {
+        throw new Error(
+          `Cursor windows cannot page a byValues sort (field '${String(criterion.field)}'): a CASE ordering has no comparable cursor axis`
+        );
+      }
+    }
+
+    const idIndex = sort.findIndex((criterion) => criterion.field === 'id');
+    if (idIndex >= 0) {
+      return sort.slice(0, idIndex + 1);
+    }
+
+    return [...sort, { field: 'id' as keyof T, desc: false }];
+  }
+}

package/src/Db.ts

@@ -59,6 +59,14 @@ export interface DefaultDbDriverFactory extends Loadable {
 export interface DbDriver {
   getDbName(): string;
   createDbIfNotExists(): Promise<void>;
+  /**
+   * Create the named database. When `ddl` is provided the statements are applied as part of
+   * creation (on Spanner: `CreateDatabase.extra_statements` — one operation, atomic with the
+   * create), so the database is born with its full schema. Fails if the database already exists.
+   */
+  createDb(name: string, options?: { ddl?: string[] }): Promise<void>;
+  /** Drop the named database. */
+  dropDb(name: string): Promise<void>;
   start?(): Promise<void>;
   stop?(): Promise<void>;
   getTableManager(): TableManager;
@@ -517,6 +525,16 @@ export class Db<R extends Record = Record> implements DbService<R> {
     }
   }
 
+  /**
+   * Run a query. Column queries (scope guards etc.) are always applied.
+   *
+   * PAGING: this method is also the cursor-window surface — `CursorWindowPager` composes each
+   * window as a fresh QueryBuilder (cursor conditions + sort + `paginate(0, windowSize)`)
+   * through here, so every window rides the caller's driver, ambient transaction, and column
+   * queries. Iterating consumers go through it (`RecordIterator` server-side,
+   * `QueryCursorLoader` in the UI) instead of positional `paginate(start, end)` offsets,
+   * which drift under concurrent writes (rows slide across window frames).
+   */
   async query<T extends R>(table: Table<T>, query: Query<T>, options?: QueryOptions<T>): Promise<T[]> {
     const qb = new QueryBuilderFactory().getQueryBuilder(table, query);
 

package/src/MigrationRunner.ts

@@ -4,12 +4,33 @@ import { Table } from './Table';
 import { SourceRecordRepo } from './source/SourceRecordRepo';
 import { MigrationRunnerService, getMigrationRunnerService } from './services/MigrationRunnerService';
 import { Migration, MigrationTable } from './tables/MigrationTable';
+import { QueryBuilderFactory } from './QueryBuilderFactory';
 import { Service } from '@proteinjs/service';
 import { Logger } from '@proteinjs/logger';
 
 export const getMigrationRunner = () =>
   typeof self === 'undefined' ? new MigrationRunner() : (getMigrationRunnerService() as MigrationRunner);
 
+/**
+ * What one deploy-gated series run did (plans/POST_RELEASE_QUEUE.md 27f). Ids appear in ledger
+ * order (oldest-first). The caller's gate is `failed`: present → the series stopped there, the
+ * deploy Job must exit non-zero, the rollout must not advance.
+ */
+export interface MigrationSeriesSummary {
+  /** Ran to success in this series. */
+  applied: string[];
+  /** Excluded by the explicit `manual` flag (they keep the Migrations-page flow). */
+  skippedManual: string[];
+  /** Already in 'success' status — skipped (ensureMigrationRun's idempotence). */
+  alreadyApplied: string[];
+  /** Ledger rows with no source record (a loader deleted after its run; the table keeps history). */
+  unresolved: string[];
+  /** The failure that stopped the series, if any. */
+  failed?: { id: string; description: string; failureMessage?: string };
+  /** Ordered after the failure — never started (later migrations may build on earlier ones). */
+  notAttempted: string[];
+}
+
 export class MigrationRunner implements MigrationRunnerService {
   private logger = new Logger({ name: this.constructor.name });
   public serviceMetadata: Service['serviceMetadata'] = {
@@ -44,18 +65,109 @@ export class MigrationRunner implements MigrationRunnerService {
    *
    * 'ensure' = idempotent: a row already in 'success' status is logged and skipped. A prior
    * 'failure' row is retried by design — a fixed migration should run on the next boot.
+   *
+   * Returns the migration with its final run state (the skipped row, or the run's outcome) —
+   * the series runner ({@link runPendingMigrations}) gates on it.
    */
-  async ensureMigrationRun(id: string): Promise<void> {
+  async ensureMigrationRun(id: string): Promise<Migration> {
     const migrationTable: Table<Migration> = new MigrationTable();
     const db = getDbAsSystem();
     const migrationRow = await db.get(migrationTable, { id });
     if (migrationRow?.status === 'success') {
       this.logger.info({ message: `Migration (${id}) already applied, skipping` });
-      return;
+      return migrationRow;
     }
 
     const migration = this.resolveMigration(migrationTable, id);
     await this.runAndRecord(migrationTable, migration, () => db);
+    return migration;
+  }
+
+  /**
+   * Deploy-path API (plans/POST_RELEASE_QUEUE.md 27f): the deploy pipeline's migration Job calls
+   * this AFTER `new Db().init()` (schema sync + source-record sync — every source-declared
+   * migration has a ledger row by then) and BEFORE the rollout advances. Pod boot never calls it:
+   * boot stays migration-free (the startupProbe invariant — DbInitStartupTask is schema sync and
+   * source records only).
+   *
+   * Discovery + order: the migration LEDGER (the migration table) is the authority. Rows run in
+   * SERIES, oldest-first by the row's `created` (id tiebreak) — a migration that shipped in an
+   * earlier release always runs before a later one. Never parallel.
+   *
+   * Policy:
+   * - `manual: true` (the source record's declaration) is EXCLUDED — the explicit non-automatable
+   *   class keeps the Migrations-page flow ({@link Migration.manual}).
+   * - rows already in 'success' are skipped; any other status is pending — including 'running'
+   *   (a crashed earlier Job): {@link ensureMigrationRun} re-runs it.
+   * - rows with no source record are history (a loader deleted after its run) — skipped,
+   *   reported as `unresolved`.
+   * - the FIRST failure STOPS the series (later migrations may build on earlier ones). The
+   *   caller exits non-zero, the Job fails, the rollout does not advance.
+   *
+   * EXPAND-CONTRACT INVARIANT (documented at this seam on purpose): every automated migration —
+   * and the schema sync that fronts it — must be backward-compatible with the STILL-RUNNING old
+   * release, because the old pods keep serving while this runs and keep serving indefinitely if
+   * it fails. "Roll back" = do not advance the code; DDL and applied migrations are never
+   * un-applied (Spanner DDL is not transactional-reversible). Contractions (drops, rewrites that
+   * break the old reader) belong in the `manual` class, run only after every consumer of the old
+   * shape is gone.
+   */
+  async runPendingMigrations(): Promise<MigrationSeriesSummary> {
+    const migrationTable: Table<Migration> = new MigrationTable();
+    const db = getDbAsSystem();
+    const qb = new QueryBuilderFactory().createQueryBuilder(migrationTable).sort([
+      { field: 'created', desc: false },
+      { field: 'id', desc: false },
+    ]);
+    const ledger = await db.query(migrationTable, qb);
+
+    const summary: MigrationSeriesSummary = {
+      applied: [],
+      skippedManual: [],
+      alreadyApplied: [],
+      unresolved: [],
+      notAttempted: [],
+    };
+    const sourceRecordRepo = new SourceRecordRepo();
+    const pending: Migration[] = [];
+    for (const row of ledger) {
+      const source = sourceRecordRepo.getSourceRecord<Migration>(migrationTable.name, row.id);
+      if (!source) {
+        summary.unresolved.push(row.id);
+        continue;
+      }
+      if (source.manual) {
+        summary.skippedManual.push(row.id);
+        continue;
+      }
+      if (row.status === 'success') {
+        summary.alreadyApplied.push(row.id);
+        continue;
+      }
+      pending.push(row);
+    }
+
+    this.logger.info({
+      message: `Running ${pending.length} pending migration${pending.length === 1 ? '' : 's'} in series, oldest-first`,
+      obj: { pending: pending.map((row) => row.id), skippedManual: summary.skippedManual },
+    });
+    for (let i = 0; i < pending.length; i++) {
+      const outcome = await this.ensureMigrationRun(pending[i].id);
+      if (outcome.status === 'success') {
+        summary.applied.push(outcome.id);
+        continue;
+      }
+      summary.failed = {
+        id: outcome.id,
+        description: outcome.description,
+        failureMessage: outcome.failureMessage,
+      };
+      summary.notAttempted = pending.slice(i + 1).map((row) => row.id);
+      break;
+    }
+
+    this.logger.info({ message: `Migration series finished`, obj: summary as any });
+    return summary;
   }
 
   // The db is taken as a provider, resolved inside this async body: on the service path,

package/src/RecordIterator.ts

@@ -1,44 +1,54 @@
-import { Db, getDb } from './Db';
+import { Db } from './Db';
 import { Table } from './Table';
 import { Record } from './Record';
 import { Query } from './services/DbService';
 import { QueryBuilderFactory } from './QueryBuilderFactory';
 import { QueryBuilder } from '@proteinjs/db-query';
+import { CursorAnchor, CursorWindowPager } from './CursorWindowPager';
 
+/**
+ * Iterate a table query in cursor-anchored windows (`CursorWindowPager`), `windowSize` rows
+ * per query. Cursor windows frame on the last row served instead of a positional offset, so
+ * rows inserted or deleted while iterating can never skip or duplicate rows — every row that
+ * exists throughout the iteration is yielded exactly once.
+ *
+ * Ordering: the sort criteria on the consumer's query builder become the cursor axes, with a
+ * unique `id` tiebreak appended (ties can't skip across window boundaries); with no sort the
+ * iteration runs in plain `id` order. Sort axes should be non-null columns — a window whose
+ * tail carries a NULL sort value ends the iteration (a null can't anchor a continuation).
+ *
+ * The caller's query builder is never mutated: every window builds on a fresh copy.
+ */
 export class RecordIterator<T extends Record> implements AsyncIterable<T> {
-  private db: Db;
-  private table: Table<T>;
-  private query: QueryBuilder<T>;
-  private pageSize: number;
-  private currentPage: number;
+  private pager: CursorWindowPager<T>;
 
-  constructor(table: Table<T>, query: Query<T>, pageSize: number = 10, db?: Db) {
-    this.db = db || getDb();
-    this.table = table;
-    this.query = new QueryBuilderFactory().getQueryBuilder(table, query);
-    this.pageSize = pageSize;
-    this.currentPage = 0;
+  constructor(
+    table: Table<T>,
+    query: Query<T>,
+    private windowSize: number = 10,
+    db?: Db
+  ) {
+    // A fresh copy of the consumer's query; its sort criteria lift off the base (the pager
+    // owns ordering — cursor axes and ORDER BY have to agree) and the conditions-only base
+    // seeds every window's fresh build.
+    const base = new QueryBuilderFactory().createQueryBuilder(table, query);
+    const sort = base.removeSortCriteria();
+    this.pager = new CursorWindowPager<T>(table, () => QueryBuilder.fromQueryBuilder(base, table.name), sort, { db });
   }
 
   async *[Symbol.asyncIterator](): AsyncIterator<T> {
+    let anchor: CursorAnchor | null = null;
     while (true) {
-      const pagination = { start: this.currentPage * this.pageSize, end: (this.currentPage + 1) * this.pageSize };
-      this.query.paginate(pagination);
-      const results = await this.db.query(this.table, this.query);
-
-      if (results.length === 0) {
-        return;
-      }
-
-      for (const item of results) {
-        yield item;
+      const window = await this.pager.loadWindow(anchor, this.windowSize);
+      for (const row of window.rows) {
+        yield row;
       }
 
-      if (results.length < this.pageSize) {
+      if (!window.nextAnchor) {
         return;
       }
 
-      this.currentPage++;
+      anchor = window.nextAnchor;
     }
   }
 }

package/src/schema/SchemaOperations.ts

@@ -1,7 +1,13 @@
 import { Table } from '../Table';
 
 export interface SchemaOperations {
-  createTable(table: Table<any>): Promise<void>;
+  /**
+   * Create every table in `tables`, in the given order. The order is load-bearing: a table whose
+   * foreign keys reference another absent table must appear after it. Drivers that support batched
+   * DDL (Spanner) apply the whole set as ONE schema-update operation — statements inside a batch
+   * apply in order, so the ordering contract is preserved.
+   */
+  createTables(tables: Table<any>[]): Promise<void>;
   alterTable(table: Table<any>, changes: TableChanges): Promise<void>;
 }
 

package/src/schema/TableManager.ts

@@ -33,10 +33,34 @@ export class TableManager {
     return await this.schemaMetadata.tableExists(table);
   }
 
+  /**
+   * Reconcile every registered table with the live schema. Absent tables are created as ONE
+   * batch (a single schema-update operation on drivers that support it — the prod-boot win),
+   * preserving their registration order so foreign keys to other absent tables resolve; existing
+   * tables are altered individually after, so an alter that adds a foreign key to a
+   * just-created table sees it live.
+   */
   async loadTables(): Promise<void> {
     const tables = getTables();
+    const absentTables: Table<any>[] = [];
+    const existingTables: Table<any>[] = [];
     for (const table of tables) {
-      await this.loadTable(table);
+      this.validateDynamicReferenceColumns(table);
+      if (await this.tableExists(table)) {
+        existingTables.push(table);
+      } else {
+        absentTables.push(table);
+      }
+    }
+
+    if (absentTables.length > 0) {
+      this.logger.info({ message: `Creating tables: ${absentTables.map((table) => table.name).join(', ')}` });
+      await this.schemaOperations.createTables(absentTables);
+      this.logger.info({ message: `Finished creating ${absentTables.length} tables` });
+    }
+
+    for (const table of existingTables) {
+      await this.alterTableIfChanged(table);
     }
   }
 
@@ -44,19 +68,23 @@ export class TableManager {
     this.validateDynamicReferenceColumns(table);
 
     if (await this.tableExists(table)) {
-      const tableChanges = await this.getTableChanges(table);
-      if (this.shouldAlterTable(tableChanges)) {
-        this.logger.info({ message: `Altering table: ${table.name}` });
-        await this.schemaOperations.alterTable(table, tableChanges);
-        this.logger.info({ message: `Finished altering table: ${table.name}` });
-      }
+      await this.alterTableIfChanged(table);
     } else {
       this.logger.info({ message: `Creating table: ${table.name}` });
-      await this.schemaOperations.createTable(table);
+      await this.schemaOperations.createTables([table]);
       this.logger.info({ message: `Finished creating table: ${table.name}` });
     }
   }
 
+  private async alterTableIfChanged(table: Table<any>): Promise<void> {
+    const tableChanges = await this.getTableChanges(table);
+    if (this.shouldAlterTable(tableChanges)) {
+      this.logger.info({ message: `Altering table: ${table.name}` });
+      await this.schemaOperations.alterTable(table, tableChanges);
+      this.logger.info({ message: `Finished altering table: ${table.name}` });
+    }
+  }
+
   private validateDynamicReferenceColumns(table: Table<any>): void {
     const isDynamicRefColumn = (column: any): column is DynamicReferenceColumn<any> =>
       typeof column.dynamicRefTableColName === 'string';

package/src/tables/MigrationTable.ts

@@ -1,10 +1,19 @@
 import { Moment } from '../opt/moment';
-import { DateTimeColumn, ObjectColumn, StringColumn } from '../Columns';
+import { BooleanColumn, DateTimeColumn, ObjectColumn, StringColumn } from '../Columns';
 import { Table } from '../Table';
 import { SourceRecord, withSourceRecordColumns } from '../source/SourceRecord';
 
 export interface Migration extends SourceRecord {
   description: string;
+  /**
+   * The explicit non-automatable class (plans/POST_RELEASE_QUEUE.md 27f): destructive
+   * contractions (column/table drops), long backfills over big tables, data-dependent one-offs.
+   * `manual: true` EXCLUDES the migration from the deploy-gated auto-run
+   * ({@link MigrationRunner.runPendingMigrations}); it keeps the Migrations-page flow
+   * (`runMigration`). Absence of the flag = automated — the invariant lives here, in the
+   * schema, not in deploy-pipeline prose.
+   */
+  manual?: boolean;
   status?: 'proposed' | 'running' | 'success' | 'failure';
   failureMessage?: string;
   failureStack?: string;
@@ -29,6 +38,7 @@ export class MigrationTable extends Table<Migration> {
   };
   public columns = withSourceRecordColumns<Migration>({
     description: new StringColumn('description', {}, 4000),
+    manual: new BooleanColumn('manual'),
     status: new StringColumn('status', { defaultValue: async () => 'proposed' }),
     failureMessage: new StringColumn('failure_message', {}, 4000),
     failureStack: new StringColumn('failure_stack', {}, 'MAX'),