@proteinjs/db 1.34.3 → 1.35.0

package/src/source/SourceRecordLoader.ts

@@ -6,32 +6,119 @@ import { Db, getDbAsSystem } from '../Db';
 import { SourceRecordRepo } from './SourceRecordRepo';
 import { RecordSerializer } from '../Record';
 
+type DeclaredRecord = {
+  /** The owning package (the declaring loader's package) — the grain the sync prunes within. */
+  source: string;
+  record: Omit<SourceRecord, 'created' | 'updated'>;
+};
+
 type SourceRecordsMap = {
-  [tableName: string]: { table: Table<any>; records: Omit<SourceRecord, 'created' | 'updated'>[] };
+  [tableName: string]: { table: Table<any>; records: DeclaredRecord[] };
+};
+
+/** What one boot of the sync did to one source-record table. */
+export type SourceRecordTableLoadSummary = {
+  inserts: number;
+  updates: number;
+  unchanged: number;
+  adopted: number;
+  deletes: number;
+  removedUpdates: number;
+  /** Rows stamped by a newer version of their declaring package — left exactly as they are. */
+  skippedNewer: number;
+  /** Source-loaded rows with no owner stamp that no declaration in this build claims. */
+  unowned: number;
 };
 
+export type SourceRecordLoadSummary = { [tableName: string]: SourceRecordTableLoadSummary };
+
 export class SourceRecordLoader {
   private logger = new Logger({ name: this.constructor.name });
 
-  async load() {
-    const sourceRecordsMap = await this.getSourceRecordsMap();
+  /** Memo of resolved package versions — one resolution (and at most one warning) per source. */
+  private sourceVersions = new Map<string, string | undefined>();
+
+  /**
+   * One boot of the source-record sync, per source-record table in this build. Returns what it
+   * did to each table (also logged).
+   *
+   * Ownership model — every row has one owner, the package that declares it, ordered by that
+   * package's version:
+   * - A boot speaks only for the packages it carries. Rows owned by a package this build does
+   *   not carry are never touched: several servers running different builds against one shared
+   *   database coexist without pruning each other's rows.
+   * - Within a package, a boot speaks only for its own version or older. A row stamped by a
+   *   NEWER version of the same package is left exactly as it is — neither pruned nor rewritten —
+   *   so ordinary version skew (an older build restarting against rows a newer build added or
+   *   redefined) cannot delete or churn data. The newest version stays authoritative for
+   *   genuine removals and redefinitions.
+   * - A package in the build is authoritative for all of its rows on every table, including
+   *   tables it no longer declares anything for: dropping the last declaration is a removal.
+   * - Rows with no owner stamp (written before `source_package` existed) are claimed by the
+   *   declaration that still matches them; the rest are reported as unowned and left alone.
+   *
+   * Accepted residuals: a package removed from every build leaves its rows behind (no surviving
+   * boot carries its authority — clean up explicitly); two builds of one package at the SAME
+   * version with differing sets (uncommitted local skew) are last-writer-wins, since versions
+   * cannot order them.
+   *
+   * With no argument, every source-record table is synced (the `Db.init` full pass). Passing
+   * `onlyTable` scopes the sync to that one table — the pre-schema-sync migration phase uses
+   * this to land the migration ledger's rows before the full schema sync has run (see
+   * {@link MigrationRunner.runPreSchemaSyncMigrations}); the later full pass re-reconciles the
+   * same rows idempotently under the same ownership model.
+   */
+  async load(onlyTable?: Table<any>): Promise<SourceRecordLoadSummary> {
+    const { tables, buildSources } = await this.getDeclarations();
     const db = getDbAsSystem();
-    for (const tableName in sourceRecordsMap) {
-      const { table, records } = sourceRecordsMap[tableName];
+    const summary: SourceRecordLoadSummary = {};
+    for (const tableName in tables) {
+      if (onlyTable && tableName !== onlyTable.name) {
+        continue;
+      }
+      const { table, records } = tables[tableName];
       // 'id' unless the table declares a natural key (validated: unique-indexed, present and
       // unambiguous across declarations).
-      const keyProperty = this.validateSyncKey(table, records);
-      const declaredKeys = records.map((record) => (record as any)[keyProperty]);
-      const { deleteCount, removedUpdateCount } = await this.reconcileRemoved(db, table, keyProperty, declaredKeys);
+      const keyProperty = this.validateSyncKey(
+        table,
+        records.map(({ record }) => record)
+      );
+      // The exclusion set is the UNION of every key the build declares for the table: a key
+      // declared by ANY package in this build is re-owned by the stamp leg, never pruned (a
+      // declaration moving between packages within one build must not transit a delete+re-insert).
+      const allDeclaredKeys = records.map(({ record }) => (record as any)[keyProperty]);
+      const removed = await this.reconcileRemoved(db, table, keyProperty, buildSources, allDeclaredKeys);
 
       let insertCount = 0;
       let updateCount = 0;
       let unchangedCount = 0;
       let adoptedCount = 0;
-      for (let sourceRecord of records) {
+      let skippedNewer = removed.skippedNewer;
+      for (const { source, record } of records) {
+        let sourceRecord = record;
         sourceRecord.isLoadedFromSource = true;
+        // Ownership stamp: the declaring package (and its version) claims the row. Stamped
+        // before the drift comparison so pre-existing rows (including pre-source_package legacy
+        // rows and rows whose declaration moved packages) converge to the current owner on
+        // their next boot.
+        sourceRecord.sourcePackage = source;
+        const sourceVersion = this.sourceVersion(source);
+        if (sourceVersion !== undefined) {
+          sourceRecord.sourcePackageVersion = sourceVersion;
+        }
         const existingRecord = await db.get(table, { [keyProperty]: (sourceRecord as any)[keyProperty] });
         if (existingRecord) {
+          if (
+            existingRecord.sourcePackage === source &&
+            this.isNewerStamp(existingRecord.sourcePackageVersion, sourceVersion)
+          ) {
+            // A newer version of this very package already defined the row: this build's older
+            // definition does not land, and the row's stamp is not downgraded.
+            skippedNewer += 1;
+            new SourceRecordRepo().loadSourceRecord(table.name, existingRecord);
+            continue;
+          }
+
           if (existingRecord.id !== sourceRecord.id) {
             // Adopt in place: the existing row keeps its id — other tables may reference it.
             // The declared id is only ever used for fresh inserts.
@@ -64,61 +151,118 @@ export class SourceRecordLoader {
         new SourceRecordRepo().loadSourceRecord(table.name, sourceRecord as any);
       }
 
+      const unowned = await this.countUnowned(db, table, keyProperty);
+      summary[table.name] = {
+        inserts: insertCount,
+        updates: updateCount,
+        unchanged: unchangedCount,
+        adopted: adoptedCount,
+        deletes: removed.deleteCount,
+        removedUpdates: removed.removedUpdateCount,
+        skippedNewer,
+        unowned,
+      };
       this.logger.info({
         message: `(${table.name}) Loaded ${records.length} ${records.length == 1 ? 'record' : 'records'} from source`,
-        obj: {
-          inserts: insertCount,
-          updates: updateCount,
-          unchanged: unchangedCount,
-          adopted: adoptedCount,
-          deletes: deleteCount,
-          removedUpdates: removedUpdateCount,
-        },
+        obj: summary[table.name],
       });
     }
+
+    return summary;
   }
 
   /**
-   * The removed-reconcile leg: rows previously loaded from source whose declaration no longer
-   * exists (`is_loaded_from_source = true AND <key> NOT IN declared`), handled per the table's
-   * `onSourceRemoved` policy — delete (default), keep, or update with a patch. The update leg
-   * applies the patch only to rows whose fields actually differ (idempotent boots), through
-   * `Db.update` so table watchers observe each write.
+   * The removed-reconcile leg: rows owned by a package this build carries, at that package's
+   * version or older, that NO package in this build still declares
+   * (`is_loaded_from_source = true AND source_package IN <build's packages> AND <key> NOT IN
+   * <build's declared keys for the table>`, minus rows stamped by a newer version of their
+   * package) are handled per the table's `onSourceRemoved` policy — delete (default), keep, or
+   * update with a patch. The update leg applies the patch only to rows whose fields actually
+   * differ (idempotent boots), through `Db.update` so table watchers observe each write.
+   * See {@link load} for the ownership model.
    */
   private async reconcileRemoved(
     db: Db,
     table: Table<any>,
     keyProperty: string,
-    declaredKeys: unknown[]
-  ): Promise<{ deleteCount: number; removedUpdateCount: number }> {
+    buildSources: Set<string>,
+    allDeclaredKeys: unknown[]
+  ): Promise<{ deleteCount: number; removedUpdateCount: number; skippedNewer: number }> {
     const policy = table.sourceRecordOptions.onSourceRemoved ?? 'delete';
-    if (policy === 'keep') {
-      return { deleteCount: 0, removedUpdateCount: 0 };
+    if (policy === 'keep' || buildSources.size == 0) {
+      return { deleteCount: 0, removedUpdateCount: 0, skippedNewer: 0 };
     }
 
     const qb = QueryBuilder.fromObject<SourceRecord>({ isLoadedFromSource: true }, table.name);
-    if (declaredKeys.length > 0) {
-      qb.condition({ field: keyProperty as any, operator: 'NOT IN', value: declaredKeys as any });
+    qb.condition({ field: 'sourcePackage', operator: 'IN', value: Array.from(buildSources) as any });
+    if (allDeclaredKeys.length > 0) {
+      qb.condition({ field: keyProperty as any, operator: 'NOT IN', value: allDeclaredKeys as any });
+    }
+
+    const candidates: SourceRecord[] = await db.query(table, qb);
+    const stampedNewer = (candidate: SourceRecord) =>
+      this.isNewerStamp(candidate.sourcePackageVersion, this.sourceVersion(candidate.sourcePackage as string));
+    const removedRecords = candidates.filter((candidate) => !stampedNewer(candidate));
+    const skippedNewer = candidates.length - removedRecords.length;
+    if (skippedNewer > 0) {
+      this.logger.info({
+        message: `(${table.name}) Left ${skippedNewer} record${skippedNewer == 1 ? '' : 's'} stamped by a newer version of ${skippedNewer == 1 ? 'its' : 'their'} package — not treated as removed`,
+        obj: { [keyProperty]: candidates.filter(stampedNewer).map((candidate) => (candidate as any)[keyProperty]) },
+      });
+    }
+
+    if (removedRecords.length == 0) {
+      return { deleteCount: 0, removedUpdateCount: 0, skippedNewer };
     }
 
     if (policy === 'delete') {
-      return { deleteCount: await db.delete(table, qb), removedUpdateCount: 0 };
+      const deleteQb = QueryBuilder.fromObject<SourceRecord>({ isLoadedFromSource: true }, table.name);
+      deleteQb.condition({ field: 'id', operator: 'IN', value: removedRecords.map((record) => record.id) });
+      return { deleteCount: await db.delete(table, deleteQb), removedUpdateCount: 0, skippedNewer };
     }
 
     let removedUpdateCount = 0;
-    const removedRecords = await db.query(table, qb);
     for (const removedRecord of removedRecords) {
       if (await this.hasChanges(table, policy.update, removedRecord)) {
         await db.update(table, { id: removedRecord.id, ...policy.update });
         removedUpdateCount += 1;
         this.logger.info({
           message: `(${table.name}) Applied onSourceRemoved update to record removed from source`,
-          obj: { id: removedRecord.id, [keyProperty]: (removedRecord as any)[keyProperty] },
+          obj: {
+            id: removedRecord.id,
+            [keyProperty]: (removedRecord as any)[keyProperty],
+            source: removedRecord.sourcePackage,
+          },
         });
       }
     }
 
-    return { deleteCount: 0, removedUpdateCount };
+    return { deleteCount: 0, removedUpdateCount, skippedNewer };
+  }
+
+  /**
+   * Source-loaded rows with no owner stamp that no declaration in this build claimed on this
+   * boot (the stamp leg has already run). They predate `source_package`, or their declaring
+   * package is not in this build; nobody prunes them. Reported so they are visible, never acted
+   * on. Tables that keep removed rows (`onSourceRemoved: 'keep'`) opted out of removal
+   * semantics altogether, so there is nothing to explain there.
+   */
+  private async countUnowned(db: Db, table: Table<any>, keyProperty: string): Promise<number> {
+    if ((table.sourceRecordOptions.onSourceRemoved ?? 'delete') === 'keep') {
+      return 0;
+    }
+
+    const qb = QueryBuilder.fromObject<SourceRecord>({ isLoadedFromSource: true }, table.name);
+    qb.condition({ field: 'sourcePackage', operator: 'IS NULL' });
+    const unowned: SourceRecord[] = await db.query(table, qb);
+    if (unowned.length > 0) {
+      this.logger.warn({
+        message: `(${table.name}) ${unowned.length} source-loaded ${unowned.length == 1 ? 'record has' : 'records have'} no owning package and no declaration in this build — left untouched`,
+        obj: { [keyProperty]: unowned.map((record) => (record as any)[keyProperty]) },
+      });
+    }
+
+    return unowned.length;
   }
 
   /**
@@ -205,28 +349,27 @@ export class SourceRecordLoader {
     return false;
   }
 
-  private async getSourceRecordsMap() {
-    const sourceRecordsMap: SourceRecordsMap = {};
-    const sourceRecordTables = getSourceRecordTables();
-    for (const sourceRecordTable of sourceRecordTables) {
-      if (!sourceRecordsMap[sourceRecordTable.name]) {
-        sourceRecordsMap[sourceRecordTable.name] = { table: sourceRecordTable, records: [] };
-      }
+  /**
+   * Every source-record table in this build with the records declared for it, plus the set of
+   * packages that declare ANY source record in this build — the packages this boot speaks for.
+   */
+  private async getDeclarations(): Promise<{ tables: SourceRecordsMap; buildSources: Set<string> }> {
+    const tables: SourceRecordsMap = {};
+    for (const table of getSourceRecordTables()) {
+      tables[table.name] = { table, records: [] };
     }
 
-    const sourceRecordLoaders = getSourceRecordLoaders();
-    for (const sourceRecordLoader of sourceRecordLoaders) {
-      if (!sourceRecordsMap[sourceRecordLoader.table.name]) {
-        sourceRecordsMap[sourceRecordLoader.table.name] = {
-          table: sourceRecordLoader.table,
-          records: [],
-        };
+    const buildSources = new Set<string>();
+    for (const { source, loader } of getSourceRecordLoaders()) {
+      buildSources.add(source);
+      if (!tables[loader.table.name]) {
+        tables[loader.table.name] = { table: loader.table, records: [] };
       }
 
-      sourceRecordsMap[sourceRecordLoader.table.name].records.push(sourceRecordLoader.record);
+      tables[loader.table.name].records.push({ source, record: loader.record });
     }
 
-    return sourceRecordsMap;
+    return { tables, buildSources };
   }
 
   /**
@@ -323,4 +466,155 @@ export class SourceRecordLoader {
       .sort();
     return '{' + keys.map((k) => JSON.stringify(k) + ':' + this.canonicalStringify(obj[k])).join(',') + '}';
   }
+
+  /** Memoized {@link resolveSourceVersion} — one resolution (and at most one warning) per source. */
+  private sourceVersion(source: string): string | undefined {
+    if (!this.sourceVersions.has(source)) {
+      this.sourceVersions.set(source, this.resolveSourceVersion(source));
+    }
+
+    return this.sourceVersions.get(source);
+  }
+
+  /**
+   * The declaring package's version, read from its own package.json at runtime. Resolution runs
+   * from the process cwd — the booting server's package, the one dependency tree every declaring
+   * package is reachable from (`@proteinjs/db` itself does not depend on the packages that
+   * declare records, so module-relative resolution could never find them). The package's entry
+   * is resolved (honoring exports maps, where `<pkg>/package.json` is usually not requireable),
+   * then the nearest package.json whose `name` matches is read walking up from it.
+   *
+   * Returns undefined where resolution is impossible — no Node `require` (browser bundles), or a
+   * package not resolvable from cwd. The sync then has no place in the version order for that
+   * package: it never touches its version-stamped rows, and what it writes is unversioned
+   * (see {@link isNewerStamp}). Logged once per boot so a misconfigured cwd is visible.
+   */
+  private resolveSourceVersion(source: string): string | undefined {
+    // The ambient CJS `require` — the only require carrying `.resolve` (module.require does
+    // not). Every use below goes through the variable, so bundlers never see a statically
+    // analyzable `require(...)` call; in a browser bundle the runtime attempts throw into
+    // their catches and the method degrades to undefined.
+    const nodeRequire: NodeRequire | undefined =
+      typeof require === 'function' && typeof require.resolve === 'function' ? require : undefined;
+    const cwd = typeof process !== 'undefined' && typeof process.cwd === 'function' ? process.cwd() : undefined;
+    if (nodeRequire && cwd) {
+      try {
+        const entryPath = nodeRequire.resolve(source, { paths: [cwd] });
+        const path = nodeRequire('path');
+        const fs = nodeRequire('fs');
+        // Walk up from the entry file to the package's own package.json (the name check skips
+        // nested stubs like a dist/package.json); stop at the filesystem root.
+        for (let directory = path.dirname(entryPath); ; directory = path.dirname(directory)) {
+          const candidate = path.join(directory, 'package.json');
+          if (fs.existsSync(candidate)) {
+            const packageJson = JSON.parse(fs.readFileSync(candidate, 'utf8'));
+            if (packageJson?.name === source && typeof packageJson.version === 'string') {
+              return packageJson.version;
+            }
+          }
+
+          if (path.dirname(directory) === directory) {
+            break;
+          }
+        }
+      } catch (error) {
+        // Unresolvable from cwd — reported below.
+      }
+    }
+
+    this.logger.warn({
+      message: `Could not resolve a version for source package '${source}' from '${cwd}' — its records sync without version ordering (this build never touches its version-stamped rows)`,
+    });
+    return undefined;
+  }
+
+  /**
+   * Whether a row's version stamp is strictly newer than the reconciling build's version of the
+   * same package — the guard that makes version skew of one package safe on a shared database:
+   * an older build never prunes (or flags) rows a newer build of the same package declared.
+   *
+   * Unversioned stamps (NULL — legacy rows, or builds whose version could not be resolved)
+   * carry no ordering and stay reconcilable (last-writer-wins, the pre-version behavior). A
+   * build whose OWN version is unresolvable cannot place itself in the order, so it never
+   * touches a version-stamped row — conservative: prefer leaving a removed row behind over
+   * deleting one that might be newer.
+   */
+  private isNewerStamp(stampedVersion: string | null | undefined, reconcilingVersion: string | undefined): boolean {
+    if (stampedVersion == null) {
+      return false;
+    }
+
+    if (reconcilingVersion == null) {
+      return true;
+    }
+
+    const comparison = this.compareVersions(stampedVersion, reconcilingVersion);
+    return comparison === undefined ? true : comparison > 0;
+  }
+
+  /**
+   * Semver comparison (major.minor.patch, prerelease-aware): negative when a < b, 0 when equal,
+   * positive when a > b, and undefined when either side does not parse as semver (unorderable —
+   * the caller treats that conservatively).
+   */
+  private compareVersions(a: string, b: string): number | undefined {
+    const parse = (version: string): { main: number[]; prerelease?: string[] } | undefined => {
+      const match = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?/.exec(version.trim());
+      if (!match) {
+        return undefined;
+      }
+
+      return { main: [Number(match[1]), Number(match[2]), Number(match[3])], prerelease: match[4]?.split('.') };
+    };
+
+    const parsedA = parse(a);
+    const parsedB = parse(b);
+    if (!parsedA || !parsedB) {
+      return undefined;
+    }
+
+    for (let i = 0; i < 3; i++) {
+      if (parsedA.main[i] !== parsedB.main[i]) {
+        return parsedA.main[i] - parsedB.main[i];
+      }
+    }
+
+    if (!parsedA.prerelease && !parsedB.prerelease) {
+      return 0;
+    }
+    if (!parsedA.prerelease) {
+      return 1; // a release outranks any prerelease of the same triple
+    }
+    if (!parsedB.prerelease) {
+      return -1;
+    }
+
+    const length = Math.max(parsedA.prerelease.length, parsedB.prerelease.length);
+    for (let i = 0; i < length; i++) {
+      const identifierA = parsedA.prerelease[i];
+      const identifierB = parsedB.prerelease[i];
+      if (identifierA === undefined) {
+        return -1; // fewer identifiers sorts first when all shared ones are equal
+      }
+      if (identifierB === undefined) {
+        return 1;
+      }
+
+      const numericA = /^\d+$/.test(identifierA) ? Number(identifierA) : undefined;
+      const numericB = /^\d+$/.test(identifierB) ? Number(identifierB) : undefined;
+      if (numericA !== undefined && numericB !== undefined) {
+        if (numericA !== numericB) {
+          return numericA - numericB;
+        }
+      } else if (numericA !== undefined) {
+        return -1; // numeric identifiers sort before alphanumeric ones
+      } else if (numericB !== undefined) {
+        return 1;
+      } else if (identifierA !== identifierB) {
+        return identifierA < identifierB ? -1 : 1;
+      }
+    }
+
+    return 0;
+  }
 }

package/src/tables/MigrationTable.ts

@@ -14,6 +14,29 @@ export interface Migration extends SourceRecord {
    * schema, not in deploy-pipeline prose.
    */
   manual?: boolean;
+  /**
+   * Runs during `Db.init()` BEFORE schema sync (the pre-schema-sync phase —
+   * {@link MigrationRunner.runPreSchemaSyncMigrations}), instead of after init like the
+   * deploy-gated series. This is the class for data repairs a NEW SCHEMA INVARIANT depends on —
+   * e.g. deduplicating rows before a unique index lands: schema sync's unique-index preflight
+   * fails loudly over violating data ({@link DuplicateValuesForUniqueIndexError}), and the
+   * ordinary series (deploy Job, after init) is too late by construction.
+   *
+   * Contract for this class (stricter than the ordinary automated class):
+   * - IDEMPOTENT and tolerant of CONCURRENT duplicate runs: every booting replica and the deploy
+   *   Job each run init — two actors can observe the row un-applied and both run the body.
+   * - TABLE-EXISTENCE tolerant: on a fresh database the body runs before ANY schema sync, so its
+   *   target tables may not exist yet (check and no-op — a fresh database has nothing to repair).
+   * - Never `manual` (a contradiction — the phase exists to run unattended before DDL; declaring
+   *   both fails init loudly).
+   * A failure fails `Db.init()` loudly (recorded on the ledger row, retried next boot) — exactly
+   * the failure the schema sync would otherwise hit, but named and retryable.
+   *
+   * The phase reads the SOURCE declaration (like {@link runPendingMigrations} reads
+   * `source.manual`); the column mirrors it into the ledger so the Migrations page shows why a
+   * row ran at boot.
+   */
+  preSchemaSync?: boolean;
   /**
    * Ledger-owned state (like `status` — never declared on a source record): stamped `true` by the
    * deploy-gated series ({@link MigrationRunner.runPendingMigrations}) when the row's source class
@@ -47,6 +70,7 @@ export class MigrationTable extends Table<Migration> {
   public columns = withSourceRecordColumns<Migration>({
     description: new StringColumn('description', {}, 4000),
     manual: new BooleanColumn('manual'),
+    preSchemaSync: new BooleanColumn('pre_schema_sync'),
     retired: new BooleanColumn('retired'),
     status: new StringColumn('status', { defaultValue: async () => 'proposed' }),
     failureMessage: new StringColumn('failure_message', {}, 4000),