@proteinjs/db 1.21.7 → 1.21.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.21.7",
+  "version": "1.21.9",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -41,19 +41,19 @@
     "test": "jest --passWithNoTests"
   },
   "dependencies": {
-    "@proteinjs/db-query": "^1.4.6",
-    "@proteinjs/logger": "^1.0.17",
+    "@proteinjs/db-query": "^1.4.7",
+    "@proteinjs/logger": "^1.0.18",
     "@proteinjs/reflection": "^1.1.11",
-    "@proteinjs/serializer": "^1.1.6",
-    "@proteinjs/server-api": "^3.0.4",
-    "@proteinjs/service": "^1.2.13",
-    "@proteinjs/user-auth": "^1.1.11",
+    "@proteinjs/serializer": "^1.1.7",
+    "@proteinjs/server-api": "^3.0.5",
+    "@proteinjs/service": "^1.2.14",
+    "@proteinjs/user-auth": "^1.1.12",
     "@proteinjs/util": "^1.6.0",
     "moment": "2.29.4",
     "uuid": "8.3.0"
   },
   "devDependencies": {
-    "@proteinjs/reflection-build": "^1.4.5",
+    "@proteinjs/reflection-build": "^1.4.6",
     "@types/jest": "29.5.5",
     "@types/node": "14.0.27",
     "@types/uuid": "8.3.0",
@@ -65,5 +65,6 @@
     "jest": "29.7.0",
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
-  }
+  },
+  "gitHead": "75411714e31c9033803f328cb43fa922d77d9782"
 }

package/src/source/SourceRecordLoader.ts

@@ -67,11 +67,12 @@ export class SourceRecordLoader {
   /**
    * Compare source record fields against the existing DB record to detect actual changes.
    * Only fields present on the source record are compared (ignoring `created`, `updated`).
-   * Uses serialization to normalize values (e.g. Reference objects, Moment, JSON) before comparison.
+   * Uses serialization to normalize values (e.g. Reference objects, Moment, JSON) before
+   * comparison, then delegates to {@link findMismatchPath}.
    *
-   * The comparison checks that every value in the source record exists with the same value
-   * in the existing DB record. Extra keys in the DB record are ignored — table watchers
-   * and hooks may enrich records with additional data after insert/update.
+   * Object-valued fields (e.g. `JsonColumn` blobs) are treated as source-authoritative:
+   * any structural drift, including extra keys left behind by earlier source versions,
+   * triggers a rewrite. Primitive columns retain their existing semantics.
    */
   private async hasChanges(table: Table<any>, sourceRecord: any, existingRecord: any): Promise<boolean> {
     const serializer = new RecordSerializer(table);
@@ -92,11 +93,43 @@ export class SourceRecordLoader {
     return false;
   }
 
+  private async getSourceRecordsMap() {
+    const sourceRecordsMap: SourceRecordsMap = {};
+    const sourceRecordTables = getSourceRecordTables();
+    for (const sourceRecordTable of sourceRecordTables) {
+      if (!sourceRecordsMap[sourceRecordTable.name]) {
+        sourceRecordsMap[sourceRecordTable.name] = { table: sourceRecordTable, records: [], recordIds: [] };
+      }
+    }
+
+    const sourceRecordLoaders = getSourceRecordLoaders();
+    for (const sourceRecordLoader of sourceRecordLoaders) {
+      if (!sourceRecordsMap[sourceRecordLoader.table.name]) {
+        sourceRecordsMap[sourceRecordLoader.table.name] = {
+          table: sourceRecordLoader.table,
+          records: [],
+          recordIds: [],
+        };
+      }
+
+      sourceRecordsMap[sourceRecordLoader.table.name].records.push(sourceRecordLoader.record);
+      sourceRecordsMap[sourceRecordLoader.table.name].recordIds.push(sourceRecordLoader.record.id);
+    }
+
+    return sourceRecordsMap;
+  }
+
   /**
    * Find the first point of divergence between source and existing values.
    * Returns a description of the mismatch path, or null if they match.
-   * For objects, extra keys in `existing` are ignored — they may have been added by
-   * table watchers or hooks after the source record was loaded.
+   *
+   * For object-valued fields (e.g. a `JsonColumn` blob), source is treated as
+   * fully authoritative: any structural drift — extra keys in existing, missing
+   * keys in existing, or value differences anywhere in the subtree — produces
+   * a mismatch. Comparison goes through {@link SourceRecordLoader.canonicalStringify}
+   * so that key ordering (which backing stores may canonicalize alphabetically)
+   * does not cause false positives.
+   *
    * For arrays, order and length must match exactly.
    */
   private findMismatchPath(source: any, existing: any, path: string): string | null {
@@ -139,47 +172,45 @@ export class SourceRecordLoader {
       return null;
     }
 
-    for (const key of Object.keys(source)) {
-      // Skip undefined values — they don't survive JSON serialization (JSON.stringify
-      // drops undefined), so the DB record won't have them.
-      if (source[key] === undefined) {
-        continue;
-      }
-      if (!(key in existing)) {
-        return `${path}.${key}: key missing in existing`;
-      }
-      const result = this.findMismatchPath(source[key], existing[key], `${path}.${key}`);
-      if (result) {
-        return result;
-      }
+    // Both values are non-null, non-array objects. Treat source as authoritative:
+    // any structural drift triggers a mismatch. Canonical stringify normalizes
+    // key order so storage-side canonicalization (e.g. Spanner alphabetizes JSON
+    // keys) doesn't register as drift.
+    if (this.canonicalStringify(source) !== this.canonicalStringify(existing)) {
+      return `${path}: object differs`;
     }
-
     return null;
   }
 
-  private async getSourceRecordsMap() {
-    const sourceRecordsMap: SourceRecordsMap = {};
-    const sourceRecordTables = getSourceRecordTables();
-    for (const sourceRecordTable of sourceRecordTables) {
-      if (!sourceRecordsMap[sourceRecordTable.name]) {
-        sourceRecordsMap[sourceRecordTable.name] = { table: sourceRecordTable, records: [], recordIds: [] };
-      }
+  /**
+   * Canonical JSON stringification with recursively sorted object keys.
+   *
+   * Why this exists: some stores (notably Spanner) canonicalize JSON object
+   * keys alphabetically on storage. Source records declared in TypeScript
+   * code don't guarantee alphabetical key order, so a plain `JSON.stringify`
+   * comparison between source and the existing DB value would produce false
+   * mismatches driven purely by key ordering. Sorting keys on both sides
+   * normalizes them so semantic equality maps to string equality.
+   *
+   * Arrays preserve order (order is semantic for arrays); only object keys
+   * are sorted.
+   */
+  private canonicalStringify(value: unknown): string {
+    if (value === null || typeof value !== 'object') {
+      return JSON.stringify(value);
     }
-
-    const sourceRecordLoaders = getSourceRecordLoaders();
-    for (const sourceRecordLoader of sourceRecordLoaders) {
-      if (!sourceRecordsMap[sourceRecordLoader.table.name]) {
-        sourceRecordsMap[sourceRecordLoader.table.name] = {
-          table: sourceRecordLoader.table,
-          records: [],
-          recordIds: [],
-        };
-      }
-
-      sourceRecordsMap[sourceRecordLoader.table.name].records.push(sourceRecordLoader.record);
-      sourceRecordsMap[sourceRecordLoader.table.name].recordIds.push(sourceRecordLoader.record.id);
+    if (Array.isArray(value)) {
+      // Mirror JSON.stringify: undefined array elements serialize as `null`.
+      return '[' + value.map((v) => (v === undefined ? 'null' : this.canonicalStringify(v))).join(',') + ']';
     }
-
-    return sourceRecordsMap;
+    // Mirror JSON.stringify: skip object properties whose value is `undefined`.
+    // This keeps source records that declare optional fields (as `undefined`)
+    // from being treated as drift vs existing rows that simply don't have the
+    // field — `undefined` would never have been written to the DB.
+    const obj = value as Record<string, unknown>;
+    const keys = Object.keys(obj)
+      .filter((k) => obj[k] !== undefined)
+      .sort();
+    return '{' + keys.map((k) => JSON.stringify(k) + ':' + this.canonicalStringify(obj[k])).join(',') + '}';
   }
 }

package/test/SourceRecordLoader.test.ts

@@ -0,0 +1,154 @@
+import { SourceRecordLoader } from '../src/source/SourceRecordLoader';
+
+/**
+ * Covers the two helpers that drive SourceRecordLoader's change-detection:
+ * - `canonicalStringify`: deterministic, key-order-independent JSON
+ * - `findMismatchPath`: per-field drift detection that uses canonical stringify
+ *   so object-valued columns (e.g. `JsonColumn`) are compared strictly while
+ *   remaining immune to storage-side key reordering (Spanner alphabetizes JSON
+ *   object keys on write).
+ *
+ * Both are private methods on `SourceRecordLoader`. Tests access them through
+ * the instance with a cast — cheaper than wiring full `hasChanges` fixtures
+ * (Table, RecordSerializer, Db) and keeps the public API surface of the class
+ * unchanged.
+ */
+
+type LoaderInternals = {
+  canonicalStringify: (value: unknown) => string;
+  findMismatchPath: (source: any, existing: any, path: string) => string | null;
+};
+
+const internals = () => new SourceRecordLoader() as unknown as LoaderInternals;
+
+describe('SourceRecordLoader.canonicalStringify', () => {
+  it('produces the same string for objects with different key orders', () => {
+    const loader = internals();
+    const a = { foo: 1, bar: 2, baz: 3 };
+    const b = { baz: 3, foo: 1, bar: 2 };
+    expect(loader.canonicalStringify(a)).toBe(loader.canonicalStringify(b));
+  });
+
+  it('sorts keys alphabetically in nested objects', () => {
+    const loader = internals();
+    const value = { z: { c: 1, a: 2, b: 3 }, a: 1 };
+    expect(loader.canonicalStringify(value)).toBe('{"a":1,"z":{"a":2,"b":3,"c":1}}');
+  });
+
+  it('preserves array order (arrays are semantically ordered)', () => {
+    const loader = internals();
+    expect(loader.canonicalStringify([3, 1, 2])).toBe('[3,1,2]');
+  });
+
+  it('handles primitives the same as JSON.stringify', () => {
+    const loader = internals();
+    expect(loader.canonicalStringify('hello')).toBe('"hello"');
+    expect(loader.canonicalStringify(42)).toBe('42');
+    expect(loader.canonicalStringify(true)).toBe('true');
+    expect(loader.canonicalStringify(null)).toBe('null');
+  });
+});
+
+describe('SourceRecordLoader.findMismatchPath', () => {
+  describe('object comparison', () => {
+    it('returns null for identical objects', () => {
+      const loader = internals();
+      const source = { content: '', type: 'body1', linkedThoughtId: '' };
+      const existing = { content: '', type: 'body1', linkedThoughtId: '' };
+      expect(loader.findMismatchPath(source, existing, 'newThoughtTemplate')).toBeNull();
+    });
+
+    it('returns null when existing has the same keys in a different order (Spanner alphabetizes)', () => {
+      // This is the critical case: Spanner canonicalizes JSON object keys
+      // alphabetically on storage. Source declared in TypeScript doesn't
+      // guarantee alphabetical key order. The two must still compare equal.
+      const loader = internals();
+      const source = { content: '', type: 'body1', linkedThoughtId: '' };
+      const existing = { content: '', linkedThoughtId: '', type: 'body1' };
+      expect(loader.findMismatchPath(source, existing, 'newThoughtTemplate')).toBeNull();
+    });
+
+    it('reports a mismatch when existing has extra keys (ThoughtLink drift case)', () => {
+      // Regression guard for the bug that prompted this change: source was
+      // simplified to drop TextType drift keys (fontSize, bold, italic, etc.),
+      // but the old "extras tolerated" behavior left the stale keys on the DB
+      // row, causing type-switch demotion to persist.
+      const loader = internals();
+      const source = {
+        thoughtType: { object: { content: '', type: 'body1', linkedThoughtId: '' } },
+      };
+      const existing = {
+        thoughtType: {
+          object: {
+            content: '',
+            type: 'body1',
+            linkedThoughtId: '',
+            fontSize: 14,
+            bold: false,
+            italic: false,
+          },
+        },
+      };
+      expect(loader.findMismatchPath(source, existing, 'newThoughtTemplate')).not.toBeNull();
+    });
+
+    it('reports a mismatch when existing is missing a key present in source', () => {
+      const loader = internals();
+      const source = { content: '', type: 'body1', linkedThoughtId: '' };
+      const existing = { content: '', type: 'body1' };
+      expect(loader.findMismatchPath(source, existing, 'newThoughtTemplate')).not.toBeNull();
+    });
+
+    it('reports a mismatch for a nested value difference', () => {
+      const loader = internals();
+      const source = { thoughtType: { object: { type: 'body1' } } };
+      const existing = { thoughtType: { object: { type: 'body2' } } };
+      expect(loader.findMismatchPath(source, existing, 'newThoughtTemplate')).not.toBeNull();
+    });
+
+    it('treats source-side undefined values as equivalent to missing in existing', () => {
+      // `JSON.stringify` drops undefined values, so the DB cannot store them
+      // in the first place. Source declaring an undefined field must not be
+      // treated as drift vs an existing row that lacks the field.
+      const loader = internals();
+      const source = { content: 'hi', optional: undefined };
+      const existing = { content: 'hi' };
+      expect(loader.findMismatchPath(source, existing, 'newThoughtTemplate')).toBeNull();
+    });
+  });
+
+  describe('array comparison', () => {
+    it('reports a mismatch on array length', () => {
+      const loader = internals();
+      expect(loader.findMismatchPath([1, 2], [1, 2, 3], 'tags')).not.toBeNull();
+    });
+
+    it('reports a mismatch on array element difference', () => {
+      const loader = internals();
+      expect(loader.findMismatchPath([1, 2], [1, 3], 'tags')).not.toBeNull();
+    });
+
+    it('returns null for equal arrays', () => {
+      const loader = internals();
+      expect(loader.findMismatchPath([1, 2, 3], [1, 2, 3], 'tags')).toBeNull();
+    });
+  });
+
+  describe('primitive comparison', () => {
+    it('returns null for equal primitives', () => {
+      const loader = internals();
+      expect(loader.findMismatchPath('x', 'x', 'name')).toBeNull();
+      expect(loader.findMismatchPath(5, 5, 'count')).toBeNull();
+    });
+
+    it('reports a mismatch for different primitives', () => {
+      const loader = internals();
+      expect(loader.findMismatchPath('x', 'y', 'name')).not.toBeNull();
+    });
+
+    it('reports a mismatch for type differences', () => {
+      const loader = internals();
+      expect(loader.findMismatchPath('1', 1, 'value')).not.toBeNull();
+    });
+  });
+});

package/test/reusable/CascadeDeleteTests.ts

@@ -40,8 +40,8 @@ export const cascadeDeleteTests = (
     const db = new Db(driver, undefined, transactionContextFactory);
     const testEnv = new DbTestEnvironment(driver, dropTable);
 
-    beforeAll(async () => await testEnv.beforeAll(), 10000);
-    afterAll(async () => await testEnv.afterAll(), 10000);
+    beforeAll(async () => await testEnv.beforeAll(), 30000);
+    afterAll(async () => await testEnv.afterAll(), 30000);
 
     /**
      * -------------------- Cascade Delete (holder → referenced) --------------------

package/test/reusable/ColumnTypesTests.ts

@@ -11,8 +11,8 @@ export const columnTypeTests = (
     const db = new Db(driver, undefined, transactionContextFactory);
     const testEnv = new DbTestEnvironment(driver, dropTable);
 
-    beforeAll(async () => await testEnv.beforeAll(), 10000);
-    afterAll(async () => await testEnv.afterAll(), 10000);
+    beforeAll(async () => await testEnv.beforeAll(), 30000);
+    afterAll(async () => await testEnv.afterAll(), 30000);
 
     test('Insert record with all null values', async () => {
       const testRecord: Omit<TestRecord, keyof Record> = {

package/test/reusable/CrudTests.ts

@@ -12,8 +12,8 @@ export const crudTests = (
     const db = new Db(driver, undefined, transactionContextFactory);
     const testEnv = new DbTestEnvironment(driver, dropTable);
 
-    beforeAll(async () => await testEnv.beforeAll(), 10000);
-    afterAll(async () => await testEnv.afterAll(), 10000);
+    beforeAll(async () => await testEnv.beforeAll(), 30000);
+    afterAll(async () => await testEnv.afterAll(), 30000);
 
     test('Insert', async () => {
       const testEmployee: Omit<Employee, keyof Record> = { name: 'Veronica' };
@@ -417,7 +417,7 @@ export const crudTests = (
       // Clean up
       await db.delete(table, { id: insertedRecord1.id });
       await db.delete(table, { id: insertedRecord2.id });
-    }, 10000);
+    }, 30000);
 
     test('Case sensitivity', async () => {
       const testEmployee: Omit<Employee, keyof Record> = {

package/test/reusable/DynamicReferenceColumnTests.ts

@@ -14,8 +14,8 @@ export const dynamicReferenceColumnTests = (driver: DbDriver, dropTable: (table:
     const db = new Db(driver);
     const testEnv = new DbTestEnvironment(driver, dropTable);
 
-    beforeAll(async () => await testEnv.beforeAll(), 10000);
-    afterAll(async () => await testEnv.afterAll(), 10000);
+    beforeAll(async () => await testEnv.beforeAll(), 30000);
+    afterAll(async () => await testEnv.afterAll(), 30000);
 
     test('should handle references to different types', async () => {
       // Create an engineer

package/test/reusable/TableManagerTests.ts

@@ -34,8 +34,8 @@ export const tableManagerTests = (
     const tableManager = driver.getTableManager();
     const testEnv = new DbTestEnvironment(driver, dropTable);
 
-    beforeAll(async () => await testEnv.beforeAll(), 10000);
-    afterAll(async () => await testEnv.afterAll(), 10000);
+    beforeAll(async () => await testEnv.beforeAll(), 30000);
+    afterAll(async () => await testEnv.afterAll(), 30000);
 
     afterEach(async () => {
       await dropTable(tableManagerTestTables.ColumnTypes);

package/test/reusable/TransactionTests.ts

@@ -16,8 +16,8 @@ export const transactionTests = (
     const db = new Db(driver, undefined, transactionContextFactory);
     const testEnv = new DbTestEnvironment(driver, dropTable);
 
-    beforeAll(async () => await testEnv.beforeAll(), 10000);
-    afterAll(async () => await testEnv.afterAll(), 10000);
+    beforeAll(async () => await testEnv.beforeAll(), 30000);
+    afterAll(async () => await testEnv.afterAll(), 30000);
 
     test('Transaction with successful operations', async () => {
       const testEmployee1: Omit<TransactionEmployee, keyof Record> = {