@powersync/service-core 1.24.0 → 1.25.0

package/src/storage/SourceTableReconciler.ts

@@ -0,0 +1,188 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { isDeepStrictEqual } from 'node:util';
+import { JsonValue, SourceEntityDescriptor } from './SourceEntity.js';
+import { SourceTable, SourceTableCandidate, SourceTableId, sourceTableIdEquals } from './SourceTable.js';
+
+/**
+ * A source connector's classification of overlapping persisted tables.
+ */
+export interface SourceTableCandidateResolution {
+  /**
+   * Records storage can reuse. Copies may include updated source metadata.
+   */
+  compatibleTables: ReadonlyArray<SourceTableCandidate>;
+
+  /**
+   * Records that cannot be reused. Every candidate must appear in exactly one result list.
+   */
+  incompatibleTables: ReadonlyArray<SourceTableCandidate>;
+
+  /**
+   * Values for records storage creates during this resolution.
+   */
+  newTableValues: SourceTableCreateValues;
+}
+
+export interface SourceTableCreateValues {
+  /**
+   * Source metadata for new records. Null means no metadata.
+   */
+  sourceMetadata: JsonValue;
+}
+
+/**
+ * Input to a source-owned reconciliation callback. The callback may run inside a storage
+ * transaction, so it must not mutate storage or perform slow external work.
+ */
+export interface SourceTableCandidateReconcilerInput {
+  /**
+   * Source entity being resolved.
+   */
+  source: SourceEntityDescriptor;
+
+  /**
+   * Persisted tables overlapping by name or object id.
+   */
+  candidates: ReadonlyArray<SourceTableCandidate>;
+}
+
+export type SourceTableCandidateReconciler = (
+  input: SourceTableCandidateReconcilerInput
+) => SourceTableCandidateResolution | Promise<SourceTableCandidateResolution>;
+
+/**
+ * Compare replica-id columns in order.
+ */
+export function sameReplicaIdColumns(
+  left: SourceTableCandidate['replicaIdColumns'],
+  right: SourceEntityDescriptor
+): boolean {
+  const target = right.replicaIdColumns;
+  return (
+    left.length == target.length &&
+    left.every(
+      (column, index) =>
+        column.name == target[index].name && column.type == target[index].type && column.typeId == target[index].typeId
+    )
+  );
+}
+
+/**
+ * Compare the shared source-table identity fields.
+ */
+export function sourceIdentityCompatible(source: SourceEntityDescriptor, candidate: SourceTableCandidate): boolean {
+  return (
+    candidate.schema == source.schema &&
+    candidate.name == source.name &&
+    (source.objectId == null || candidate.objectId == source.objectId) &&
+    sameReplicaIdColumns(candidate.replicaIdColumns, source)
+  );
+}
+
+/**
+ * Default identity-based reconciliation for connectors without source-specific metadata.
+ */
+export const defaultSourceTableReconciler: SourceTableCandidateReconciler = ({ source, candidates }) => {
+  const compatibleTables: SourceTableCandidate[] = [];
+  const incompatibleTables: SourceTableCandidate[] = [];
+  for (const candidate of candidates) {
+    if (sourceIdentityCompatible(source, candidate)) {
+      compatibleTables.push(candidate);
+    } else {
+      incompatibleTables.push(candidate);
+    }
+  }
+  return { compatibleTables, incompatibleTables, newTableValues: { sourceMetadata: null } };
+};
+
+/**
+ * Check that every candidate was classified exactly once.
+ */
+export function validateSourceTableCandidateResolution(
+  candidates: ReadonlyArray<SourceTableCandidate>,
+  resolution: SourceTableCandidateResolution
+): void {
+  const classifiedTables = [...resolution.compatibleTables, ...resolution.incompatibleTables];
+
+  for (const candidate of candidates) {
+    const classifications = classifiedTables.filter((table) => sourceTableIdEquals(table.id, candidate.id));
+    if (classifications.length !== 1) {
+      throw new ServiceAssertionError(
+        `Source table candidate ${candidate.id.toString()} must be classified exactly once, got ${classifications.length}`
+      );
+    }
+  }
+
+  for (const table of classifiedTables) {
+    if (!candidates.some((candidate) => sourceTableIdEquals(candidate.id, table.id))) {
+      throw new ServiceAssertionError(`Source table reconciliation returned unknown candidate ${table.id.toString()}`);
+    }
+  }
+}
+
+/**
+ * A source-metadata update to persist.
+ */
+export interface SourceTableMetadataUpdate {
+  id: SourceTableId;
+  sourceMetadata: JsonValue;
+}
+
+/**
+ * Rebuild a resolution from storage-owned tables, applying only reconciler-owned metadata to
+ * compatible tables. All other mutable table state comes from storage.
+ *
+ * Reconciler candidates are typed as read-only, but TypeScript types provide no runtime protection:
+ * callback code can cast a cloned candidate and mutate it. Rematerializing by id ensures those
+ * mutations are not trusted even when the compile-time boundary is bypassed.
+ */
+export function materializeSourceTableResolution(
+  tables: ReadonlyArray<SourceTable>,
+  resolution: SourceTableCandidateResolution
+): MaterializedSourceTableResolution {
+  const findTable = (candidate: SourceTableCandidate): SourceTable => {
+    const table = tables.find((table) => sourceTableIdEquals(table.id, candidate.id));
+    if (table == null) {
+      throw new ServiceAssertionError(`Source table candidate ${candidate.id.toString()} was not persisted`);
+    }
+    return table;
+  };
+  return {
+    compatibleTables: resolution.compatibleTables.map((candidate) =>
+      findTable(candidate).withSourceMetadata(candidate.sourceMetadata)
+    ),
+    incompatibleTables: resolution.incompatibleTables.map(findTable),
+    newTableValues: resolution.newTableValues
+  };
+}
+
+export interface MaterializedSourceTableResolution {
+  compatibleTables: SourceTable[];
+  incompatibleTables: SourceTable[];
+  newTableValues: SourceTableCreateValues;
+}
+
+/**
+ * Return source-metadata changes from compatible candidates, comparing metadata by value against
+ * the original storage-owned tables. The reconciler may mutate its isolated candidate clones, so
+ * those clones cannot be used as the persisted baseline.
+ */
+export function diffSourceTableUpdates(
+  persistedTables: ReadonlyArray<SourceTable>,
+  resolution: SourceTableCandidateResolution
+): SourceTableMetadataUpdate[] {
+  const updates: SourceTableMetadataUpdate[] = [];
+  for (const resolvedTable of resolution.compatibleTables) {
+    const persistedTable = persistedTables.find((table) => sourceTableIdEquals(table.id, resolvedTable.id));
+    if (persistedTable == null) {
+      throw new ServiceAssertionError(
+        `Source table reconciliation returned unknown candidate ${resolvedTable.id.toString()}`
+      );
+    }
+    if (isDeepStrictEqual(persistedTable.sourceMetadata, resolvedTable.sourceMetadata)) {
+      continue;
+    }
+    updates.push({ id: resolvedTable.id, sourceMetadata: resolvedTable.sourceMetadata });
+  }
+  return updates;
+}

package/src/storage/SyncRulesBucketStorage.ts

@@ -14,6 +14,7 @@ import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
 import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { SourceTableCandidateReconciler } from './SourceTableReconciler.js';
 import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
@@ -212,6 +213,11 @@ export interface ResolveTablesOptions {
    * Source table or collection metadata discovered during snapshot or streaming.
    */
   source: SourceEntityDescriptor;
+  /**
+   * Classifies overlapping persisted tables. Defaults to identity-based reconciliation.
+   * This may run inside a storage transaction and must not mutate storage.
+   */
+  reconcileSourceTables?: SourceTableCandidateReconciler;
   /**
    * For tests only - custom id generator for stable ids.
    */

package/src/storage/storage-index.ts

@@ -13,6 +13,7 @@ export * from './ReplicationLock.js';
 export * from './ReportStorage.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
+export * from './SourceTableReconciler.js';
 export * from './storage-metrics.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';

package/src/util/utils.ts

@@ -33,7 +33,7 @@ export interface PartialChecksum {
  */
 export type InternalOpId = bigint;
 
-export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
+export const ID_NAMESPACE = uuid.parse('a396dd91-09fc-4017-a28d-3df722f651e9');
 
 export function escapeIdentifier(identifier: string) {
   return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;

package/test/src/source-table-reconciler.test.ts

@@ -0,0 +1,244 @@
+import { SourceEntityDescriptor } from '@/storage/SourceEntity.js';
+import { SourceTable, sourceTableIdEquals } from '@/storage/SourceTable.js';
+import {
+  defaultSourceTableReconciler,
+  diffSourceTableUpdates,
+  materializeSourceTableResolution,
+  sourceIdentityCompatible,
+  validateSourceTableCandidateResolution
+} from '@/storage/SourceTableReconciler.js';
+import * as bson from 'bson';
+import { describe, expect, it } from 'vitest';
+
+function descriptor(overrides: Partial<SourceEntityDescriptor> = {}): SourceEntityDescriptor {
+  return {
+    connectionTag: 'default',
+    schema: 'public',
+    name: 'users',
+    objectId: 100,
+    replicaIdColumns: [{ name: 'id', type: 'int', typeId: 23 }],
+    ...overrides
+  };
+}
+
+function candidate(overrides: Partial<ConstructorParameters<typeof SourceTable>[0]> = {}): SourceTable {
+  return new SourceTable({
+    id: overrides.id ?? 'table-1',
+    ref: overrides.ref ?? { connectionTag: 'default', schema: 'public', name: 'users' },
+    objectId: 'objectId' in overrides ? overrides.objectId! : 100,
+    replicaIdColumns: overrides.replicaIdColumns ?? [{ name: 'id', type: 'int', typeId: 23 }],
+    snapshotComplete: overrides.snapshotComplete ?? true,
+    bucketDataSources: overrides.bucketDataSources ?? [],
+    parameterLookupSources: overrides.parameterLookupSources ?? [],
+    sourceMetadata: overrides.sourceMetadata
+  });
+}
+
+describe('sourceIdentityCompatible', () => {
+  it('matches identical identity', () => {
+    expect(sourceIdentityCompatible(descriptor(), candidate())).toBe(true);
+  });
+
+  it('rejects a different object id', () => {
+    expect(sourceIdentityCompatible(descriptor({ objectId: 200 }), candidate({ objectId: 100 }))).toBe(false);
+  });
+
+  it('rejects a different schema/name', () => {
+    expect(sourceIdentityCompatible(descriptor({ name: 'accounts' }), candidate())).toBe(false);
+  });
+
+  it('rejects changed replica-id columns', () => {
+    expect(
+      sourceIdentityCompatible(
+        descriptor(),
+        candidate({ replicaIdColumns: [{ name: 'id', type: 'bigint', typeId: 20 }] })
+      )
+    ).toBe(false);
+  });
+
+  it('treats an undefined descriptor object id as a wildcard on object id', () => {
+    expect(sourceIdentityCompatible(descriptor({ objectId: undefined }), candidate({ objectId: 999 }))).toBe(true);
+  });
+});
+
+describe('defaultSourceTableReconciler', () => {
+  it('returns all identity-compatible candidates and no metadata', async () => {
+    const a = candidate({ id: 'a' });
+    const b = candidate({ id: 'b', ref: { connectionTag: 'default', schema: 'public', name: 'accounts' } });
+    const resolution = await defaultSourceTableReconciler({ source: descriptor(), candidates: [a, b] });
+    expect(resolution.compatibleTables).toEqual([a]);
+    expect(resolution.incompatibleTables).toEqual([b]);
+    expect(resolution.newTableValues).toEqual({ sourceMetadata: null });
+  });
+
+  it('returns an empty set when nothing matches', async () => {
+    const resolution = await defaultSourceTableReconciler({
+      source: descriptor({ objectId: 200 }),
+      candidates: [candidate({ objectId: 100 })]
+    });
+    expect(resolution.compatibleTables).toHaveLength(0);
+    expect(resolution.incompatibleTables.map((table) => table.id)).toEqual(['table-1']);
+  });
+});
+
+describe('validateSourceTableCandidateResolution', () => {
+  it('accepts an explicit compatible/incompatible partition', () => {
+    const a = candidate({ id: 'a' });
+    const b = candidate({ id: 'b' });
+    expect(() =>
+      validateSourceTableCandidateResolution([a, b], {
+        compatibleTables: [a],
+        incompatibleTables: [b],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).not.toThrow();
+  });
+
+  it('rejects omitted, duplicate, and unknown candidates', () => {
+    const a = candidate({ id: 'a' });
+    expect(() =>
+      validateSourceTableCandidateResolution([a], {
+        compatibleTables: [],
+        incompatibleTables: [],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).toThrow(/exactly once/);
+    expect(() =>
+      validateSourceTableCandidateResolution([a], {
+        compatibleTables: [a],
+        incompatibleTables: [a],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).toThrow(/exactly once/);
+    expect(() =>
+      validateSourceTableCandidateResolution([a], {
+        compatibleTables: [a],
+        incompatibleTables: [candidate({ id: 'unknown' })],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).toThrow(/unknown candidate/);
+  });
+});
+
+describe('diffSourceTableUpdates', () => {
+  function resolution(compatibleTables: SourceTable[]) {
+    return { compatibleTables, incompatibleTables: [], newTableValues: { sourceMetadata: null } };
+  }
+
+  it('returns nothing when the reconciler returned the candidates untouched', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    expect(diffSourceTableUpdates([a], resolution([a]))).toEqual([]);
+  });
+
+  it('compares by value, not by reference', () => {
+    // A new object with equal metadata should not cause a write.
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    const rebuilt = a.withSourceMetadata({ captureTableObjectId: 7 });
+
+    expect(rebuilt.sourceMetadata).not.toBe(a.sourceMetadata);
+    expect(diffSourceTableUpdates([a], resolution([rebuilt]))).toEqual([]);
+  });
+
+  it('returns changed metadata', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata({ captureTableObjectId: 8 })]))).toEqual([
+      { id: 'a', sourceMetadata: { captureTableObjectId: 8 } }
+    ]);
+  });
+
+  it('returns metadata added to a legacy record', () => {
+    const legacy = candidate({ id: 'a' });
+    expect(legacy.sourceMetadata).toBeNull();
+    expect(
+      diffSourceTableUpdates([legacy], resolution([legacy.withSourceMetadata({ captureTableObjectId: 7 })]))
+    ).toEqual([{ id: 'a', sourceMetadata: { captureTableObjectId: 7 } }]);
+  });
+
+  it('returns cleared metadata as null', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata(null)]))).toEqual([
+      { id: 'a', sourceMetadata: null }
+    ]);
+  });
+
+  it('materializes only source metadata from reconciler results', () => {
+    const persisted = candidate({ id: 'a', snapshotComplete: false });
+    const modified = persisted.withSourceMetadata({ captureTableObjectId: 8 });
+    modified.snapshotComplete = true;
+    modified.syncData = false;
+
+    const [materialized] = materializeSourceTableResolution([persisted], resolution([modified])).compatibleTables;
+
+    expect(materialized.sourceMetadata).toEqual({ captureTableObjectId: 8 });
+    expect(materialized.snapshotComplete).toBe(false);
+    expect(materialized.syncData).toBe(persisted.syncData);
+  });
+
+  it('does not trust incompatible candidate mutations made through a cast', () => {
+    const persisted = candidate({ id: 'a', snapshotComplete: false });
+    const exposed = persisted.clone();
+    // Simulate callback code bypassing the read-only type boundary.
+    (exposed as any).snapshotComplete = true;
+
+    const materialized = materializeSourceTableResolution([persisted], {
+      compatibleTables: [],
+      incompatibleTables: [exposed],
+      newTableValues: { sourceMetadata: null }
+    }).incompatibleTables[0];
+
+    expect(exposed.snapshotComplete).toBe(true);
+    expect(materialized).toBe(persisted);
+    expect(materialized.snapshotComplete).toBe(false);
+    expect(materialized.syncData).toBe(persisted.syncData);
+  });
+
+  it('only reports the candidates that changed', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    const b = candidate({ id: 'b', sourceMetadata: { captureTableObjectId: 7 } });
+    const updates = diffSourceTableUpdates([a, b], resolution([a, b.withSourceMetadata({ captureTableObjectId: 9 })]));
+    expect(updates).toEqual([{ id: 'b', sourceMetadata: { captureTableObjectId: 9 } }]);
+  });
+
+  it('rejects a compatible table that was never a candidate', () => {
+    expect(() => diffSourceTableUpdates([candidate({ id: 'a' })], resolution([candidate({ id: 'ghost' })]))).toThrow(
+      /unknown candidate/
+    );
+  });
+});
+
+describe('SourceTable.clone', () => {
+  it('isolates mutable identity and metadata values', () => {
+    const persisted = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    const exposed = persisted.clone();
+
+    (exposed.ref as any).name = 'changed';
+    exposed.replicaIdColumns[0].name = 'changed';
+    (exposed.sourceMetadata as any).captureTableObjectId = 8;
+
+    expect(persisted.name).toBe('users');
+    expect(persisted.replicaIdColumns[0].name).toBe('id');
+    expect(persisted.sourceMetadata).toEqual({ captureTableObjectId: 7 });
+  });
+});
+
+describe('sourceTableIdEquals', () => {
+  it('compares string ids by value', () => {
+    expect(sourceTableIdEquals('table-1', 'table-1')).toBe(true);
+    expect(sourceTableIdEquals('table-1', 'table-2')).toBe(false);
+  });
+
+  it('compares BSON ObjectIds by value', () => {
+    const id = new bson.ObjectId();
+    const copy = new bson.ObjectId(id.toHexString());
+
+    expect(id).not.toBe(copy);
+    expect(sourceTableIdEquals(id, copy)).toBe(true);
+  });
+
+  it('does not mix string and BSON ObjectId representations', () => {
+    const id = new bson.ObjectId();
+
+    expect(sourceTableIdEquals(id.toHexString(), id)).toBe(false);
+    expect(sourceTableIdEquals(id, id.toHexString())).toBe(false);
+  });
+});