@powersync/service-module-postgres-storage 0.17.0 → 0.19.0

package/src/storage/batch/PostgresBucketBatch.ts

@@ -19,6 +19,7 @@ import {
   utils
 } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
+import { orNull } from '@powersync/service-types';
 import * as timers from 'timers/promises';
 import * as t from 'ts-codec';
 import * as uuid from 'uuid';
@@ -64,10 +65,10 @@ const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(
 const CheckpointWithStatus = StatefulCheckpoint.and(
   t.object({
     snapshot_done: t.boolean,
-    no_checkpoint_before: t.string.or(t.Null),
+    no_checkpoint_before: orNull(t.string),
     can_checkpoint: t.boolean,
-    keepalive_op: bigint.or(t.Null),
-    new_last_checkpoint: bigint.or(t.Null),
+    keepalive_op: orNull(bigint),
+    new_last_checkpoint: orNull(bigint),
     created_checkpoint: t.boolean
   })
 );
@@ -144,6 +145,7 @@ export class PostgresBucketBatch
 
   async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
     const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
+    const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
     const { connection_id, source } = options;
     const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
 
@@ -154,9 +156,10 @@ export class PostgresBucketBatch
     }));
 
     return this.db.transaction(async (db) => {
-      let sourceTableRow: SourceTableDecoded | null;
+      // Find records that overlap by name or relation id.
+      let candidateRows: SourceTableDecoded[];
       if (objectId != null) {
-        sourceTableRow = await db.sql`
+        candidateRows = await db.sql`
           SELECT
             *
           FROM
@@ -164,15 +167,18 @@ export class PostgresBucketBatch
           WHERE
             group_id = ${{ type: 'int4', value: this.group_id }}
             AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-            AND schema_name = ${{ type: 'varchar', value: schema }}
-            AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
+            AND (
+              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
+              OR (
+                schema_name = ${{ type: 'varchar', value: schema }}
+                AND table_name = ${{ type: 'varchar', value: table }}
+              )
+            )
         `
           .decoded(models.SourceTable)
-          .first();
+          .rows();
       } else {
-        sourceTableRow = await db.sql`
+        candidateRows = await db.sql`
           SELECT
             *
           FROM
@@ -182,15 +188,42 @@ export class PostgresBucketBatch
             AND connection_id = ${{ type: 'int4', value: connection_id }}
             AND schema_name = ${{ type: 'varchar', value: schema }}
             AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
         `
           .decoded(models.SourceTable)
-          .first();
+          .rows();
       }
 
-      if (sourceTableRow == null) {
+      const candidateTables = candidateRows.map((row) => {
+        const table = this.sourceTableFromRow(row, connectionTag, syncRules);
+        table.storeCurrentData = sendsCompleteRows !== true;
+        return table;
+      });
+      const candidates = candidateTables.map((table) => table.clone());
+      const resolution = await reconcile({ source, candidates });
+      storage.validateSourceTableCandidateResolution(candidates, resolution);
+
+      for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
+        await db.sql`
+          UPDATE source_tables
+          SET
+            source_metadata = ${{ type: 'jsonb', value: sourceMetadata ?? null }}
+          WHERE
+            id = ${{ type: 'varchar', value: id.toString() }}
+        `.execute();
+      }
+
+      const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
+      const compatibleTables = materializedResolution.compatibleTables;
+
+      // Keep one record, preferring one that has already been snapshotted.
+      const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
+
+      let sourceTable: storage.SourceTable;
+      if (reuse != null) {
+        sourceTable = reuse;
+      } else {
         const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
-        sourceTableRow = await db.sql`
+        const insertedRow = await db.sql`
           INSERT INTO
             source_tables (
               id,
@@ -199,7 +232,8 @@ export class PostgresBucketBatch
               relation_id,
               schema_name,
               table_name,
-              replica_id_columns
+              replica_id_columns,
+              source_metadata
             )
           VALUES
             (
@@ -209,95 +243,27 @@ export class PostgresBucketBatch
               ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
               ${{ type: 'varchar', value: schema }},
               ${{ type: 'varchar', value: table }},
-              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
+              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }},
+              ${{ type: 'jsonb', value: resolution.newTableValues.sourceMetadata ?? null }}
             )
           RETURNING
             *
         `
           .decoded(models.SourceTable)
           .first();
+        sourceTable = this.sourceTableFromRow(insertedRow!, connectionTag, syncRules);
+        sourceTable.storeCurrentData = sendsCompleteRows !== true;
       }
 
-      const sourceTable = new storage.SourceTable({
-        id: sourceTableRow!.id,
-        ref: source,
-        objectId,
-        replicaIdColumns,
-        snapshotComplete: sourceTableRow!.snapshot_done ?? true,
-        ...syncRules.getMatchingSources(source)
-      });
-      if (!sourceTable.snapshotComplete) {
-        sourceTable.snapshotStatus = {
-          totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
-          replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
-          lastKey: sourceTableRow!.snapshot_last_key
-        };
-      }
-      sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
-      sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
-      sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
-      sourceTable.storeCurrentData = sendsCompleteRows !== true;
-
-      let truncatedTables: SourceTableDecoded[] = [];
-      if (objectId != null) {
-        truncatedTables = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-            AND (
-              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-              OR (
-                schema_name = ${{ type: 'varchar', value: schema }}
-                AND table_name = ${{ type: 'varchar', value: table }}
-              )
-            )
-        `
-          .decoded(models.SourceTable)
-          .rows();
-      } else {
-        truncatedTables = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-            AND (
-              schema_name = ${{ type: 'varchar', value: schema }}
-              AND table_name = ${{ type: 'varchar', value: table }}
-            )
-        `
-          .decoded(models.SourceTable)
-          .rows();
-      }
+      const dropTables = [
+        ...materializedResolution.incompatibleTables,
+        // PostgreSQL storage only keeps one SourceTable per physical table.
+        ...compatibleTables.filter((candidate) => !storage.sourceTableIdEquals(candidate.id, sourceTable.id))
+      ];
 
       return {
         tables: [sourceTable],
-        dropTables: truncatedTables.map((doc) => {
-          const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
-          const dropTable = new storage.SourceTable({
-            id: doc.id,
-            ref,
-            objectId: doc.relation_id?.object_id ?? 0,
-            replicaIdColumns:
-              doc.replica_id_columns?.map(
-                (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
-              ) ?? [],
-            snapshotComplete: doc.snapshot_done ?? true,
-            ...syncRules.getMatchingSources(ref)
-          });
-          dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
-          dropTable.syncData = dropTable.bucketDataSources.length > 0;
-          dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
-          return dropTable;
-        })
+        dropTables
       };
     });
   }
@@ -334,9 +300,10 @@ export class PostgresBucketBatch
       objectId: row.relation_id?.object_id,
       replicaIdColumns:
         row.replica_id_columns?.map(
-          (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
+          (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
         ) ?? [],
       snapshotComplete: row.snapshot_done ?? true,
+      sourceMetadata: row.source_metadata,
       ...syncRules.getMatchingSources(ref)
     });
 
@@ -357,7 +324,9 @@ export class PostgresBucketBatch
     // TODO maybe share with abstract class
     const { after, before, sourceTable, tag } = record;
     const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
-    // Only the table designated as event carrier by resolveTables may fire events.
+    // PostgreSQL storage resolves each physical table to one SourceTable. syncEvent records whether
+    // that table matches any event definition; each matching definition is evaluated once here and
+    // may produce multiple payloads from its matching queries.
     if (sourceTable.syncEvent) {
       for (const event of this.getTableEvents(sourceTable)) {
         this.iterateListeners((cb) =>
@@ -1368,10 +1337,10 @@ export class PostgresBucketBatch
   }
 
   /**
-   * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
+   * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable}
    * TODO maybe share this with an abstract class
    */
-  protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
+  protected getTableEvents(table: storage.SourceTable): sync_rules.HydratedEventDescriptor[] {
     return this.sync_rules.eventDescriptors.filter((evt) =>
       [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
     );

package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts

@@ -21,8 +21,8 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     return this._mode;
   }
 
-  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
-    this._mode = mode;
+  setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this._mode = config.mode;
   }
 
   async batchCreateCustomWriteCheckpoints(

package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts

@@ -14,7 +14,8 @@ export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncCon
       compiled_plan: row.sync_plan,
       replicationStreamName: row.slot_name,
       storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
-      syncConfigState: row.state as storage.SyncRuleState
+      syncConfigState: row.state as storage.SyncRuleState,
+      version_label: row.version_label ?? undefined
     });
   }
 
@@ -61,6 +62,12 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
       state: row.state as storage.SyncRuleState,
       storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
     });
+    if (this.storageVersion >= storage.STORAGE_VERSION_4) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for PostgreSQL storage`
+      );
+    }
     this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
   }
 

package/src/types/models/ActiveCheckpoint.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
 
@@ -7,8 +8,8 @@ import { bigint, pgwire_number } from '../codecs.js';
  */
 export const ActiveCheckpoint = t.object({
   id: pgwire_number,
-  last_checkpoint: t.Null.or(bigint),
-  last_checkpoint_lsn: t.Null.or(t.string)
+  last_checkpoint: orNull(bigint),
+  last_checkpoint_lsn: orNull(t.string)
 });
 
 export type ActiveCheckpoint = t.Encoded<typeof ActiveCheckpoint>;

package/src/types/models/BucketData.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
 
@@ -13,13 +14,13 @@ export const BucketData = t.object({
   bucket_name: t.string,
   op_id: bigint,
   op: t.Enum(OpType),
-  source_table: t.Null.or(t.string),
-  source_key: t.Null.or(hexBuffer),
-  table_name: t.string.or(t.Null),
-  row_id: t.string.or(t.Null),
+  source_table: orNull(t.string),
+  source_key: orNull(hexBuffer),
+  table_name: orNull(t.string),
+  row_id: orNull(t.string),
   checksum: bigint,
-  data: t.Null.or(t.string),
-  target_op: t.Null.or(bigint)
+  data: orNull(t.string),
+  target_op: orNull(bigint)
 });
 
 export type BucketData = t.Encoded<typeof BucketData>;

package/src/types/models/CurrentData.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
 
@@ -26,7 +27,7 @@ export const V3CurrentData = t.object({
   lookups: t.array(hexBuffer),
   source_key: hexBuffer,
   source_table: t.string,
-  pending_delete: t.Null.or(bigint)
+  pending_delete: orNull(bigint)
 });
 
 export type V1CurrentData = t.Encoded<typeof V1CurrentData>;

package/src/types/models/SdkReporting.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 
@@ -11,12 +12,11 @@ export type Sdks = t.Encoded<typeof Sdks>;
 
 export const SdkReporting = t.object({
   users: bigint,
-  sdks: t
-    .object({
+  sdks: orNull(
+    t.object({
       data: jsonb<Sdks[]>(t.array(Sdks))
     })
-    .optional()
-    .or(t.Null)
+  ).optional()
 });
 
 export type SdkReporting = t.Encoded<typeof SdkReporting>;

package/src/types/models/SourceTable.ts

@@ -1,3 +1,5 @@
+import { JsonValue } from '@powersync/service-core';
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, jsonb, jsonb_raw, pgwire_number } from '../codecs.js';
 
@@ -14,21 +16,25 @@ export const ColumnDescriptor = t.object({
   /**
    *  Some data sources have a type id that can be used to identify the type of the column
    */
-  typeId: t.number.optional()
+  type_oid: t.number.optional()
 });
 
 export const SourceTable = t.object({
   id: t.string,
   group_id: pgwire_number,
   connection_id: bigint,
-  relation_id: t.Null.or(jsonb_raw<StoredRelationId>()),
+  relation_id: orNull(jsonb_raw<StoredRelationId>()),
   schema_name: t.string,
   table_name: t.string,
-  replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))),
+  replica_id_columns: orNull(jsonb(t.array(ColumnDescriptor))),
   snapshot_done: t.boolean,
-  snapshot_total_estimated_count: t.Null.or(bigint),
-  snapshot_replicated_count: t.Null.or(bigint),
-  snapshot_last_key: t.Null.or(hexBuffer)
+  snapshot_total_estimated_count: orNull(bigint),
+  snapshot_replicated_count: orNull(bigint),
+  snapshot_last_key: orNull(hexBuffer),
+  /**
+   * Source-specific metadata. Null for legacy records.
+   */
+  source_metadata: orNull(jsonb_raw<JsonValue>())
 });
 
 export type SourceTable = t.Encoded<typeof SourceTable>;

package/src/types/models/SyncRules.ts

@@ -1,5 +1,5 @@
 import { framework, storage } from '@powersync/service-core';
-import { ReplicationError } from '@powersync/service-types';
+import { orNull, ReplicationError } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
 import { jsonContainerObject } from './json.js';
@@ -16,42 +16,43 @@ export const SyncRules = t.object({
   /**
    * May be set if snapshot_done = false, if the replication stream requires it.
    */
-  snapshot_lsn: t.Null.or(t.string),
+  snapshot_lsn: orNull(t.string),
   /**
    * The last consistent checkpoint.
    *
    * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
    */
-  last_checkpoint: t.Null.or(bigint),
+  last_checkpoint: orNull(bigint),
   /**
    * The LSN associated with the last consistent checkpoint.
    */
-  last_checkpoint_lsn: t.Null.or(t.string),
+  last_checkpoint_lsn: orNull(t.string),
   /**
    * If set, no new checkpoints may be created < this value.
    */
-  no_checkpoint_before: t.Null.or(t.string),
+  no_checkpoint_before: orNull(t.string),
   slot_name: t.string,
   /**
    * Last time we persisted a checkpoint.
    *
    * This may be old if no data is incoming.
    */
-  last_checkpoint_ts: t.Null.or(framework.codecs.date),
+  last_checkpoint_ts: orNull(framework.codecs.date),
   /**
    * Last time we persisted a checkpoint or keepalive.
    *
    * This should stay fairly current while replicating.
    */
-  last_keepalive_ts: t.Null.or(framework.codecs.date),
+  last_keepalive_ts: orNull(framework.codecs.date),
   /**
    * If an error is stopping replication, it will be stored here.
    */
-  last_fatal_error: t.Null.or(t.string),
-  keepalive_op: t.Null.or(bigint),
-  storage_version: t.Null.or(pgwire_number).optional(),
+  last_fatal_error: orNull(t.string),
+  keepalive_op: orNull(bigint),
+  storage_version: orNull(pgwire_number).optional(),
+  version_label: orNull(t.string).optional(),
   content: t.string,
-  sync_plan: t.Null.or(
+  sync_plan: orNull(
     jsonContainerObject(
       t.object({
         plan: t.any,

package/src/types/models/WriteCheckpoint.ts

@@ -1,4 +1,5 @@
 import { framework } from '@powersync/service-core';
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 
@@ -6,7 +7,7 @@ export const WriteCheckpoint = t.object({
   user_id: t.string,
   lsns: jsonb(t.record(t.string)),
   write_checkpoint: bigint,
-  checkpoint_requested_at: t.Null.or(framework.codecs.date)
+  checkpoint_requested_at: orNull(framework.codecs.date)
 });
 
 export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
@@ -16,7 +17,7 @@ export const CustomWriteCheckpoint = t.object({
   user_id: t.string,
   write_checkpoint: bigint,
   sync_rules_id: bigint,
-  checkpoint_requested_at: t.Null.or(framework.codecs.date)
+  checkpoint_requested_at: orNull(framework.codecs.date)
 });
 
 export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;