@powersync/service-module-postgres-storage 0.16.3 → 0.18.0

package/dist/storage/batch/PostgresBucketBatch.js

@@ -85,6 +85,7 @@ export class PostgresBucketBatch extends BaseObserver {
     }
     async resolveTables(options) {
         const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
+        const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
         const { connection_id, source } = options;
         const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
         const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
@@ -93,9 +94,10 @@ export class PostgresBucketBatch extends BaseObserver {
             type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
         }));
         return this.db.transaction(async (db) => {
-            let sourceTableRow;
+            // Find records that overlap by name or relation id.
+            let candidateRows;
             if (objectId != null) {
-                sourceTableRow = await db.sql `
+                candidateRows = await db.sql `
           SELECT
             *
           FROM
@@ -103,16 +105,19 @@ export class PostgresBucketBatch extends BaseObserver {
           WHERE
             group_id = ${{ type: 'int4', value: this.group_id }}
             AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } }}
-            AND schema_name = ${{ type: 'varchar', value: schema }}
-            AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
+            AND (
+              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } }}
+              OR (
+                schema_name = ${{ type: 'varchar', value: schema }}
+                AND table_name = ${{ type: 'varchar', value: table }}
+              )
+            )
         `
                     .decoded(models.SourceTable)
-                    .first();
+                    .rows();
             }
             else {
-                sourceTableRow = await db.sql `
+                candidateRows = await db.sql `
           SELECT
             *
           FROM
@@ -122,14 +127,38 @@ export class PostgresBucketBatch extends BaseObserver {
             AND connection_id = ${{ type: 'int4', value: connection_id }}
             AND schema_name = ${{ type: 'varchar', value: schema }}
             AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
         `
                     .decoded(models.SourceTable)
-                    .first();
+                    .rows();
+            }
+            const candidateTables = candidateRows.map((row) => {
+                const table = this.sourceTableFromRow(row, connectionTag, syncRules);
+                table.storeCurrentData = sendsCompleteRows !== true;
+                return table;
+            });
+            const candidates = candidateTables.map((table) => table.clone());
+            const resolution = await reconcile({ source, candidates });
+            storage.validateSourceTableCandidateResolution(candidates, resolution);
+            for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
+                await db.sql `
+          UPDATE source_tables
+          SET
+            source_metadata = ${{ type: 'jsonb', value: sourceMetadata ?? null }}
+          WHERE
+            id = ${{ type: 'varchar', value: id.toString() }}
+        `.execute();
+            }
+            const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
+            const compatibleTables = materializedResolution.compatibleTables;
+            // Keep one record, preferring one that has already been snapshotted.
+            const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
+            let sourceTable;
+            if (reuse != null) {
+                sourceTable = reuse;
             }
-            if (sourceTableRow == null) {
+            else {
                 const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
-                sourceTableRow = await db.sql `
+                const insertedRow = await db.sql `
           INSERT INTO
             source_tables (
               id,
@@ -138,7 +167,8 @@ export class PostgresBucketBatch extends BaseObserver {
               relation_id,
               schema_name,
               table_name,
-              replica_id_columns
+              replica_id_columns,
+              source_metadata
             )
           VALUES
             (
@@ -148,90 +178,25 @@ export class PostgresBucketBatch extends BaseObserver {
               ${{ type: 'jsonb', value: { object_id: objectId } }},
               ${{ type: 'varchar', value: schema }},
               ${{ type: 'varchar', value: table }},
-              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
+              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }},
+              ${{ type: 'jsonb', value: resolution.newTableValues.sourceMetadata ?? null }}
             )
           RETURNING
             *
         `
                     .decoded(models.SourceTable)
                     .first();
+                sourceTable = this.sourceTableFromRow(insertedRow, connectionTag, syncRules);
+                sourceTable.storeCurrentData = sendsCompleteRows !== true;
             }
-            const sourceTable = new storage.SourceTable({
-                id: sourceTableRow.id,
-                ref: source,
-                objectId,
-                replicaIdColumns,
-                snapshotComplete: sourceTableRow.snapshot_done ?? true,
-                ...syncRules.getMatchingSources(source)
-            });
-            if (!sourceTable.snapshotComplete) {
-                sourceTable.snapshotStatus = {
-                    totalEstimatedCount: Number(sourceTableRow.snapshot_total_estimated_count ?? -1n),
-                    replicatedCount: Number(sourceTableRow.snapshot_replicated_count ?? 0n),
-                    lastKey: sourceTableRow.snapshot_last_key
-                };
-            }
-            sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
-            sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
-            sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
-            sourceTable.storeCurrentData = sendsCompleteRows !== true;
-            let truncatedTables = [];
-            if (objectId != null) {
-                truncatedTables = await db.sql `
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow.id }}
-            AND (
-              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } }}
-              OR (
-                schema_name = ${{ type: 'varchar', value: schema }}
-                AND table_name = ${{ type: 'varchar', value: table }}
-              )
-            )
-        `
-                    .decoded(models.SourceTable)
-                    .rows();
-            }
-            else {
-                truncatedTables = await db.sql `
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow.id }}
-            AND (
-              schema_name = ${{ type: 'varchar', value: schema }}
-              AND table_name = ${{ type: 'varchar', value: table }}
-            )
-        `
-                    .decoded(models.SourceTable)
-                    .rows();
-            }
+            const dropTables = [
+                ...materializedResolution.incompatibleTables,
+                // PostgreSQL storage only keeps one SourceTable per physical table.
+                ...compatibleTables.filter((candidate) => !storage.sourceTableIdEquals(candidate.id, sourceTable.id))
+            ];
             return {
                 tables: [sourceTable],
-                dropTables: truncatedTables.map((doc) => {
-                    const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
-                    const dropTable = new storage.SourceTable({
-                        id: doc.id,
-                        ref,
-                        objectId: doc.relation_id?.object_id ?? 0,
-                        replicaIdColumns: doc.replica_id_columns?.map((c) => ({ name: c.name, typeId: c.typeId, type: c.type })) ?? [],
-                        snapshotComplete: doc.snapshot_done ?? true,
-                        ...syncRules.getMatchingSources(ref)
-                    });
-                    dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
-                    dropTable.syncData = dropTable.bucketDataSources.length > 0;
-                    dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
-                    return dropTable;
-                })
+                dropTables
             };
         });
     }
@@ -258,8 +223,9 @@ export class PostgresBucketBatch extends BaseObserver {
             id: row.id,
             ref,
             objectId: row.relation_id?.object_id,
-            replicaIdColumns: row.replica_id_columns?.map((c) => ({ name: c.name, typeId: c.typeId, type: c.type })) ?? [],
+            replicaIdColumns: row.replica_id_columns?.map((column) => ({ name: column.name, typeId: column.type_oid, type: column.type })) ?? [],
             snapshotComplete: row.snapshot_done ?? true,
+            sourceMetadata: row.source_metadata,
             ...syncRules.getMatchingSources(ref)
         });
         if (!sourceTable.snapshotComplete) {
@@ -375,12 +341,11 @@ export class PostgresBucketBatch extends BaseObserver {
                 }
                 const { flushedAny } = await persistedBatch.flush(db);
                 clearedError = flushedAny && !this.clearedError;
-                if (clearedError) {
-                    // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
-                    await this.clearError(db);
-                }
             });
             if (clearedError) {
+                // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
+                // Cleared outside the replication transaction - see flushInner.
+                await this.clearError();
                 this.clearedError = true;
             }
         }
@@ -437,6 +402,10 @@ export class PostgresBucketBatch extends BaseObserver {
             return this.getLastOpIdSequence(db);
         });
         if (clearedError) {
+            // Clear the error outside the replication transaction (plain autocommit update,
+            // like the keepalive), to avoid serialization conflicts on the sync_rules row
+            // when multiple writers flush concurrently.
+            await this.clearError();
             this.clearedError = true;
         }
         // null if done, set if we need another flush
@@ -454,136 +423,143 @@ export class PostgresBucketBatch extends BaseObserver {
         await this.flush();
         const now = new Date().toISOString();
         const persisted_op = this.persisted_op ?? null;
-        const result = await this.db.sql `
-      WITH
-        selected AS (
-          SELECT
-            id,
-            state,
-            last_checkpoint,
-            last_checkpoint_lsn,
-            snapshot_done,
-            no_checkpoint_before,
-            keepalive_op,
-            (
-              snapshot_done = TRUE
-              AND (
-                last_checkpoint_lsn IS NULL
-                OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
-              )
+        // Transaction failures restart replication from the last durable checkpoint.
+        const result = await this.db.transaction(async (db) => {
+            const checkpoint = await db.sql `
+        WITH
+          selected AS (
+            SELECT
+              id,
+              state,
+              last_checkpoint,
+              last_checkpoint_lsn,
+              snapshot_done,
+              no_checkpoint_before,
+              keepalive_op,
+              (
+                snapshot_done = TRUE
+                AND (
+                  last_checkpoint_lsn IS NULL
+                  OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
+                )
+                AND (
+                  no_checkpoint_before IS NULL
+                  OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
+                )
+              ) AS can_checkpoint
+            FROM
+              sync_rules
+            WHERE
+              id = ${{ type: 'int4', value: this.group_id }}
+            FOR UPDATE
+          ),
+          computed AS (
+            SELECT
+              selected.*,
+              CASE
+                WHEN selected.can_checkpoint THEN GREATEST(
+                  selected.last_checkpoint,
+                  ${{ type: 'int8', value: persisted_op }},
+                  selected.keepalive_op,
+                  0
+                )
+                ELSE selected.last_checkpoint
+              END AS new_last_checkpoint,
+              CASE
+                WHEN selected.can_checkpoint THEN NULL
+                ELSE GREATEST(
+                  selected.keepalive_op,
+                  ${{ type: 'int8', value: persisted_op }},
+                  0
+                )
+              END AS new_keepalive_op
+            FROM
+              selected
+          ),
+          updated AS (
+            UPDATE sync_rules AS sr
+            SET
+              last_checkpoint_lsn = CASE
+                WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
+                ELSE sr.last_checkpoint_lsn
+              END,
+              last_checkpoint_ts = CASE
+                WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
+                ELSE sr.last_checkpoint_ts
+              END,
+              last_keepalive_ts = ${{ type: 1184, value: now }},
+              last_fatal_error = CASE
+                WHEN computed.can_checkpoint THEN NULL
+                ELSE sr.last_fatal_error
+              END,
+              keepalive_op = computed.new_keepalive_op,
+              last_checkpoint = computed.new_last_checkpoint,
+              snapshot_lsn = CASE
+                WHEN computed.can_checkpoint THEN NULL
+                ELSE sr.snapshot_lsn
+              END
+            FROM
+              computed
+            WHERE
+              sr.id = computed.id
               AND (
-                no_checkpoint_before IS NULL
-                OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
+                sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
+                OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
+                OR ${{ type: 'bool', value: createEmptyCheckpoints }}
               )
-            ) AS can_checkpoint
-          FROM
-            sync_rules
-          WHERE
-            id = ${{ type: 'int4', value: this.group_id }}
-          FOR UPDATE
-        ),
-        computed AS (
-          SELECT
-            selected.*,
-            CASE
-              WHEN selected.can_checkpoint THEN GREATEST(
-                selected.last_checkpoint,
-                ${{ type: 'int8', value: persisted_op }},
-                selected.keepalive_op,
-                0
-              )
-              ELSE selected.last_checkpoint
-            END AS new_last_checkpoint,
-            CASE
-              WHEN selected.can_checkpoint THEN NULL
-              ELSE GREATEST(
-                selected.keepalive_op,
-                ${{ type: 'int8', value: persisted_op }},
-                0
-              )
-            END AS new_keepalive_op
-          FROM
-            selected
-        ),
-        updated AS (
-          UPDATE sync_rules AS sr
-          SET
-            last_checkpoint_lsn = CASE
-              WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
-              ELSE sr.last_checkpoint_lsn
-            END,
-            last_checkpoint_ts = CASE
-              WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
-              ELSE sr.last_checkpoint_ts
-            END,
-            last_keepalive_ts = ${{ type: 1184, value: now }},
-            last_fatal_error = CASE
-              WHEN computed.can_checkpoint THEN NULL
-              ELSE sr.last_fatal_error
-            END,
-            keepalive_op = computed.new_keepalive_op,
-            last_checkpoint = computed.new_last_checkpoint,
-            snapshot_lsn = CASE
-              WHEN computed.can_checkpoint THEN NULL
-              ELSE sr.snapshot_lsn
-            END
-          FROM
-            computed
-          WHERE
-            sr.id = computed.id
-            AND (
-              sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
-              OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
-              OR ${{ type: 'bool', value: createEmptyCheckpoints }}
-            )
-          RETURNING
-            sr.id,
-            sr.state,
-            sr.last_checkpoint,
-            sr.last_checkpoint_lsn,
-            sr.snapshot_done,
-            sr.no_checkpoint_before,
-            computed.can_checkpoint,
-            computed.keepalive_op,
-            computed.new_last_checkpoint
-        )
-      SELECT
-        id,
-        state,
-        last_checkpoint,
-        last_checkpoint_lsn,
-        snapshot_done,
-        no_checkpoint_before,
-        can_checkpoint,
-        keepalive_op,
-        new_last_checkpoint,
-        TRUE AS created_checkpoint
-      FROM
-        updated
-      UNION ALL
-      SELECT
-        id,
-        state,
-        new_last_checkpoint AS last_checkpoint,
-        last_checkpoint_lsn,
-        snapshot_done,
-        no_checkpoint_before,
-        can_checkpoint,
-        keepalive_op,
-        new_last_checkpoint,
-        FALSE AS created_checkpoint
-      FROM
-        computed
-      WHERE
-        NOT EXISTS (
-          SELECT
-            1
-          FROM
-            updated
-        )
-    `
-            .decoded(CheckpointWithStatus)
-            .first();
+            RETURNING
+              sr.id,
+              sr.state,
+              sr.last_checkpoint,
+              sr.last_checkpoint_lsn,
+              sr.snapshot_done,
+              sr.no_checkpoint_before,
+              computed.can_checkpoint,
+              computed.keepalive_op,
+              computed.new_last_checkpoint
+          )
+        SELECT
+          id,
+          state,
+          last_checkpoint,
+          last_checkpoint_lsn,
+          snapshot_done,
+          no_checkpoint_before,
+          can_checkpoint,
+          keepalive_op,
+          new_last_checkpoint,
+          TRUE AS created_checkpoint
+        FROM
+          updated
+        UNION ALL
+        SELECT
+          id,
+          state,
+          new_last_checkpoint AS last_checkpoint,
+          last_checkpoint_lsn,
+          snapshot_done,
+          no_checkpoint_before,
+          can_checkpoint,
+          keepalive_op,
+          new_last_checkpoint,
+          FALSE AS created_checkpoint
+        FROM
+          computed
+        WHERE
+          NOT EXISTS (
+            SELECT
+              1
+            FROM
+              updated
+          )
+      `
+                .decoded(CheckpointWithStatus)
+                .first();
+            if (checkpoint?.can_checkpoint && checkpoint.state == storage.SyncRuleState.ACTIVE) {
+                await notifySyncRulesUpdate(db, checkpoint);
+            }
+            return checkpoint;
+        });
         if (result == null) {
             throw new ReplicationAssertionError('Failed to update sync_rules during checkpoint');
         }
@@ -607,10 +583,8 @@ export class PostgresBucketBatch extends BaseObserver {
                 });
             }
         }
-        await this.autoActivate(lsn);
-        await notifySyncRulesUpdate(this.db, {
+        await this.autoActivate(lsn, {
             id: result.id,
-            state: result.state,
             last_checkpoint: result.last_checkpoint,
             last_checkpoint_lsn: result.last_checkpoint_lsn
         });
@@ -872,10 +846,8 @@ export class PostgresBucketBatch extends BaseObserver {
                 didFlush ||= flushedAny;
             }
         }
+        // The error itself is cleared by the caller, outside the replication transaction - see flushInner.
         const clearedError = didFlush && !this.clearedError;
-        if (clearedError) {
-            await this.clearError(db);
-        }
         // Don't return empty batches
         return {
             resumeBatch: resumeBatch?.batch.length ? resumeBatch : null,
@@ -1101,13 +1073,12 @@ export class PostgresBucketBatch extends BaseObserver {
      *
      * Called on new commits.
      */
-    async autoActivate(lsn) {
+    async autoActivate(lsn, checkpoint) {
         if (!this.needsActivation) {
             // Already activated
             return;
         }
-        let didActivate = false;
-        await this.db.transaction(async (db) => {
+        const activationResult = await this.db.transaction(async (db) => {
             const syncRulesRow = await db.sql `
         SELECT
           state,
@@ -1139,14 +1110,18 @@ export class PostgresBucketBatch extends BaseObserver {
             )
             AND id != ${{ type: 'int4', value: this.group_id }}
         `.execute();
-                didActivate = true;
-                this.needsActivation = false;
+                await notifySyncRulesUpdate(db, checkpoint);
+                return 'activated';
             }
             else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) {
-                this.needsActivation = false;
+                return 'not-processing';
             }
+            return 'pending';
         });
-        if (didActivate) {
+        if (activationResult != 'pending') {
+            this.needsActivation = false;
+        }
+        if (activationResult == 'activated') {
             this.logger.info(`Activated new replication stream at ${lsn}`);
         }
     }
@@ -1224,11 +1199,13 @@ export class PostgresBucketBatch extends BaseObserver {
  * active checkpoint has been updated.
  */
 export const notifySyncRulesUpdate = async (db, update) => {
-    if (update.state != storage.SyncRuleState.ACTIVE) {
-        return;
-    }
-    await db.query({
-        statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'`
-    });
+    const payload = models.ActiveCheckpointNotification.encode({ active_checkpoint: update });
+    await db.sql `
+    SELECT
+      pg_notify (
+        ${{ type: 'varchar', value: NOTIFICATION_CHANNEL }},
+        ${{ type: 'varchar', value: payload }}
+      )
+  `.execute();
 };
 //# sourceMappingURL=PostgresBucketBatch.js.map