@powersync/service-module-postgres-storage 0.14.0 → 0.16.0

package/src/storage/PostgresSyncRulesStorage.ts

@@ -22,7 +22,6 @@ import {
 import { JSONBig } from '@powersync/service-jsonbig';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
-import * as uuid from 'uuid';
 import { bigint, BIGINT_MAX } from '../types/codecs.js';
 import { models, RequiredOperationBatchLimits } from '../types/types.js';
 import { replicaIdToSubkey } from '../utils/bson.js';
@@ -32,7 +31,6 @@ import * as framework from '@powersync/lib-services-framework';
 import { StatementParam } from '@powersync/service-jpgwire';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import * as t from 'ts-codec';
-import { SourceTableDecoded, StoredRelationId } from '../types/models/SourceTable.js';
 import { pick } from '../utils/ts-codec.js';
 import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
 import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
@@ -43,7 +41,7 @@ import { PostgresCompactor } from './PostgresCompactor.js';
 export type PostgresSyncRulesStorageOptions = {
   factory: PostgresBucketStorageFactory;
   db: lib_postgres.DatabaseClient;
-  sync_rules: storage.PersistedSyncRulesContent;
+  replicationStream: storage.PersistedReplicationStream;
   write_checkpoint_mode?: storage.WriteCheckpointMode;
   batchLimits: RequiredOperationBatchLimits;
 };
@@ -54,9 +52,10 @@ export class PostgresSyncRulesStorage
 {
   [framework.DO_NOT_LOG] = true;
 
-  public readonly group_id: number;
-  public readonly sync_rules: storage.PersistedSyncRulesContent;
-  public readonly slot_name: string;
+  public readonly replicationStreamId: number;
+  public readonly replicationStream: storage.PersistedReplicationStream;
+  public readonly sync_rules: storage.PersistedSyncConfigContent;
+  public readonly replicationStreamName: string;
   public readonly factory: PostgresBucketStorageFactory;
   public readonly storageConfig: StorageVersionConfig;
   public readonly logger: framework.Logger;
@@ -67,22 +66,27 @@ export class PostgresSyncRulesStorage
   protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
   private readonly currentDataStore: PostgresCurrentDataStore;
 
-  //   TODO we might be able to share this in an abstract class
-  private parsedSyncRulesCache:
-    | { parsed: sync_rules.HydratedSyncRules; options: storage.ParseSyncRulesOptions }
-    | undefined;
+  /**
+   * Canonical parsed sync config sets, keyed by defaultSchema. Entries are never evicted,
+   * so each parse options value maps to exactly one parsed set for the lifetime of this
+   * storage instance.
+   *
+   * TODO we might be able to share this in an abstract class
+   */
+  private readonly parsedSyncConfigSets = new Map<string, storage.ParsedSyncConfigSet>();
   private _checksumCache: storage.ChecksumCache | undefined;
 
   constructor(protected options: PostgresSyncRulesStorageOptions) {
     super();
-    this.group_id = options.sync_rules.id;
+    this.replicationStream = options.replicationStream;
+    this.replicationStreamId = options.replicationStream.replicationStreamId;
     this.db = options.db;
-    this.sync_rules = options.sync_rules;
-    this.slot_name = options.sync_rules.slot_name;
+    this.sync_rules = options.replicationStream.syncConfigContent[0];
+    this.replicationStreamName = options.replicationStream.replicationStreamName;
     this.factory = options.factory;
-    this.storageConfig = options.sync_rules.getStorageConfig();
+    this.storageConfig = options.replicationStream.getStorageConfig();
     this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
-    this.logger = options.sync_rules.logger;
+    this.logger = options.replicationStream.logger;
 
     this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
       db: this.db,
@@ -108,18 +112,17 @@ export class PostgresSyncRulesStorage
     return this.writeCheckpointAPI.writeCheckpointMode;
   }
 
-  //   TODO we might be able to share this in an abstract class
-  getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncRules {
-    const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
-    /**
-     * Check if the cached sync config, if present, had the same options.
-     * Parse sync config if the options are different or if there is no cached value.
-     */
-    if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
-      this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options };
+  getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
+    let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
+    if (parsed == null) {
+      parsed = this.replicationStream.parsed(options);
+      this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
     }
+    return parsed;
+  }
 
-    return this.parsedSyncRulesCache!.parsed;
+  getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig {
+    return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
   }
 
   async reportError(e: any): Promise<void> {
@@ -129,7 +132,7 @@ export class PostgresSyncRulesStorage
       SET
         last_fatal_error = ${{ type: 'varchar', value: message }}
       WHERE
-        id = ${{ type: 'int4', value: this.group_id }};
+        id = ${{ type: 'int4', value: this.replicationStreamId }};
     `.execute();
   }
 
@@ -141,7 +144,7 @@ export class PostgresSyncRulesStorage
       maxOpId = checkpoint.checkpoint;
     }
 
-    return new PostgresCompactor(this.db, this.group_id, {
+    return new PostgresCompactor(this.db, this.replicationStreamId, {
       ...options,
       maxOpId,
       logger: this.logger
@@ -156,7 +159,7 @@ export class PostgresSyncRulesStorage
   lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
     return this.writeCheckpointAPI.lastWriteCheckpoint({
       ...filters,
-      sync_rules_id: this.group_id
+      sync_rules_id: this.replicationStreamId
     });
   }
 
@@ -176,7 +179,7 @@ export class PostgresSyncRulesStorage
       FROM
         sync_rules
       WHERE
-        id = ${{ type: 'int4', value: this.group_id }}
+        id = ${{ type: 'int4', value: this.replicationStreamId }}
     `
       .decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn']))
       .first();
@@ -188,168 +191,6 @@ export class PostgresSyncRulesStorage
     );
   }
 
-  async resolveTable(options: storage.ResolveTableOptions): Promise<storage.ResolveTableResult> {
-    const { group_id, connection_id, connection_tag, entity_descriptor } = options;
-
-    const { schema, name: table, objectId, replicaIdColumns } = entity_descriptor;
-
-    const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
-      name: column.name,
-      type: column.type,
-      // The PGWire returns this as a BigInt. We want to store this as JSONB
-      type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
-    }));
-    return this.db.transaction(async (db) => {
-      let sourceTableRow: SourceTableDecoded | null;
-      if (objectId != null) {
-        sourceTableRow = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-            AND schema_name = ${{ type: 'varchar', value: schema }}
-            AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
-        `
-          .decoded(models.SourceTable)
-          .first();
-      } else {
-        sourceTableRow = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND schema_name = ${{ type: 'varchar', value: schema }}
-            AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
-        `
-          .decoded(models.SourceTable)
-          .first();
-      }
-
-      if (sourceTableRow == null) {
-        const row = await db.sql`
-          INSERT INTO
-            source_tables (
-              id,
-              group_id,
-              connection_id,
-              relation_id,
-              schema_name,
-              table_name,
-              replica_id_columns
-            )
-          VALUES
-            (
-              ${{ type: 'varchar', value: uuid.v4() }},
-              ${{ type: 'int4', value: group_id }},
-              ${{ type: 'int4', value: connection_id }},
-              --- The objectId can be string | number | undefined, we store it as jsonb value
-              ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
-              ${{ type: 'varchar', value: schema }},
-              ${{ type: 'varchar', value: table }},
-              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
-            )
-          RETURNING
-            *
-        `
-          .decoded(models.SourceTable)
-          .first();
-        sourceTableRow = row;
-      }
-
-      const sourceTable = new storage.SourceTable({
-        id: sourceTableRow!.id,
-        connectionTag: connection_tag,
-        objectId: objectId,
-        schema: schema,
-        name: table,
-        replicaIdColumns: replicaIdColumns,
-        snapshotComplete: sourceTableRow!.snapshot_done ?? true
-      });
-      if (!sourceTable.snapshotComplete) {
-        sourceTable.snapshotStatus = {
-          totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
-          replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
-          lastKey: sourceTableRow!.snapshot_last_key
-        };
-      }
-      sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable);
-      sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
-      sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);
-
-      let truncatedTables: SourceTableDecoded[] = [];
-      if (objectId != null) {
-        // relation_id present - check for renamed tables
-        truncatedTables = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-            AND (
-              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-              OR (
-                schema_name = ${{ type: 'varchar', value: schema }}
-                AND table_name = ${{ type: 'varchar', value: table }}
-              )
-            )
-        `
-          .decoded(models.SourceTable)
-          .rows();
-      } else {
-        // relation_id not present - only check for changed replica_id_columns
-        truncatedTables = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-            AND (
-              schema_name = ${{ type: 'varchar', value: schema }}
-              AND table_name = ${{ type: 'varchar', value: table }}
-            )
-        `
-          .decoded(models.SourceTable)
-          .rows();
-      }
-
-      return {
-        table: sourceTable,
-        dropTables: truncatedTables.map(
-          (doc) =>
-            new storage.SourceTable({
-              id: doc.id,
-              connectionTag: connection_tag,
-              objectId: doc.relation_id?.object_id ?? 0,
-              schema: doc.schema_name,
-              name: doc.table_name,
-              replicaIdColumns:
-                doc.replica_id_columns?.map((c) => ({
-                  name: c.name,
-                  typeOid: c.typeId,
-                  type: c.type
-                })) ?? [],
-              snapshotComplete: doc.snapshot_done ?? true
-            })
-        )
-      };
-    });
-  }
-
   async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
     const syncRules = await this.db.sql`
       SELECT
@@ -360,7 +201,7 @@ export class PostgresSyncRulesStorage
       FROM
         sync_rules
       WHERE
-        id = ${{ type: 'int4', value: this.group_id }}
+        id = ${{ type: 'int4', value: this.replicationStreamId }}
     `
       .decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op', 'snapshot_lsn']))
       .first();
@@ -370,9 +211,9 @@ export class PostgresSyncRulesStorage
     const writer = new PostgresBucketBatch({
       logger: options.logger ?? this.logger,
       db: this.db,
-      sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(),
-      group_id: this.group_id,
-      slot_name: this.slot_name,
+      sync_rules: this.getParsedSyncRules(options),
+      replicationStreamId: this.replicationStreamId,
+      replicationStreamName: this.replicationStreamName,
       last_checkpoint_lsn: checkpoint_lsn,
       keep_alive_op: syncRules?.keepalive_op,
       resumeFromLsn: maxLsn(syncRules?.snapshot_lsn, checkpoint_lsn),
@@ -380,6 +221,7 @@ export class PostgresSyncRulesStorage
       skip_existing_rows: options.skipExistingRows ?? false,
       batch_limits: this.options.batchLimits,
       markRecordUnavailable: options.markRecordUnavailable,
+      hooks: options.hooks,
       storageConfig: this.storageConfig
     });
     this.iterateListeners((cb) => cb.batchStarted?.(writer));
@@ -417,7 +259,7 @@ export class PostgresSyncRulesStorage
         value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
       }}) WITH ORDINALITY AS requested (value, index)
           WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
+            group_id = ${{ type: 'int4', value: this.replicationStreamId }}
             AND lookup = decode((requested.value ->> 0)::text, 'hex') -- Decode the hex string to bytea
             AND id <= ${{ type: 'int8', value: checkpoint.checkpoint }}
           ORDER BY
@@ -544,7 +386,7 @@ export class PostgresSyncRulesStorage
           LIMIT
             $3;`,
       params: [
-        { type: 'int4', value: this.group_id },
+        { type: 'int4', value: this.replicationStreamId },
         { type: 'int8', value: end },
         { type: 'int4', value: batchRowLimit },
         ...filters.flatMap((f) => [
@@ -660,11 +502,11 @@ export class PostgresSyncRulesStorage
         state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }},
         snapshot_done = ${{ type: 'bool', value: false }}
       WHERE
-        id = ${{ type: 'int4', value: this.group_id }}
+        id = ${{ type: 'int4', value: this.replicationStreamId }}
     `.execute();
   }
 
-  async getStatus(): Promise<storage.SyncRuleStatus> {
+  async getStatus(): Promise<storage.ReplicationStreamStatus> {
     const syncRulesRow = await this.db.sql`
       SELECT
         snapshot_done,
@@ -674,9 +516,9 @@ export class PostgresSyncRulesStorage
       FROM
         sync_rules
       WHERE
-        id = ${{ type: 'int4', value: this.group_id }}
+        id = ${{ type: 'int4', value: this.replicationStreamId }}
     `
-      .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state', 'snapshot_lsn']))
+      .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'snapshot_lsn']))
       .first();
 
     if (syncRulesRow == null) {
@@ -684,10 +526,8 @@ export class PostgresSyncRulesStorage
     }
 
     return {
-      snapshot_done: syncRulesRow.snapshot_done,
-      active: syncRulesRow.state == storage.SyncRuleState.ACTIVE,
-      checkpoint_lsn: syncRulesRow.last_checkpoint_lsn ?? null,
-      snapshot_lsn: syncRulesRow.snapshot_lsn ?? null
+      snapshotDone: syncRulesRow.snapshot_done && syncRulesRow.last_checkpoint_lsn != null,
+      resumeLsn: maxLsn(syncRulesRow.snapshot_lsn, syncRulesRow.last_checkpoint_lsn)
     };
   }
 
@@ -701,27 +541,27 @@ export class PostgresSyncRulesStorage
         last_checkpoint = NULL,
         no_checkpoint_before = NULL
       WHERE
-        id = ${{ type: 'int4', value: this.group_id }}
+        id = ${{ type: 'int4', value: this.replicationStreamId }}
     `.execute();
 
     await this.db.sql`
       DELETE FROM bucket_data
       WHERE
-        group_id = ${{ type: 'int4', value: this.group_id }}
+        group_id = ${{ type: 'int4', value: this.replicationStreamId }}
     `.execute();
 
     await this.db.sql`
       DELETE FROM bucket_parameters
       WHERE
-        group_id = ${{ type: 'int4', value: this.group_id }}
+        group_id = ${{ type: 'int4', value: this.replicationStreamId }}
     `.execute();
 
-    await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.group_id });
+    await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.replicationStreamId });
 
     await this.db.sql`
       DELETE FROM source_tables
       WHERE
-        group_id = ${{ type: 'int4', value: this.group_id }}
+        group_id = ${{ type: 'int4', value: this.replicationStreamId }}
     `.execute();
   }
 
@@ -762,7 +602,7 @@ export class PostgresSyncRulesStorage
         AND b.op_id > f.start_op_id
         AND b.op_id <= f.end_op_id
       WHERE
-        b.group_id = ${{ type: 'int4', value: this.group_id }}
+        b.group_id = ${{ type: 'int4', value: this.replicationStreamId }}
       GROUP BY
         b.bucket_name;
     `.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>();