@powersync/service-module-postgres-storage 0.18.0 → 0.19.1

package/src/storage/PostgresBucketStorageFactory.ts

@@ -15,6 +15,7 @@ export type PostgresBucketStorageOptions = {
   config: NormalizedPostgresStorageConfig;
   replicationStreamNamePrefix: string;
   checksumCacheTtlMs?: number;
+  defaultStorageVersion?: number;
 };
 
 export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
@@ -78,7 +79,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
       return {
         operations_size_bytes: 0,
         parameters_size_bytes: 0,
-        replication_size_bytes: 0
+        replication_size_bytes: 0,
+        object_storage_size_bytes: 0
       };
     }
 
@@ -104,7 +106,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
     return {
       operations_size_bytes: Number(sizes!.operations_size_bytes),
       parameters_size_bytes: Number(sizes!.parameter_size_bytes),
-      replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes)
+      replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes),
+      object_storage_size_bytes: 0
     };
   }
 
@@ -156,12 +159,15 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
     const storageVersion =
-      options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+      options.storageVersion ??
+      options.config.parsed.config.storageVersion ??
+      this.options.defaultStorageVersion ??
+      storage.CURRENT_STORAGE_VERSION;
     const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
-    if (storageConfig == null) {
+    if (storageConfig == null || storageVersion >= storage.STORAGE_VERSION_4) {
       throw new framework.ServiceError(
         framework.ErrorCode.PSYNC_S1005,
-        `Unsupported storage version ${storageVersion}`
+        `Unsupported storage version ${storageVersion} for PostgreSQL storage`
       );
     }
     await this.initializeStorageVersion(storageConfig);
@@ -187,7 +193,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
             sync_plan,
             state,
             slot_name,
-            storage_version
+            storage_version,
+            version_label
           )
         VALUES
           (
@@ -211,7 +218,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
               '_',
               ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
             ),
-            ${{ type: 'int4', value: storageVersion }}
+            ${{ type: 'int4', value: storageVersion }},
+            ${{ type: 'varchar', value: options.version_label ?? null }}
           )
         RETURNING
           *

package/src/storage/PostgresStorageProvider.ts

@@ -28,7 +28,8 @@ export class PostgresStorageProvider implements storage.StorageProvider {
     const storageFactory = new PostgresBucketStorageFactory({
       config: normalizedConfig,
       replicationStreamNamePrefix: options.resolvedConfig.slot_name_prefix,
-      checksumCacheTtlMs: options.resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000
+      checksumCacheTtlMs: options.resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
+      defaultStorageVersion: decodedConfig.default_storage_version
     });
 
     const reportStorageFactory = new PostgresReportStorage({

package/src/storage/PostgresSyncRulesStorage.ts

@@ -4,6 +4,8 @@ import {
   BucketChecksum,
   CHECKPOINT_INVALIDATE_ALL,
   CheckpointChanges,
+  CompactInitialReplicationOptions,
+  CompactInitialReplicationResults,
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
@@ -11,11 +13,10 @@ import {
   maxLsn,
   ParameterSetLimitExceededError,
   PartialChecksum,
-  PopulateChecksumCacheOptions,
-  PopulateChecksumCacheResults,
   ReplicationCheckpoint,
   storage,
   StorageVersionConfig,
+  SyncRuleState,
   utils,
   WatchWriteCheckpointOptions
 } from '@powersync/service-core';
@@ -94,7 +95,7 @@ export class PostgresSyncRulesStorage
     this.factory = options.factory;
     this.storageConfig = options.replicationStream.getStorageConfig();
     this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
-    this.logger = options.replicationStream.logger;
+    this.logger = this.sync_rules.logger;
 
     this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
       db: this.db,
@@ -145,7 +146,19 @@ export class PostgresSyncRulesStorage
     `.execute();
   }
 
+  supportsIncrementalParameterCompaction(): boolean {
+    return false;
+  }
+
   async compact(options?: storage.CompactOptions): Promise<void> {
+    if (options?.incrementalOnly) {
+      // Not supported yet
+      this.logger.info('Incremental compacting is not supported on Postgres storage yet.');
+      return;
+    } else if (this.replicationStream.state != SyncRuleState.ACTIVE) {
+      this.logger.info(`Skipping compacting of replication stream in ${this.replicationStream.state} state.`);
+      return;
+    }
     let maxOpId = options?.maxOpId;
     if (maxOpId == null) {
       const checkpoint = await this.getCheckpoint();
@@ -160,7 +173,9 @@ export class PostgresSyncRulesStorage
     }).compact();
   }
 
-  async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
+  async compactInitialReplication(
+    _options: CompactInitialReplicationOptions
+  ): Promise<CompactInitialReplicationResults> {
     // no-op - checksum cache is not implemented for Postgres yet
     return { buckets: 0 };
   }
@@ -172,8 +187,8 @@ export class PostgresSyncRulesStorage
     });
   }
 
-  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
-    return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
+  setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    return this.writeCheckpointAPI.setWriteCheckpointMode(config);
   }
 
   createManagedWriteCheckpoints(

package/src/storage/batch/PostgresBucketBatch.ts

@@ -19,6 +19,7 @@ import {
   utils
 } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
+import { orNull } from '@powersync/service-types';
 import * as timers from 'timers/promises';
 import * as t from 'ts-codec';
 import * as uuid from 'uuid';
@@ -64,10 +65,10 @@ const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(
 const CheckpointWithStatus = StatefulCheckpoint.and(
   t.object({
     snapshot_done: t.boolean,
-    no_checkpoint_before: t.string.or(t.Null),
+    no_checkpoint_before: orNull(t.string),
     can_checkpoint: t.boolean,
-    keepalive_op: bigint.or(t.Null),
-    new_last_checkpoint: bigint.or(t.Null),
+    keepalive_op: orNull(bigint),
+    new_last_checkpoint: orNull(bigint),
     created_checkpoint: t.boolean
   })
 );
@@ -323,7 +324,9 @@ export class PostgresBucketBatch
     // TODO maybe share with abstract class
     const { after, before, sourceTable, tag } = record;
     const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
-    // Only the table designated as event carrier by resolveTables may fire events.
+    // PostgreSQL storage resolves each physical table to one SourceTable. syncEvent records whether
+    // that table matches any event definition; each matching definition is evaluated once here and
+    // may produce multiple payloads from its matching queries.
     if (sourceTable.syncEvent) {
       for (const event of this.getTableEvents(sourceTable)) {
         this.iterateListeners((cb) =>
@@ -1334,10 +1337,10 @@ export class PostgresBucketBatch
   }
 
   /**
-   * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
+   * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable}
    * TODO maybe share this with an abstract class
    */
-  protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
+  protected getTableEvents(table: storage.SourceTable): sync_rules.HydratedEventDescriptor[] {
     return this.sync_rules.eventDescriptors.filter((evt) =>
       [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
     );

package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts

@@ -21,8 +21,8 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     return this._mode;
   }
 
-  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
-    this._mode = mode;
+  setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this._mode = config.mode;
   }
 
   async batchCreateCustomWriteCheckpoints(

package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts

@@ -14,7 +14,8 @@ export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncCon
       compiled_plan: row.sync_plan,
       replicationStreamName: row.slot_name,
       storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
-      syncConfigState: row.state as storage.SyncRuleState
+      syncConfigState: row.state as storage.SyncRuleState,
+      version_label: row.version_label ?? undefined
     });
   }
 
@@ -61,6 +62,12 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
       state: row.state as storage.SyncRuleState,
       storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
     });
+    if (this.storageVersion >= storage.STORAGE_VERSION_4) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for PostgreSQL storage`
+      );
+    }
     this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
   }
 

package/src/types/models/ActiveCheckpoint.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
 
@@ -7,8 +8,8 @@ import { bigint, pgwire_number } from '../codecs.js';
  */
 export const ActiveCheckpoint = t.object({
   id: pgwire_number,
-  last_checkpoint: t.Null.or(bigint),
-  last_checkpoint_lsn: t.Null.or(t.string)
+  last_checkpoint: orNull(bigint),
+  last_checkpoint_lsn: orNull(t.string)
 });
 
 export type ActiveCheckpoint = t.Encoded<typeof ActiveCheckpoint>;

package/src/types/models/BucketData.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
 
@@ -13,13 +14,13 @@ export const BucketData = t.object({
   bucket_name: t.string,
   op_id: bigint,
   op: t.Enum(OpType),
-  source_table: t.Null.or(t.string),
-  source_key: t.Null.or(hexBuffer),
-  table_name: t.string.or(t.Null),
-  row_id: t.string.or(t.Null),
+  source_table: orNull(t.string),
+  source_key: orNull(hexBuffer),
+  table_name: orNull(t.string),
+  row_id: orNull(t.string),
   checksum: bigint,
-  data: t.Null.or(t.string),
-  target_op: t.Null.or(bigint)
+  data: orNull(t.string),
+  target_op: orNull(bigint)
 });
 
 export type BucketData = t.Encoded<typeof BucketData>;

package/src/types/models/CurrentData.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
 
@@ -26,7 +27,7 @@ export const V3CurrentData = t.object({
   lookups: t.array(hexBuffer),
   source_key: hexBuffer,
   source_table: t.string,
-  pending_delete: t.Null.or(bigint)
+  pending_delete: orNull(bigint)
 });
 
 export type V1CurrentData = t.Encoded<typeof V1CurrentData>;

package/src/types/models/SdkReporting.ts

@@ -1,3 +1,4 @@
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 
@@ -11,12 +12,11 @@ export type Sdks = t.Encoded<typeof Sdks>;
 
 export const SdkReporting = t.object({
   users: bigint,
-  sdks: t
-    .object({
+  sdks: orNull(
+    t.object({
       data: jsonb<Sdks[]>(t.array(Sdks))
     })
-    .optional()
-    .or(t.Null)
+  ).optional()
 });
 
 export type SdkReporting = t.Encoded<typeof SdkReporting>;

package/src/types/models/SourceTable.ts

@@ -1,4 +1,5 @@
 import { JsonValue } from '@powersync/service-core';
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, hexBuffer, jsonb, jsonb_raw, pgwire_number } from '../codecs.js';
 
@@ -22,18 +23,18 @@ export const SourceTable = t.object({
   id: t.string,
   group_id: pgwire_number,
   connection_id: bigint,
-  relation_id: t.Null.or(jsonb_raw<StoredRelationId>()),
+  relation_id: orNull(jsonb_raw<StoredRelationId>()),
   schema_name: t.string,
   table_name: t.string,
-  replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))),
+  replica_id_columns: orNull(jsonb(t.array(ColumnDescriptor))),
   snapshot_done: t.boolean,
-  snapshot_total_estimated_count: t.Null.or(bigint),
-  snapshot_replicated_count: t.Null.or(bigint),
-  snapshot_last_key: t.Null.or(hexBuffer),
+  snapshot_total_estimated_count: orNull(bigint),
+  snapshot_replicated_count: orNull(bigint),
+  snapshot_last_key: orNull(hexBuffer),
   /**
    * Source-specific metadata. Null for legacy records.
    */
-  source_metadata: t.Null.or(jsonb_raw<JsonValue>())
+  source_metadata: orNull(jsonb_raw<JsonValue>())
 });
 
 export type SourceTable = t.Encoded<typeof SourceTable>;

package/src/types/models/SyncRules.ts

@@ -1,5 +1,5 @@
 import { framework, storage } from '@powersync/service-core';
-import { ReplicationError } from '@powersync/service-types';
+import { orNull, ReplicationError } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
 import { jsonContainerObject } from './json.js';
@@ -16,42 +16,43 @@ export const SyncRules = t.object({
   /**
    * May be set if snapshot_done = false, if the replication stream requires it.
    */
-  snapshot_lsn: t.Null.or(t.string),
+  snapshot_lsn: orNull(t.string),
   /**
    * The last consistent checkpoint.
    *
    * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
    */
-  last_checkpoint: t.Null.or(bigint),
+  last_checkpoint: orNull(bigint),
   /**
    * The LSN associated with the last consistent checkpoint.
    */
-  last_checkpoint_lsn: t.Null.or(t.string),
+  last_checkpoint_lsn: orNull(t.string),
   /**
    * If set, no new checkpoints may be created < this value.
    */
-  no_checkpoint_before: t.Null.or(t.string),
+  no_checkpoint_before: orNull(t.string),
   slot_name: t.string,
   /**
    * Last time we persisted a checkpoint.
    *
    * This may be old if no data is incoming.
    */
-  last_checkpoint_ts: t.Null.or(framework.codecs.date),
+  last_checkpoint_ts: orNull(framework.codecs.date),
   /**
    * Last time we persisted a checkpoint or keepalive.
    *
    * This should stay fairly current while replicating.
    */
-  last_keepalive_ts: t.Null.or(framework.codecs.date),
+  last_keepalive_ts: orNull(framework.codecs.date),
   /**
    * If an error is stopping replication, it will be stored here.
    */
-  last_fatal_error: t.Null.or(t.string),
-  keepalive_op: t.Null.or(bigint),
-  storage_version: t.Null.or(pgwire_number).optional(),
+  last_fatal_error: orNull(t.string),
+  keepalive_op: orNull(bigint),
+  storage_version: orNull(pgwire_number).optional(),
+  version_label: orNull(t.string).optional(),
   content: t.string,
-  sync_plan: t.Null.or(
+  sync_plan: orNull(
     jsonContainerObject(
       t.object({
         plan: t.any,

package/src/types/models/WriteCheckpoint.ts

@@ -1,4 +1,5 @@
 import { framework } from '@powersync/service-core';
+import { orNull } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 
@@ -6,7 +7,7 @@ export const WriteCheckpoint = t.object({
   user_id: t.string,
   lsns: jsonb(t.record(t.string)),
   write_checkpoint: bigint,
-  checkpoint_requested_at: t.Null.or(framework.codecs.date)
+  checkpoint_requested_at: orNull(framework.codecs.date)
 });
 
 export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
@@ -16,7 +17,7 @@ export const CustomWriteCheckpoint = t.object({
   user_id: t.string,
   write_checkpoint: bigint,
   sync_rules_id: bigint,
-  checkpoint_requested_at: t.Null.or(framework.codecs.date)
+  checkpoint_requested_at: orNull(framework.codecs.date)
 });
 
 export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;