@powersync/service-module-postgres-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/CHANGELOG.md +125 -8
  2. package/dist/migrations/scripts/1784900000000-source-metadata.d.ts +3 -0
  3. package/dist/migrations/scripts/1784900000000-source-metadata.js +22 -0
  4. package/dist/migrations/scripts/1784900000000-source-metadata.js.map +1 -0
  5. package/dist/migrations/scripts/1788000000000-sync-config-version-label.d.ts +3 -0
  6. package/dist/migrations/scripts/1788000000000-sync-config-version-label.js +12 -0
  7. package/dist/migrations/scripts/1788000000000-sync-config-version-label.js.map +1 -0
  8. package/dist/storage/PostgresBucketStorageFactory.d.ts +2 -1
  9. package/dist/storage/PostgresBucketStorageFactory.js +27 -14
  10. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  11. package/dist/storage/PostgresStorageProvider.js +3 -2
  12. package/dist/storage/PostgresStorageProvider.js.map +1 -1
  13. package/dist/storage/PostgresSyncRulesStorage.d.ts +4 -3
  14. package/dist/storage/PostgresSyncRulesStorage.js +17 -5
  15. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  16. package/dist/storage/batch/PostgresBucketBatch.d.ts +2 -2
  17. package/dist/storage/batch/PostgresBucketBatch.js +63 -94
  18. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  19. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
  20. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +2 -2
  21. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
  22. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +3 -2
  23. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +12 -5
  24. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -1
  25. package/dist/types/models/ActiveCheckpoint.d.ts +2 -2
  26. package/dist/types/models/ActiveCheckpoint.js +3 -2
  27. package/dist/types/models/ActiveCheckpoint.js.map +1 -1
  28. package/dist/types/models/ActiveCheckpointNotification.d.ts +2 -2
  29. package/dist/types/models/BucketData.d.ts +6 -6
  30. package/dist/types/models/BucketData.js +7 -6
  31. package/dist/types/models/BucketData.js.map +1 -1
  32. package/dist/types/models/CurrentData.d.ts +1 -1
  33. package/dist/types/models/CurrentData.js +2 -1
  34. package/dist/types/models/CurrentData.js.map +1 -1
  35. package/dist/types/models/SdkReporting.d.ts +3 -3
  36. package/dist/types/models/SdkReporting.js +3 -5
  37. package/dist/types/models/SdkReporting.js.map +1 -1
  38. package/dist/types/models/SourceTable.d.ts +13 -8
  39. package/dist/types/models/SourceTable.js +11 -6
  40. package/dist/types/models/SourceTable.js.map +1 -1
  41. package/dist/types/models/SyncRules.d.ts +11 -10
  42. package/dist/types/models/SyncRules.js +12 -11
  43. package/dist/types/models/SyncRules.js.map +1 -1
  44. package/dist/types/models/WriteCheckpoint.d.ts +2 -2
  45. package/dist/types/models/WriteCheckpoint.js +3 -2
  46. package/dist/types/models/WriteCheckpoint.js.map +1 -1
  47. package/dist/types/types.d.ts +2 -0
  48. package/dist/utils/test-utils.d.ts +2 -1
  49. package/dist/utils/test-utils.js +3 -3
  50. package/dist/utils/test-utils.js.map +1 -1
  51. package/package.json +8 -8
  52. package/src/migrations/scripts/1784900000000-source-metadata.ts +31 -0
  53. package/src/migrations/scripts/1788000000000-sync-config-version-label.ts +18 -0
  54. package/src/storage/PostgresBucketStorageFactory.ts +30 -14
  55. package/src/storage/PostgresStorageProvider.ts +9 -5
  56. package/src/storage/PostgresSyncRulesStorage.ts +21 -6
  57. package/src/storage/batch/PostgresBucketBatch.ts +67 -98
  58. package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +2 -2
  59. package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +24 -11
  60. package/src/types/models/ActiveCheckpoint.ts +3 -2
  61. package/src/types/models/BucketData.ts +7 -6
  62. package/src/types/models/CurrentData.ts +2 -1
  63. package/src/types/models/SdkReporting.ts +4 -4
  64. package/src/types/models/SourceTable.ts +12 -6
  65. package/src/types/models/SyncRules.ts +12 -11
  66. package/src/types/models/WriteCheckpoint.ts +3 -2
  67. package/src/utils/test-utils.ts +23 -9
  68. package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -995
  69. package/test/src/checkpoint_notifications.test.ts +8 -2
  70. package/test/src/migrations.test.ts +8 -4
  71. package/test/src/storage.test.ts +12 -4
  72. package/test/src/storage_compacting.test.ts +2 -2
  73. package/test/src/storage_sync.test.ts +44 -0
  74. package/test/src/util.ts +1 -1
  75. package/tsconfig.tsbuildinfo +1 -1
@@ -15,6 +15,7 @@ export type PostgresBucketStorageOptions = {
15
15
  config: NormalizedPostgresStorageConfig;
16
16
  replicationStreamNamePrefix: string;
17
17
  checksumCacheTtlMs?: number;
18
+ defaultStorageVersion?: number;
18
19
  };
19
20
 
20
21
  export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
@@ -24,8 +25,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
24
25
 
25
26
  private activeStorageCache: storage.SyncRulesBucketStorage | undefined;
26
27
 
27
- constructor(protected options: PostgresBucketStorageOptions) {
28
- super();
28
+ constructor(
29
+ protected options: PostgresBucketStorageOptions,
30
+ syncConfigParser: storage.SyncConfigParser
31
+ ) {
32
+ super({ syncConfigParser });
29
33
  this.db = new lib_postgres.DatabaseClient({
30
34
  config: options.config,
31
35
  schema: STORAGE_SCHEMA_NAME,
@@ -40,7 +44,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
40
44
  }
41
45
 
42
46
  async [Symbol.asyncDispose]() {
43
- await this.db[Symbol.asyncDispose]();
47
+ try {
48
+ await super[Symbol.asyncDispose]();
49
+ } finally {
50
+ await this.db[Symbol.asyncDispose]();
51
+ }
44
52
  }
45
53
 
46
54
  async prepareStatements(connection: pg_wire.PgConnection) {
@@ -78,7 +86,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
78
86
  return {
79
87
  operations_size_bytes: 0,
80
88
  parameters_size_bytes: 0,
81
- replication_size_bytes: 0
89
+ replication_size_bytes: 0,
90
+ object_storage_size_bytes: 0
82
91
  };
83
92
  }
84
93
 
@@ -104,7 +113,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
104
113
  return {
105
114
  operations_size_bytes: Number(sizes!.operations_size_bytes),
106
115
  parameters_size_bytes: Number(sizes!.parameter_size_bytes),
107
- replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes)
116
+ replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes),
117
+ object_storage_size_bytes: 0
108
118
  };
109
119
  }
110
120
 
@@ -156,12 +166,15 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
156
166
 
157
167
  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
158
168
  const storageVersion =
159
- options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
169
+ options.storageVersion ??
170
+ options.config.parsed.config.storageVersion ??
171
+ this.options.defaultStorageVersion ??
172
+ storage.CURRENT_STORAGE_VERSION;
160
173
  const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
161
- if (storageConfig == null) {
174
+ if (storageConfig == null || storageVersion >= storage.STORAGE_VERSION_4) {
162
175
  throw new framework.ServiceError(
163
176
  framework.ErrorCode.PSYNC_S1005,
164
- `Unsupported storage version ${storageVersion}`
177
+ `Unsupported storage version ${storageVersion} for PostgreSQL storage`
165
178
  );
166
179
  }
167
180
  await this.initializeStorageVersion(storageConfig);
@@ -174,6 +187,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
174
187
  state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
175
188
  `.execute();
176
189
 
190
+ // Uses the legacy sequence name "sync_rules_id_sequence" for backwards-compatibility. See AGENTS.md.
177
191
  const newSyncRulesRow = await db.sql`
178
192
  WITH
179
193
  next_id AS (
@@ -187,7 +201,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
187
201
  sync_plan,
188
202
  state,
189
203
  slot_name,
190
- storage_version
204
+ storage_version,
205
+ version_label
191
206
  )
192
207
  VALUES
193
208
  (
@@ -211,7 +226,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
211
226
  '_',
212
227
  ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
213
228
  ),
214
- ${{ type: 'int4', value: storageVersion }}
229
+ ${{ type: 'int4', value: storageVersion }},
230
+ ${{ type: 'varchar', value: options.version_label ?? null }}
215
231
  )
216
232
  RETURNING
217
233
  *
@@ -219,7 +235,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
219
235
  .decoded(models.SyncRules)
220
236
  .first();
221
237
 
222
- return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
238
+ return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!, this.syncConfigParser);
223
239
  });
224
240
  }
225
241
 
@@ -353,7 +369,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
353
369
  .decoded(models.SyncRules)
354
370
  .rows();
355
371
 
356
- return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
372
+ return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row, this.syncConfigParser));
357
373
  }
358
374
 
359
375
  async getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
@@ -368,14 +384,14 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
368
384
  .decoded(models.SyncRules)
369
385
  .rows();
370
386
 
371
- return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
387
+ return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row, this.syncConfigParser));
372
388
  }
373
389
 
374
390
  private resolvedSyncConfigFromRow(
375
391
  row: models.SyncRulesDecoded,
376
392
  options: { cacheActiveStorage?: boolean } = {}
377
393
  ): storage.ResolvedSyncConfig {
378
- const stream = new PostgresPersistedReplicationStream(this.db, row);
394
+ const stream = new PostgresPersistedReplicationStream(this.db, row, this.syncConfigParser);
379
395
  const content = stream.syncConfigContent[0];
380
396
  const thisFactory = this;
381
397
 
@@ -25,11 +25,15 @@ export class PostgresStorageProvider implements storage.StorageProvider {
25
25
 
26
26
  const decodedConfig = PostgresStorageConfig.decode(storage);
27
27
  const normalizedConfig = normalizePostgresStorageConfig(decodedConfig);
28
- const storageFactory = new PostgresBucketStorageFactory({
29
- config: normalizedConfig,
30
- replicationStreamNamePrefix: options.resolvedConfig.slot_name_prefix,
31
- checksumCacheTtlMs: options.resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000
32
- });
28
+ const storageFactory = new PostgresBucketStorageFactory(
29
+ {
30
+ config: normalizedConfig,
31
+ replicationStreamNamePrefix: options.resolvedConfig.slot_name_prefix,
32
+ checksumCacheTtlMs: options.resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
33
+ defaultStorageVersion: decodedConfig.default_storage_version
34
+ },
35
+ options.syncConfigParser
36
+ );
33
37
 
34
38
  const reportStorageFactory = new PostgresReportStorage({
35
39
  config: normalizedConfig
@@ -4,6 +4,8 @@ import {
4
4
  BucketChecksum,
5
5
  CHECKPOINT_INVALIDATE_ALL,
6
6
  CheckpointChanges,
7
+ CompactInitialReplicationOptions,
8
+ CompactInitialReplicationResults,
7
9
  GetCheckpointChangesOptions,
8
10
  InternalOpId,
9
11
  internalToExternalOpId,
@@ -11,11 +13,10 @@ import {
11
13
  maxLsn,
12
14
  ParameterSetLimitExceededError,
13
15
  PartialChecksum,
14
- PopulateChecksumCacheOptions,
15
- PopulateChecksumCacheResults,
16
16
  ReplicationCheckpoint,
17
17
  storage,
18
18
  StorageVersionConfig,
19
+ SyncRuleState,
19
20
  utils,
20
21
  WatchWriteCheckpointOptions
21
22
  } from '@powersync/service-core';
@@ -94,7 +95,7 @@ export class PostgresSyncRulesStorage
94
95
  this.factory = options.factory;
95
96
  this.storageConfig = options.replicationStream.getStorageConfig();
96
97
  this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
97
- this.logger = options.replicationStream.logger;
98
+ this.logger = this.sync_rules.logger;
98
99
 
99
100
  this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
100
101
  db: this.db,
@@ -145,7 +146,19 @@ export class PostgresSyncRulesStorage
145
146
  `.execute();
146
147
  }
147
148
 
149
+ supportsIncrementalParameterCompaction(): boolean {
150
+ return false;
151
+ }
152
+
148
153
  async compact(options?: storage.CompactOptions): Promise<void> {
154
+ if (options?.incrementalOnly) {
155
+ // Not supported yet
156
+ this.logger.info('Incremental compacting is not supported on Postgres storage yet.');
157
+ return;
158
+ } else if (this.replicationStream.state != SyncRuleState.ACTIVE) {
159
+ this.logger.info(`Skipping compacting of replication stream in ${this.replicationStream.state} state.`);
160
+ return;
161
+ }
149
162
  let maxOpId = options?.maxOpId;
150
163
  if (maxOpId == null) {
151
164
  const checkpoint = await this.getCheckpoint();
@@ -160,7 +173,9 @@ export class PostgresSyncRulesStorage
160
173
  }).compact();
161
174
  }
162
175
 
163
- async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
176
+ async compactInitialReplication(
177
+ _options: CompactInitialReplicationOptions
178
+ ): Promise<CompactInitialReplicationResults> {
164
179
  // no-op - checksum cache is not implemented for Postgres yet
165
180
  return { buckets: 0 };
166
181
  }
@@ -172,8 +187,8 @@ export class PostgresSyncRulesStorage
172
187
  });
173
188
  }
174
189
 
175
- setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
176
- return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
190
+ setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
191
+ return this.writeCheckpointAPI.setWriteCheckpointMode(config);
177
192
  }
178
193
 
179
194
  createManagedWriteCheckpoints(
@@ -19,6 +19,7 @@ import {
19
19
  utils
20
20
  } from '@powersync/service-core';
21
21
  import * as sync_rules from '@powersync/service-sync-rules';
22
+ import { orNull } from '@powersync/service-types';
22
23
  import * as timers from 'timers/promises';
23
24
  import * as t from 'ts-codec';
24
25
  import * as uuid from 'uuid';
@@ -64,10 +65,10 @@ const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(
64
65
  const CheckpointWithStatus = StatefulCheckpoint.and(
65
66
  t.object({
66
67
  snapshot_done: t.boolean,
67
- no_checkpoint_before: t.string.or(t.Null),
68
+ no_checkpoint_before: orNull(t.string),
68
69
  can_checkpoint: t.boolean,
69
- keepalive_op: bigint.or(t.Null),
70
- new_last_checkpoint: bigint.or(t.Null),
70
+ keepalive_op: orNull(bigint),
71
+ new_last_checkpoint: orNull(bigint),
71
72
  created_checkpoint: t.boolean
72
73
  })
73
74
  );
@@ -144,6 +145,7 @@ export class PostgresBucketBatch
144
145
 
145
146
  async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
146
147
  const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
148
+ const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
147
149
  const { connection_id, source } = options;
148
150
  const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
149
151
 
@@ -154,9 +156,10 @@ export class PostgresBucketBatch
154
156
  }));
155
157
 
156
158
  return this.db.transaction(async (db) => {
157
- let sourceTableRow: SourceTableDecoded | null;
159
+ // Find records that overlap by name or relation id.
160
+ let candidateRows: SourceTableDecoded[];
158
161
  if (objectId != null) {
159
- sourceTableRow = await db.sql`
162
+ candidateRows = await db.sql`
160
163
  SELECT
161
164
  *
162
165
  FROM
@@ -164,15 +167,18 @@ export class PostgresBucketBatch
164
167
  WHERE
165
168
  group_id = ${{ type: 'int4', value: this.group_id }}
166
169
  AND connection_id = ${{ type: 'int4', value: connection_id }}
167
- AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
168
- AND schema_name = ${{ type: 'varchar', value: schema }}
169
- AND table_name = ${{ type: 'varchar', value: table }}
170
- AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
170
+ AND (
171
+ relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
172
+ OR (
173
+ schema_name = ${{ type: 'varchar', value: schema }}
174
+ AND table_name = ${{ type: 'varchar', value: table }}
175
+ )
176
+ )
171
177
  `
172
178
  .decoded(models.SourceTable)
173
- .first();
179
+ .rows();
174
180
  } else {
175
- sourceTableRow = await db.sql`
181
+ candidateRows = await db.sql`
176
182
  SELECT
177
183
  *
178
184
  FROM
@@ -182,15 +188,42 @@ export class PostgresBucketBatch
182
188
  AND connection_id = ${{ type: 'int4', value: connection_id }}
183
189
  AND schema_name = ${{ type: 'varchar', value: schema }}
184
190
  AND table_name = ${{ type: 'varchar', value: table }}
185
- AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
186
191
  `
187
192
  .decoded(models.SourceTable)
188
- .first();
193
+ .rows();
189
194
  }
190
195
 
191
- if (sourceTableRow == null) {
196
+ const candidateTables = candidateRows.map((row) => {
197
+ const table = this.sourceTableFromRow(row, connectionTag, syncRules);
198
+ table.storeCurrentData = sendsCompleteRows !== true;
199
+ return table;
200
+ });
201
+ const candidates = candidateTables.map((table) => table.clone());
202
+ const resolution = await reconcile({ source, candidates });
203
+ storage.validateSourceTableCandidateResolution(candidates, resolution);
204
+
205
+ for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
206
+ await db.sql`
207
+ UPDATE source_tables
208
+ SET
209
+ source_metadata = ${{ type: 'jsonb', value: sourceMetadata ?? null }}
210
+ WHERE
211
+ id = ${{ type: 'varchar', value: id.toString() }}
212
+ `.execute();
213
+ }
214
+
215
+ const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
216
+ const compatibleTables = materializedResolution.compatibleTables;
217
+
218
+ // Keep one record, preferring one that has already been snapshotted.
219
+ const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
220
+
221
+ let sourceTable: storage.SourceTable;
222
+ if (reuse != null) {
223
+ sourceTable = reuse;
224
+ } else {
192
225
  const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
193
- sourceTableRow = await db.sql`
226
+ const insertedRow = await db.sql`
194
227
  INSERT INTO
195
228
  source_tables (
196
229
  id,
@@ -199,7 +232,8 @@ export class PostgresBucketBatch
199
232
  relation_id,
200
233
  schema_name,
201
234
  table_name,
202
- replica_id_columns
235
+ replica_id_columns,
236
+ source_metadata
203
237
  )
204
238
  VALUES
205
239
  (
@@ -209,95 +243,27 @@ export class PostgresBucketBatch
209
243
  ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
210
244
  ${{ type: 'varchar', value: schema }},
211
245
  ${{ type: 'varchar', value: table }},
212
- ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
246
+ ${{ type: 'jsonb', value: normalizedReplicaIdColumns }},
247
+ ${{ type: 'jsonb', value: resolution.newTableValues.sourceMetadata ?? null }}
213
248
  )
214
249
  RETURNING
215
250
  *
216
251
  `
217
252
  .decoded(models.SourceTable)
218
253
  .first();
254
+ sourceTable = this.sourceTableFromRow(insertedRow!, connectionTag, syncRules);
255
+ sourceTable.storeCurrentData = sendsCompleteRows !== true;
219
256
  }
220
257
 
221
- const sourceTable = new storage.SourceTable({
222
- id: sourceTableRow!.id,
223
- ref: source,
224
- objectId,
225
- replicaIdColumns,
226
- snapshotComplete: sourceTableRow!.snapshot_done ?? true,
227
- ...syncRules.getMatchingSources(source)
228
- });
229
- if (!sourceTable.snapshotComplete) {
230
- sourceTable.snapshotStatus = {
231
- totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
232
- replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
233
- lastKey: sourceTableRow!.snapshot_last_key
234
- };
235
- }
236
- sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
237
- sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
238
- sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
239
- sourceTable.storeCurrentData = sendsCompleteRows !== true;
240
-
241
- let truncatedTables: SourceTableDecoded[] = [];
242
- if (objectId != null) {
243
- truncatedTables = await db.sql`
244
- SELECT
245
- *
246
- FROM
247
- source_tables
248
- WHERE
249
- group_id = ${{ type: 'int4', value: this.group_id }}
250
- AND connection_id = ${{ type: 'int4', value: connection_id }}
251
- AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
252
- AND (
253
- relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
254
- OR (
255
- schema_name = ${{ type: 'varchar', value: schema }}
256
- AND table_name = ${{ type: 'varchar', value: table }}
257
- )
258
- )
259
- `
260
- .decoded(models.SourceTable)
261
- .rows();
262
- } else {
263
- truncatedTables = await db.sql`
264
- SELECT
265
- *
266
- FROM
267
- source_tables
268
- WHERE
269
- group_id = ${{ type: 'int4', value: this.group_id }}
270
- AND connection_id = ${{ type: 'int4', value: connection_id }}
271
- AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
272
- AND (
273
- schema_name = ${{ type: 'varchar', value: schema }}
274
- AND table_name = ${{ type: 'varchar', value: table }}
275
- )
276
- `
277
- .decoded(models.SourceTable)
278
- .rows();
279
- }
258
+ const dropTables = [
259
+ ...materializedResolution.incompatibleTables,
260
+ // PostgreSQL storage only keeps one SourceTable per physical table.
261
+ ...compatibleTables.filter((candidate) => !storage.sourceTableIdEquals(candidate.id, sourceTable.id))
262
+ ];
280
263
 
281
264
  return {
282
265
  tables: [sourceTable],
283
- dropTables: truncatedTables.map((doc) => {
284
- const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
285
- const dropTable = new storage.SourceTable({
286
- id: doc.id,
287
- ref,
288
- objectId: doc.relation_id?.object_id ?? 0,
289
- replicaIdColumns:
290
- doc.replica_id_columns?.map(
291
- (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
292
- ) ?? [],
293
- snapshotComplete: doc.snapshot_done ?? true,
294
- ...syncRules.getMatchingSources(ref)
295
- });
296
- dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
297
- dropTable.syncData = dropTable.bucketDataSources.length > 0;
298
- dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
299
- return dropTable;
300
- })
266
+ dropTables
301
267
  };
302
268
  });
303
269
  }
@@ -334,9 +300,10 @@ export class PostgresBucketBatch
334
300
  objectId: row.relation_id?.object_id,
335
301
  replicaIdColumns:
336
302
  row.replica_id_columns?.map(
337
- (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
303
+ (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
338
304
  ) ?? [],
339
305
  snapshotComplete: row.snapshot_done ?? true,
306
+ sourceMetadata: row.source_metadata,
340
307
  ...syncRules.getMatchingSources(ref)
341
308
  });
342
309
 
@@ -357,7 +324,9 @@ export class PostgresBucketBatch
357
324
  // TODO maybe share with abstract class
358
325
  const { after, before, sourceTable, tag } = record;
359
326
  const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
360
- // Only the table designated as event carrier by resolveTables may fire events.
327
+ // PostgreSQL storage resolves each physical table to one SourceTable. syncEvent records whether
328
+ // that table matches any event definition; each matching definition is evaluated once here and
329
+ // may produce multiple payloads from its matching queries.
361
330
  if (sourceTable.syncEvent) {
362
331
  for (const event of this.getTableEvents(sourceTable)) {
363
332
  this.iterateListeners((cb) =>
@@ -1368,10 +1337,10 @@ export class PostgresBucketBatch
1368
1337
  }
1369
1338
 
1370
1339
  /**
1371
- * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
1340
+ * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable}
1372
1341
  * TODO maybe share this with an abstract class
1373
1342
  */
1374
- protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
1343
+ protected getTableEvents(table: storage.SourceTable): sync_rules.HydratedEventDescriptor[] {
1375
1344
  return this.sync_rules.eventDescriptors.filter((evt) =>
1376
1345
  [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
1377
1346
  );
@@ -21,8 +21,8 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
21
21
  return this._mode;
22
22
  }
23
23
 
24
- setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
25
- this._mode = mode;
24
+ setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
25
+ this._mode = config.mode;
26
26
  }
27
27
 
28
28
  async batchCreateCustomWriteCheckpoints(
@@ -6,16 +6,21 @@ import { models } from '../../types/types.js';
6
6
  export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
7
7
  constructor(
8
8
  private db: lib_postgres.DatabaseClient,
9
- row: models.SyncRulesDecoded
9
+ row: models.SyncRulesDecoded,
10
+ syncConfigParser: storage.SyncConfigParser
10
11
  ) {
11
- super({
12
- replicationStreamId: Number(row.id),
13
- sync_rules_content: row.content,
14
- compiled_plan: row.sync_plan,
15
- replicationStreamName: row.slot_name,
16
- storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
17
- syncConfigState: row.state as storage.SyncRuleState
18
- });
12
+ super(
13
+ {
14
+ replicationStreamId: Number(row.id),
15
+ sync_rules_content: row.content,
16
+ compiled_plan: row.sync_plan,
17
+ replicationStreamName: row.slot_name,
18
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
19
+ syncConfigState: row.state as storage.SyncRuleState,
20
+ version_label: row.version_label ?? undefined
21
+ },
22
+ syncConfigParser
23
+ );
19
24
  }
20
25
 
21
26
  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
@@ -53,7 +58,8 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
53
58
 
54
59
  constructor(
55
60
  private db: lib_postgres.DatabaseClient,
56
- private readonly row: models.SyncRulesDecoded
61
+ private readonly row: models.SyncRulesDecoded,
62
+ private readonly syncConfigParser: storage.SyncConfigParser
57
63
  ) {
58
64
  super({
59
65
  replicationStreamId: Number(row.id),
@@ -61,7 +67,13 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
61
67
  state: row.state as storage.SyncRuleState,
62
68
  storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
63
69
  });
64
- this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
70
+ if (this.storageVersion >= storage.STORAGE_VERSION_4) {
71
+ throw new ServiceError(
72
+ ErrorCode.PSYNC_S1005,
73
+ `Unsupported storage version ${this.storageVersion} for PostgreSQL storage`
74
+ );
75
+ }
76
+ this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row, this.syncConfigParser)];
65
77
  }
66
78
 
67
79
  parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
@@ -71,6 +83,7 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
71
83
  async lock(): Promise<storage.ReplicationLock> {
72
84
  const manager = new lib_postgres.PostgresLockManager({
73
85
  db: this.db,
86
+ // Uses the legacy "sync_rules_" lock name prefix for backwards-compatibility. See AGENTS.md.
74
87
  name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
75
88
  });
76
89
  const lockHandle = await manager.acquire();
@@ -1,3 +1,4 @@
1
+ import { orNull } from '@powersync/service-types';
1
2
  import * as t from 'ts-codec';
2
3
  import { bigint, pgwire_number } from '../codecs.js';
3
4
 
@@ -7,8 +8,8 @@ import { bigint, pgwire_number } from '../codecs.js';
7
8
  */
8
9
  export const ActiveCheckpoint = t.object({
9
10
  id: pgwire_number,
10
- last_checkpoint: t.Null.or(bigint),
11
- last_checkpoint_lsn: t.Null.or(t.string)
11
+ last_checkpoint: orNull(bigint),
12
+ last_checkpoint_lsn: orNull(t.string)
12
13
  });
13
14
 
14
15
  export type ActiveCheckpoint = t.Encoded<typeof ActiveCheckpoint>;
@@ -1,3 +1,4 @@
1
+ import { orNull } from '@powersync/service-types';
1
2
  import * as t from 'ts-codec';
2
3
  import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
3
4
 
@@ -13,13 +14,13 @@ export const BucketData = t.object({
13
14
  bucket_name: t.string,
14
15
  op_id: bigint,
15
16
  op: t.Enum(OpType),
16
- source_table: t.Null.or(t.string),
17
- source_key: t.Null.or(hexBuffer),
18
- table_name: t.string.or(t.Null),
19
- row_id: t.string.or(t.Null),
17
+ source_table: orNull(t.string),
18
+ source_key: orNull(hexBuffer),
19
+ table_name: orNull(t.string),
20
+ row_id: orNull(t.string),
20
21
  checksum: bigint,
21
- data: t.Null.or(t.string),
22
- target_op: t.Null.or(bigint)
22
+ data: orNull(t.string),
23
+ target_op: orNull(bigint)
23
24
  });
24
25
 
25
26
  export type BucketData = t.Encoded<typeof BucketData>;
@@ -1,3 +1,4 @@
1
+ import { orNull } from '@powersync/service-types';
1
2
  import * as t from 'ts-codec';
2
3
  import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
3
4
 
@@ -26,7 +27,7 @@ export const V3CurrentData = t.object({
26
27
  lookups: t.array(hexBuffer),
27
28
  source_key: hexBuffer,
28
29
  source_table: t.string,
29
- pending_delete: t.Null.or(bigint)
30
+ pending_delete: orNull(bigint)
30
31
  });
31
32
 
32
33
  export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
@@ -1,3 +1,4 @@
1
+ import { orNull } from '@powersync/service-types';
1
2
  import * as t from 'ts-codec';
2
3
  import { bigint, jsonb } from '../codecs.js';
3
4
 
@@ -11,12 +12,11 @@ export type Sdks = t.Encoded<typeof Sdks>;
11
12
 
12
13
  export const SdkReporting = t.object({
13
14
  users: bigint,
14
- sdks: t
15
- .object({
15
+ sdks: orNull(
16
+ t.object({
16
17
  data: jsonb<Sdks[]>(t.array(Sdks))
17
18
  })
18
- .optional()
19
- .or(t.Null)
19
+ ).optional()
20
20
  });
21
21
 
22
22
  export type SdkReporting = t.Encoded<typeof SdkReporting>;