@powersync/service-module-postgres-storage 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +12 -12
  3. package/dist/storage/PostgresBucketStorageFactory.js +57 -52
  4. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  5. package/dist/storage/PostgresReportStorage.js.map +1 -1
  6. package/dist/storage/PostgresStorageProvider.js +1 -1
  7. package/dist/storage/PostgresStorageProvider.js.map +1 -1
  8. package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +16 -8
  9. package/dist/storage/PostgresSyncRulesStorage.js +48 -196
  10. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  11. package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +13 -5
  12. package/dist/storage/batch/PostgresBucketBatch.js +280 -29
  13. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  14. package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
  15. package/dist/storage/storage-index.js +1 -0
  16. package/dist/storage/storage-index.js.map +1 -1
  17. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
  18. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +89 -0
  19. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
  20. package/dist/utils/test-utils.js +2 -2
  21. package/dist/utils/test-utils.js.map +1 -1
  22. package/package.json +10 -22
  23. package/src/storage/PostgresBucketStorageFactory.ts +64 -54
  24. package/src/storage/PostgresReportStorage.ts +1 -2
  25. package/src/storage/PostgresStorageProvider.ts +1 -1
  26. package/src/storage/PostgresSyncRulesStorage.ts +51 -211
  27. package/src/storage/batch/PostgresBucketBatch.ts +325 -45
  28. package/src/storage/storage-index.ts +1 -0
  29. package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +97 -0
  30. package/src/utils/test-utils.ts +2 -2
  31. package/test/src/migrations.test.ts +1 -1
  32. package/test/src/storage.test.ts +7 -2
  33. package/test/src/storage_compacting.test.ts +6 -4
  34. package/test/src/storage_sync.test.ts +2 -1
  35. package/test/tsconfig.json +0 -2
  36. package/tsconfig.json +1 -4
  37. package/tsconfig.tsbuildinfo +1 -0
  38. package/dist/.tsbuildinfo +0 -1
  39. package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
  40. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
  41. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
  42. package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
  43. /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
  44. /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
  45. /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
  46. /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
  47. /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
  48. /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
  49. /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
  50. /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
  51. /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
  52. /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
  53. /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
  54. /package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +0 -0
  55. /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
  56. /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
  57. /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
  58. /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
  59. /package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +0 -0
  60. /package/dist/{@types/storage → storage}/current-data-store.d.ts +0 -0
  61. /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
  62. /package/dist/{@types/types → types}/codecs.d.ts +0 -0
  63. /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
  64. /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
  65. /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
  66. /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
  67. /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
  68. /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
  69. /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
  70. /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
  71. /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
  72. /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
  73. /package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +0 -0
  74. /package/dist/{@types/types → types}/models/json.d.ts +0 -0
  75. /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
  76. /package/dist/{@types/types → types}/types.d.ts +0 -0
  77. /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
  78. /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
  79. /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
  80. /package/dist/{@types/utils → utils}/db.d.ts +0 -0
  81. /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
  82. /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
  83. /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
@@ -12,6 +12,7 @@ import {
12
12
  import {
13
13
  BucketStorageMarkRecordUnavailable,
14
14
  CheckpointResult,
15
+ ColumnDescriptor,
15
16
  deserializeReplicaId,
16
17
  InternalOpId,
17
18
  storage,
@@ -20,8 +21,10 @@ import {
20
21
  import * as sync_rules from '@powersync/service-sync-rules';
21
22
  import * as timers from 'timers/promises';
22
23
  import * as t from 'ts-codec';
24
+ import * as uuid from 'uuid';
23
25
  import { bigint } from '../../types/codecs.js';
24
26
  import { CurrentBucket, V3CurrentDataDecoded } from '../../types/models/CurrentData.js';
27
+ import { SourceTableDecoded, StoredRelationId } from '../../types/models/SourceTable.js';
25
28
  import { models, RequiredOperationBatchLimits } from '../../types/types.js';
26
29
  import { NOTIFICATION_CHANNEL } from '../../utils/db.js';
27
30
  import { pick } from '../../utils/ts-codec.js';
@@ -34,9 +37,9 @@ import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
34
37
  export interface PostgresBucketBatchOptions {
35
38
  logger: Logger;
36
39
  db: lib_postgres.DatabaseClient;
37
- sync_rules: sync_rules.HydratedSyncRules;
38
- group_id: number;
39
- slot_name: string;
40
+ sync_rules: sync_rules.HydratedSyncConfig;
41
+ replicationStreamId: number;
42
+ replicationStreamName: string;
40
43
  last_checkpoint_lsn: string | null;
41
44
  store_current_data: boolean;
42
45
  keep_alive_op?: InternalOpId | null;
@@ -48,6 +51,7 @@ export interface PostgresBucketBatchOptions {
48
51
  batch_limits: RequiredOperationBatchLimits;
49
52
 
50
53
  markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
54
+ hooks: storage.StorageHooks | undefined;
51
55
  storageConfig: storage.StorageVersionConfig;
52
56
  }
53
57
 
@@ -85,6 +89,7 @@ export class PostgresBucketBatch
85
89
  public last_flushed_op: InternalOpId | null = null;
86
90
 
87
91
  public resumeFromLsn: string | null;
92
+ public readonly skipExistingRows: boolean;
88
93
 
89
94
  protected db: lib_postgres.DatabaseClient;
90
95
  protected group_id: number;
@@ -93,10 +98,11 @@ export class PostgresBucketBatch
93
98
  protected persisted_op: InternalOpId | null;
94
99
 
95
100
  protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[];
96
- protected readonly sync_rules: sync_rules.HydratedSyncRules;
101
+ protected readonly sync_rules: sync_rules.HydratedSyncConfig;
97
102
  protected batch: OperationBatch | null;
98
103
  private lastWaitingLogThrottled = 0;
99
104
  private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
105
+ private hooks: storage.StorageHooks | undefined;
100
106
  private needsActivation = true;
101
107
  private clearedError = false;
102
108
  private readonly storageConfig: storage.StorageVersionConfig;
@@ -106,12 +112,14 @@ export class PostgresBucketBatch
106
112
  super();
107
113
  this.logger = options.logger;
108
114
  this.db = options.db;
109
- this.group_id = options.group_id;
115
+ this.group_id = options.replicationStreamId;
110
116
  this.last_checkpoint_lsn = options.last_checkpoint_lsn;
111
117
  this.resumeFromLsn = options.resumeFromLsn;
118
+ this.skipExistingRows = options.skip_existing_rows;
112
119
  this.write_checkpoint_batch = [];
113
120
  this.sync_rules = options.sync_rules;
114
121
  this.markRecordUnavailable = options.markRecordUnavailable;
122
+ this.hooks = options.hooks;
115
123
  this.batch = null;
116
124
  this.persisted_op = null;
117
125
  this.storageConfig = options.storageConfig;
@@ -121,10 +129,6 @@ export class PostgresBucketBatch
121
129
  }
122
130
  }
123
131
 
124
- get lastCheckpointLsn() {
125
- return this.last_checkpoint_lsn;
126
- }
127
-
128
132
  async [Symbol.asyncDispose]() {
129
133
  if (this.batch != null || this.write_checkpoint_batch.length > 0) {
130
134
  // We don't error here, since:
@@ -139,22 +143,237 @@ export class PostgresBucketBatch
139
143
  await this[Symbol.asyncDispose]();
140
144
  }
141
145
 
146
+ async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
147
+ const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
148
+ const { connection_id, source } = options;
149
+ const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
150
+
151
+ const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
152
+ name: column.name,
153
+ type: column.type,
154
+ type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
155
+ }));
156
+
157
+ return this.db.transaction(async (db) => {
158
+ let sourceTableRow: SourceTableDecoded | null;
159
+ if (objectId != null) {
160
+ sourceTableRow = await db.sql`
161
+ SELECT
162
+ *
163
+ FROM
164
+ source_tables
165
+ WHERE
166
+ group_id = ${{ type: 'int4', value: this.group_id }}
167
+ AND connection_id = ${{ type: 'int4', value: connection_id }}
168
+ AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
169
+ AND schema_name = ${{ type: 'varchar', value: schema }}
170
+ AND table_name = ${{ type: 'varchar', value: table }}
171
+ AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
172
+ `
173
+ .decoded(models.SourceTable)
174
+ .first();
175
+ } else {
176
+ sourceTableRow = await db.sql`
177
+ SELECT
178
+ *
179
+ FROM
180
+ source_tables
181
+ WHERE
182
+ group_id = ${{ type: 'int4', value: this.group_id }}
183
+ AND connection_id = ${{ type: 'int4', value: connection_id }}
184
+ AND schema_name = ${{ type: 'varchar', value: schema }}
185
+ AND table_name = ${{ type: 'varchar', value: table }}
186
+ AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
187
+ `
188
+ .decoded(models.SourceTable)
189
+ .first();
190
+ }
191
+
192
+ if (sourceTableRow == null) {
193
+ const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
194
+ sourceTableRow = await db.sql`
195
+ INSERT INTO
196
+ source_tables (
197
+ id,
198
+ group_id,
199
+ connection_id,
200
+ relation_id,
201
+ schema_name,
202
+ table_name,
203
+ replica_id_columns
204
+ )
205
+ VALUES
206
+ (
207
+ ${{ type: 'varchar', value: id }},
208
+ ${{ type: 'int4', value: this.group_id }},
209
+ ${{ type: 'int4', value: connection_id }},
210
+ ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
211
+ ${{ type: 'varchar', value: schema }},
212
+ ${{ type: 'varchar', value: table }},
213
+ ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
214
+ )
215
+ RETURNING
216
+ *
217
+ `
218
+ .decoded(models.SourceTable)
219
+ .first();
220
+ }
221
+
222
+ const sourceTable = new storage.SourceTable({
223
+ id: sourceTableRow!.id,
224
+ ref: source,
225
+ objectId,
226
+ replicaIdColumns,
227
+ snapshotComplete: sourceTableRow!.snapshot_done ?? true,
228
+ ...syncRules.getMatchingSources(source)
229
+ });
230
+ if (!sourceTable.snapshotComplete) {
231
+ sourceTable.snapshotStatus = {
232
+ totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
233
+ replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
234
+ lastKey: sourceTableRow!.snapshot_last_key
235
+ };
236
+ }
237
+ sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
238
+ sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
239
+ sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
240
+ sourceTable.storeCurrentData = sendsCompleteRows !== true;
241
+
242
+ let truncatedTables: SourceTableDecoded[] = [];
243
+ if (objectId != null) {
244
+ truncatedTables = await db.sql`
245
+ SELECT
246
+ *
247
+ FROM
248
+ source_tables
249
+ WHERE
250
+ group_id = ${{ type: 'int4', value: this.group_id }}
251
+ AND connection_id = ${{ type: 'int4', value: connection_id }}
252
+ AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
253
+ AND (
254
+ relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
255
+ OR (
256
+ schema_name = ${{ type: 'varchar', value: schema }}
257
+ AND table_name = ${{ type: 'varchar', value: table }}
258
+ )
259
+ )
260
+ `
261
+ .decoded(models.SourceTable)
262
+ .rows();
263
+ } else {
264
+ truncatedTables = await db.sql`
265
+ SELECT
266
+ *
267
+ FROM
268
+ source_tables
269
+ WHERE
270
+ group_id = ${{ type: 'int4', value: this.group_id }}
271
+ AND connection_id = ${{ type: 'int4', value: connection_id }}
272
+ AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
273
+ AND (
274
+ schema_name = ${{ type: 'varchar', value: schema }}
275
+ AND table_name = ${{ type: 'varchar', value: table }}
276
+ )
277
+ `
278
+ .decoded(models.SourceTable)
279
+ .rows();
280
+ }
281
+
282
+ return {
283
+ tables: [sourceTable],
284
+ dropTables: truncatedTables.map((doc) => {
285
+ const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
286
+ const dropTable = new storage.SourceTable({
287
+ id: doc.id,
288
+ ref,
289
+ objectId: doc.relation_id?.object_id ?? 0,
290
+ replicaIdColumns:
291
+ doc.replica_id_columns?.map(
292
+ (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
293
+ ) ?? [],
294
+ snapshotComplete: doc.snapshot_done ?? true,
295
+ ...syncRules.getMatchingSources(ref)
296
+ });
297
+ dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
298
+ dropTable.syncData = dropTable.bucketDataSources.length > 0;
299
+ dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
300
+ return dropTable;
301
+ })
302
+ };
303
+ });
304
+ }
305
+
306
+ async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
307
+ const row = await this.db.sql`
308
+ SELECT
309
+ *
310
+ FROM
311
+ source_tables
312
+ WHERE
313
+ group_id = ${{ type: 'int4', value: this.group_id }}
314
+ AND id = ${{ type: 'varchar', value: table.id.toString() }}
315
+ `
316
+ .decoded(models.SourceTable)
317
+ .first();
318
+
319
+ if (row == null) {
320
+ return null;
321
+ }
322
+
323
+ return this.sourceTableFromRow(row, table.ref.connectionTag, this.sync_rules);
324
+ }
325
+
326
+ private sourceTableFromRow(
327
+ row: SourceTableDecoded,
328
+ connectionTag: string,
329
+ syncRules: sync_rules.HydratedSyncConfig
330
+ ): storage.SourceTable {
331
+ const ref = { connectionTag, schema: row.schema_name, name: row.table_name };
332
+ const sourceTable = new storage.SourceTable({
333
+ id: row.id,
334
+ ref,
335
+ objectId: row.relation_id?.object_id,
336
+ replicaIdColumns:
337
+ row.replica_id_columns?.map(
338
+ (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
339
+ ) ?? [],
340
+ snapshotComplete: row.snapshot_done ?? true,
341
+ ...syncRules.getMatchingSources(ref)
342
+ });
343
+
344
+ if (!sourceTable.snapshotComplete) {
345
+ sourceTable.snapshotStatus = {
346
+ totalEstimatedCount: Number(row.snapshot_total_estimated_count ?? -1n),
347
+ replicatedCount: Number(row.snapshot_replicated_count ?? 0n),
348
+ lastKey: row.snapshot_last_key
349
+ };
350
+ }
351
+ sourceTable.syncEvent = syncRules.tableTriggersEvent(ref);
352
+ sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
353
+ sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
354
+ return sourceTable;
355
+ }
356
+
142
357
  async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
143
358
  // TODO maybe share with abstract class
144
359
  const { after, before, sourceTable, tag } = record;
145
- for (const event of this.getTableEvents(sourceTable)) {
146
- this.iterateListeners((cb) =>
147
- cb.replicationEvent?.({
148
- batch: this,
149
- table: sourceTable,
150
- data: {
151
- op: tag,
152
- after: after && utils.isCompleteRow(this.options.store_current_data, after) ? after : undefined,
153
- before: before && utils.isCompleteRow(this.options.store_current_data, before) ? before : undefined
154
- },
155
- event
156
- })
157
- );
360
+ const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
361
+ // Only the table designated as event carrier by resolveTables may fire events.
362
+ if (sourceTable.syncEvent) {
363
+ for (const event of this.getTableEvents(sourceTable)) {
364
+ this.iterateListeners((cb) =>
365
+ cb.replicationEvent?.({
366
+ batch: this,
367
+ table: sourceTable,
368
+ data: {
369
+ op: tag,
370
+ after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
371
+ before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
372
+ },
373
+ event
374
+ })
375
+ );
376
+ }
158
377
  }
159
378
  /**
160
379
  * Return if the table is just an event table
@@ -296,6 +515,8 @@ export class PostgresBucketBatch
296
515
  return null;
297
516
  }
298
517
 
518
+ await this.hooks?.beforeBatchFlush?.(this);
519
+
299
520
  let resumeBatch: OperationBatch | null = null;
300
521
 
301
522
  const lastOp = await this.withReplicationTransaction(async (db) => {
@@ -313,6 +534,7 @@ export class PostgresBucketBatch
313
534
 
314
535
  this.persisted_op = lastOp;
315
536
  this.last_flushed_op = lastOp;
537
+ await this.hooks?.afterBatchFlush?.(this);
316
538
  return { flushed_op: lastOp };
317
539
  }
318
540
 
@@ -536,6 +758,49 @@ export class PostgresBucketBatch
536
758
  });
537
759
  }
538
760
 
761
+ async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
762
+ await this.db.transaction(async (db) => {
763
+ const blockingTables = await db.sql`
764
+ SELECT
765
+ schema_name,
766
+ table_name
767
+ FROM
768
+ source_tables
769
+ WHERE
770
+ group_id = ${{ type: 'int4', value: this.group_id }}
771
+ AND snapshot_done = FALSE
772
+ `
773
+ .decoded(t.object({ schema_name: t.string, table_name: t.string }))
774
+ .rows();
775
+ if (blockingTables.length > 0) {
776
+ if (options?.throwOnConflict ?? true) {
777
+ throw new ReplicationAssertionError(
778
+ `Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
779
+ );
780
+ } else {
781
+ return;
782
+ }
783
+ }
784
+
785
+ await db.sql`
786
+ UPDATE sync_rules
787
+ SET
788
+ snapshot_done = TRUE,
789
+ last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }},
790
+ no_checkpoint_before = CASE
791
+ WHEN no_checkpoint_before IS NULL
792
+ OR no_checkpoint_before < ${{ type: 'varchar', value: no_checkpoint_before_lsn }} THEN ${{
793
+ type: 'varchar',
794
+ value: no_checkpoint_before_lsn
795
+ }}
796
+ ELSE no_checkpoint_before
797
+ END
798
+ WHERE
799
+ id = ${{ type: 'int4', value: this.group_id }}
800
+ `.execute();
801
+ });
802
+ }
803
+
539
804
  async markTableSnapshotRequired(table: storage.SourceTable): Promise<void> {
540
805
  await this.db.sql`
541
806
  UPDATE sync_rules
@@ -628,8 +893,12 @@ export class PostgresBucketBatch
628
893
 
629
894
  protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) {
630
895
  let sizes: Map<string, number> | undefined = undefined;
631
- if (this.options.store_current_data && !this.options.skip_existing_rows) {
632
- // We skip this step if we don't store current_data, since the sizes will
896
+ // Check if any table in this batch needs to store current_data
897
+ const anyTableStoresCurrentData =
898
+ this.options.store_current_data && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
899
+
900
+ if (anyTableStoresCurrentData && !this.options.skip_existing_rows) {
901
+ // We skip this step if no tables store current_data, since the sizes will
633
902
  // always be small in that case.
634
903
 
635
904
  // With skipExistingRows, we don't load the full documents into memory,
@@ -637,15 +906,19 @@ export class PostgresBucketBatch
637
906
 
638
907
  // Find sizes of current_data documents, to assist in intelligent batching without
639
908
  // exceeding memory limits.
640
- const sizeLookups = batch.batch.map((r) => {
641
- return {
642
- source_table: postgresTableId(r.record.sourceTable.id),
643
- /**
644
- * Encode to hex in order to pass a jsonb
645
- */
646
- source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
647
- };
648
- });
909
+ // Within this branch the batch stores current_data, so the per-table flag is the
910
+ // effective value - only look up sizes for tables that actually store current_data.
911
+ const sizeLookups = batch.batch
912
+ .filter((r) => r.record.sourceTable.storeCurrentData)
913
+ .map((r) => {
914
+ return {
915
+ source_table: postgresTableId(r.record.sourceTable.id),
916
+ /**
917
+ * Encode to hex in order to pass a jsonb
918
+ */
919
+ source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
920
+ };
921
+ });
649
922
 
650
923
  sizes = new Map<string, number>();
651
924
 
@@ -764,6 +1037,9 @@ export class PostgresBucketBatch
764
1037
  const afterId = operation.afterId;
765
1038
  let after = record.after;
766
1039
  const sourceTable = record.sourceTable;
1040
+ // Effective per-record flag: store current_data only if both the batch (source-level)
1041
+ // and the table (e.g. non-FULL replica identity) require it.
1042
+ const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
767
1043
 
768
1044
  let existingBuckets: CurrentBucket[] = [];
769
1045
  let newBuckets: CurrentBucket[] = [];
@@ -792,7 +1068,7 @@ export class PostgresBucketBatch
792
1068
  existingLookups = [];
793
1069
  // Log to help with debugging if there was a consistency issue
794
1070
 
795
- if (this.options.store_current_data) {
1071
+ if (storeCurrentData) {
796
1072
  if (this.markRecordUnavailable != null) {
797
1073
  // This will trigger a "resnapshot" of the record.
798
1074
  // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -808,7 +1084,7 @@ export class PostgresBucketBatch
808
1084
  } else {
809
1085
  existingBuckets = result.buckets;
810
1086
  existingLookups = result.lookups;
811
- if (this.options.store_current_data) {
1087
+ if (storeCurrentData) {
812
1088
  const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow;
813
1089
  after = storage.mergeToast(after!, data);
814
1090
  }
@@ -819,7 +1095,9 @@ export class PostgresBucketBatch
819
1095
  // Not an error if we re-apply a transaction
820
1096
  existingBuckets = [];
821
1097
  existingLookups = [];
822
- // Log to help with debugging if there was a consistency issue
1098
+ // Log to help with debugging if there was a consistency issue.
1099
+ // Gate on the batch-level flag: FULL tables (per-record flag false) still get a
1100
+ // current_data entry, so a missing record on DELETE is meaningful for them too.
823
1101
  if (this.options.store_current_data && this.markRecordUnavailable == null) {
824
1102
  this.logger.warn(
825
1103
  `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
@@ -832,7 +1110,7 @@ export class PostgresBucketBatch
832
1110
  }
833
1111
 
834
1112
  let afterData: Buffer<ArrayBuffer> | undefined;
835
- if (afterId != null && !this.options.store_current_data) {
1113
+ if (afterId != null && !storeCurrentData) {
836
1114
  afterData = storage.serializeBson({});
837
1115
  } else if (afterId != null) {
838
1116
  try {
@@ -854,7 +1132,7 @@ export class PostgresBucketBatch
854
1132
  {
855
1133
  level: errors.ErrorSeverity.WARNING,
856
1134
  metadata: {
857
- replication_slot: this.options.slot_name,
1135
+ replication_slot: this.options.replicationStreamName,
858
1136
  table: record.sourceTable.qualifiedName
859
1137
  }
860
1138
  }
@@ -895,12 +1173,13 @@ export class PostgresBucketBatch
895
1173
  // However, it will be valid by the end of the transaction.
896
1174
  //
897
1175
  // In this case, we don't save the op, but we do save the current data.
898
- if (afterId && after && utils.isCompleteRow(this.options.store_current_data, after)) {
1176
+ if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
899
1177
  // Insert or update
900
1178
  if (sourceTable.syncData) {
901
1179
  const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
902
1180
  record: after,
903
- sourceTable
1181
+ sourceTable: sourceTable.ref,
1182
+ bucketDataSources: sourceTable.bucketDataSources
904
1183
  });
905
1184
 
906
1185
  for (const error of syncErrors) {
@@ -909,7 +1188,7 @@ export class PostgresBucketBatch
909
1188
  {
910
1189
  level: errors.ErrorSeverity.WARNING,
911
1190
  metadata: {
912
- replication_slot: this.options.slot_name,
1191
+ replication_slot: this.options.replicationStreamName,
913
1192
  table: record.sourceTable.qualifiedName
914
1193
  }
915
1194
  }
@@ -939,8 +1218,9 @@ export class PostgresBucketBatch
939
1218
  if (sourceTable.syncParameters) {
940
1219
  // Parameters
941
1220
  const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors(
942
- sourceTable,
943
- after
1221
+ sourceTable.ref,
1222
+ after,
1223
+ { parameterLookupSources: sourceTable.parameterLookupSources }
944
1224
  );
945
1225
 
946
1226
  for (let error of paramErrors) {
@@ -949,7 +1229,7 @@ export class PostgresBucketBatch
949
1229
  {
950
1230
  level: errors.ErrorSeverity.WARNING,
951
1231
  metadata: {
952
- replication_slot: this.options.slot_name,
1232
+ replication_slot: this.options.replicationStreamName,
953
1233
  table: record.sourceTable.qualifiedName
954
1234
  }
955
1235
  }
@@ -1064,7 +1344,7 @@ export class PostgresBucketBatch
1064
1344
  */
1065
1345
  protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
1066
1346
  return this.sync_rules.eventDescriptors.filter((evt) =>
1067
- [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
1347
+ [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
1068
1348
  );
1069
1349
  }
1070
1350
 
@@ -1,4 +1,5 @@
1
1
  export * from './PostgresBucketStorageFactory.js';
2
2
  export * from './PostgresCompactor.js';
3
+ export * from './PostgresReportStorage.js';
3
4
  export * from './PostgresStorageProvider.js';
4
5
  export * from './PostgresSyncRulesStorage.js';
@@ -0,0 +1,97 @@
1
+ import * as lib_postgres from '@powersync/lib-service-postgres';
2
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
3
+ import { storage } from '@powersync/service-core';
4
+ import { models } from '../../types/types.js';
5
+
6
+ export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
7
+ constructor(
8
+ private db: lib_postgres.DatabaseClient,
9
+ row: models.SyncRulesDecoded
10
+ ) {
11
+ super({
12
+ replicationStreamId: Number(row.id),
13
+ sync_rules_content: row.content,
14
+ compiled_plan: row.sync_plan,
15
+ replicationStreamName: row.slot_name,
16
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
17
+ });
18
+ }
19
+
20
+ async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
21
+ const row = await this.db.sql`
22
+ SELECT
23
+ *
24
+ FROM
25
+ sync_rules
26
+ WHERE
27
+ id = ${{ value: this.replicationStreamId, type: 'int4' }}
28
+ `
29
+ .decoded(models.SyncRules)
30
+ .first();
31
+
32
+ return row == null ? null : syncConfigStatusFromRow(row);
33
+ }
34
+ }
35
+
36
+ function syncConfigStatusFromRow(row: models.SyncRulesDecoded): storage.PersistedSyncConfigStatus {
37
+ return {
38
+ id: String(row.id),
39
+ replicationStreamId: Number(row.id),
40
+ state: row.state as storage.SyncRuleState,
41
+ snapshot_done: row.snapshot_done,
42
+ last_checkpoint_lsn: row.last_checkpoint_lsn,
43
+ last_fatal_error: row.last_fatal_error,
44
+ last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
45
+ last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
46
+ };
47
+ }
48
+
49
+ export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
50
+ current_lock: storage.ReplicationLock | null = null;
51
+ readonly syncConfigContent: readonly PostgresPersistedSyncConfigContent[];
52
+
53
+ constructor(
54
+ private db: lib_postgres.DatabaseClient,
55
+ private readonly row: models.SyncRulesDecoded
56
+ ) {
57
+ super({
58
+ replicationStreamId: Number(row.id),
59
+ replicationStreamName: row.slot_name,
60
+ state: row.state as storage.SyncRuleState,
61
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
62
+ });
63
+ this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
64
+ }
65
+
66
+ parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
67
+ return this.syncConfigContent[0].parsed(options);
68
+ }
69
+
70
+ async lock(): Promise<storage.ReplicationLock> {
71
+ const manager = new lib_postgres.PostgresLockManager({
72
+ db: this.db,
73
+ name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
74
+ });
75
+ const lockHandle = await manager.acquire();
76
+ if (!lockHandle) {
77
+ throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
78
+ }
79
+
80
+ const interval = setInterval(async () => {
81
+ try {
82
+ await lockHandle.refresh();
83
+ } catch (e) {
84
+ this.logger.error('Failed to refresh lock', e);
85
+ clearInterval(interval);
86
+ }
87
+ }, 30_130);
88
+
89
+ return (this.current_lock = {
90
+ sync_rules_id: this.replicationStreamId,
91
+ release: async () => {
92
+ clearInterval(interval);
93
+ return lockHandle.release();
94
+ }
95
+ });
96
+ }
97
+ }
@@ -73,7 +73,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
73
73
 
74
74
  await using storageFactory = new PostgresBucketStorageFactory({
75
75
  config: TEST_CONNECTION_OPTIONS,
76
- slot_name_prefix: 'test_'
76
+ replicationStreamNamePrefix: 'test_'
77
77
  });
78
78
  await truncateTables(storageFactory.db);
79
79
  };
@@ -102,7 +102,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
102
102
 
103
103
  return new PostgresBucketStorageFactory({
104
104
  config: TEST_CONNECTION_OPTIONS,
105
- slot_name_prefix: 'test_'
105
+ replicationStreamNamePrefix: 'test_'
106
106
  });
107
107
  } catch (ex) {
108
108
  // Vitest does not display these errors nicely when using the `await using` syntx
@@ -18,7 +18,7 @@ describe('Migrations', () => {
18
18
  const setup = POSTGRES_STORAGE_SETUP;
19
19
  await using factory = new PostgresBucketStorageFactory({
20
20
  config: TEST_CONNECTION_OPTIONS,
21
- slot_name_prefix: 'test_'
21
+ replicationStreamNamePrefix: 'test_'
22
22
  });
23
23
 
24
24
  await dropTables(factory.db);