@powersync/service-module-postgres-storage 0.15.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +12 -12
  3. package/dist/storage/PostgresBucketStorageFactory.js +57 -52
  4. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  5. package/dist/storage/PostgresReportStorage.js.map +1 -1
  6. package/dist/storage/PostgresStorageProvider.js +1 -1
  7. package/dist/storage/PostgresStorageProvider.js.map +1 -1
  8. package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +16 -7
  9. package/dist/storage/PostgresSyncRulesStorage.js +48 -45
  10. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  11. package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +2 -3
  12. package/dist/storage/batch/PostgresBucketBatch.js +26 -25
  13. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  14. package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
  15. package/dist/storage/storage-index.js +1 -0
  16. package/dist/storage/storage-index.js.map +1 -1
  17. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
  18. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +90 -0
  19. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
  20. package/dist/utils/test-utils.js +2 -2
  21. package/dist/utils/test-utils.js.map +1 -1
  22. package/package.json +10 -22
  23. package/src/storage/PostgresBucketStorageFactory.ts +64 -54
  24. package/src/storage/PostgresReportStorage.ts +1 -2
  25. package/src/storage/PostgresStorageProvider.ts +1 -1
  26. package/src/storage/PostgresSyncRulesStorage.ts +51 -52
  27. package/src/storage/batch/PostgresBucketBatch.ts +30 -32
  28. package/src/storage/storage-index.ts +1 -0
  29. package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +98 -0
  30. package/src/utils/test-utils.ts +2 -2
  31. package/test/src/migrations.test.ts +1 -1
  32. package/test/src/storage.test.ts +7 -2
  33. package/test/src/storage_compacting.test.ts +6 -4
  34. package/test/src/storage_sync.test.ts +2 -1
  35. package/test/tsconfig.json +0 -2
  36. package/tsconfig.json +1 -4
  37. package/tsconfig.tsbuildinfo +1 -0
  38. package/dist/.tsbuildinfo +0 -1
  39. package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
  40. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
  41. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
  42. package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
  43. /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
  44. /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
  45. /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
  46. /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
  47. /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
  48. /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
  49. /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
  50. /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
  51. /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
  52. /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
  53. /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
  54. /package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +0 -0
  55. /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
  56. /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
  57. /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
  58. /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
  59. /package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +0 -0
  60. /package/dist/{@types/storage → storage}/current-data-store.d.ts +0 -0
  61. /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
  62. /package/dist/{@types/types → types}/codecs.d.ts +0 -0
  63. /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
  64. /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
  65. /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
  66. /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
  67. /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
  68. /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
  69. /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
  70. /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
  71. /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
  72. /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
  73. /package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +0 -0
  74. /package/dist/{@types/types → types}/models/json.d.ts +0 -0
  75. /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
  76. /package/dist/{@types/types → types}/types.d.ts +0 -0
  77. /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
  78. /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
  79. /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
  80. /package/dist/{@types/utils → utils}/db.d.ts +0 -0
  81. /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
  82. /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
  83. /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
@@ -41,7 +41,7 @@ import { PostgresCompactor } from './PostgresCompactor.js';
41
41
  export type PostgresSyncRulesStorageOptions = {
42
42
  factory: PostgresBucketStorageFactory;
43
43
  db: lib_postgres.DatabaseClient;
44
- sync_rules: storage.PersistedSyncRulesContent;
44
+ replicationStream: storage.PersistedReplicationStream;
45
45
  write_checkpoint_mode?: storage.WriteCheckpointMode;
46
46
  batchLimits: RequiredOperationBatchLimits;
47
47
  };
@@ -52,9 +52,10 @@ export class PostgresSyncRulesStorage
52
52
  {
53
53
  [framework.DO_NOT_LOG] = true;
54
54
 
55
- public readonly group_id: number;
56
- public readonly sync_rules: storage.PersistedSyncRulesContent;
57
- public readonly slot_name: string;
55
+ public readonly replicationStreamId: number;
56
+ public readonly replicationStream: storage.PersistedReplicationStream;
57
+ public readonly sync_rules: storage.PersistedSyncConfigContent;
58
+ public readonly replicationStreamName: string;
58
59
  public readonly factory: PostgresBucketStorageFactory;
59
60
  public readonly storageConfig: StorageVersionConfig;
60
61
  public readonly logger: framework.Logger;
@@ -65,22 +66,27 @@ export class PostgresSyncRulesStorage
65
66
  protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
66
67
  private readonly currentDataStore: PostgresCurrentDataStore;
67
68
 
68
- // TODO we might be able to share this in an abstract class
69
- private parsedSyncRulesCache:
70
- | { parsed: sync_rules.HydratedSyncConfig; options: storage.ParseSyncRulesOptions }
71
- | undefined;
69
+ /**
70
+ * Canonical parsed sync config sets, keyed by defaultSchema. Entries are never evicted,
71
+ * so each parse options value maps to exactly one parsed set for the lifetime of this
72
+ * storage instance.
73
+ *
74
+ * TODO we might be able to share this in an abstract class
75
+ */
76
+ private readonly parsedSyncConfigSets = new Map<string, storage.ParsedSyncConfigSet>();
72
77
  private _checksumCache: storage.ChecksumCache | undefined;
73
78
 
74
79
  constructor(protected options: PostgresSyncRulesStorageOptions) {
75
80
  super();
76
- this.group_id = options.sync_rules.id;
81
+ this.replicationStream = options.replicationStream;
82
+ this.replicationStreamId = options.replicationStream.replicationStreamId;
77
83
  this.db = options.db;
78
- this.sync_rules = options.sync_rules;
79
- this.slot_name = options.sync_rules.slot_name;
84
+ this.sync_rules = options.replicationStream.syncConfigContent[0];
85
+ this.replicationStreamName = options.replicationStream.replicationStreamName;
80
86
  this.factory = options.factory;
81
- this.storageConfig = options.sync_rules.getStorageConfig();
87
+ this.storageConfig = options.replicationStream.getStorageConfig();
82
88
  this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
83
- this.logger = options.sync_rules.logger;
89
+ this.logger = options.replicationStream.logger;
84
90
 
85
91
  this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
86
92
  db: this.db,
@@ -106,18 +112,17 @@ export class PostgresSyncRulesStorage
106
112
  return this.writeCheckpointAPI.writeCheckpointMode;
107
113
  }
108
114
 
109
- // TODO we might be able to share this in an abstract class
110
- getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncConfig {
111
- const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
112
- /**
113
- * Check if the cached sync config, if present, had the same options.
114
- * Parse sync config if the options are different or if there is no cached value.
115
- */
116
- if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
117
- this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncConfig(), options };
115
+ getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
116
+ let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
117
+ if (parsed == null) {
118
+ parsed = this.replicationStream.parsed(options);
119
+ this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
118
120
  }
121
+ return parsed;
122
+ }
119
123
 
120
- return this.parsedSyncRulesCache!.parsed;
124
+ getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig {
125
+ return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
121
126
  }
122
127
 
123
128
  async reportError(e: any): Promise<void> {
@@ -127,7 +132,7 @@ export class PostgresSyncRulesStorage
127
132
  SET
128
133
  last_fatal_error = ${{ type: 'varchar', value: message }}
129
134
  WHERE
130
- id = ${{ type: 'int4', value: this.group_id }};
135
+ id = ${{ type: 'int4', value: this.replicationStreamId }};
131
136
  `.execute();
132
137
  }
133
138
 
@@ -139,7 +144,7 @@ export class PostgresSyncRulesStorage
139
144
  maxOpId = checkpoint.checkpoint;
140
145
  }
141
146
 
142
- return new PostgresCompactor(this.db, this.group_id, {
147
+ return new PostgresCompactor(this.db, this.replicationStreamId, {
143
148
  ...options,
144
149
  maxOpId,
145
150
  logger: this.logger
@@ -154,7 +159,7 @@ export class PostgresSyncRulesStorage
154
159
  lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
155
160
  return this.writeCheckpointAPI.lastWriteCheckpoint({
156
161
  ...filters,
157
- sync_rules_id: this.group_id
162
+ sync_rules_id: this.replicationStreamId
158
163
  });
159
164
  }
160
165
 
@@ -174,7 +179,7 @@ export class PostgresSyncRulesStorage
174
179
  FROM
175
180
  sync_rules
176
181
  WHERE
177
- id = ${{ type: 'int4', value: this.group_id }}
182
+ id = ${{ type: 'int4', value: this.replicationStreamId }}
178
183
  `
179
184
  .decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn']))
180
185
  .first();
@@ -196,7 +201,7 @@ export class PostgresSyncRulesStorage
196
201
  FROM
197
202
  sync_rules
198
203
  WHERE
199
- id = ${{ type: 'int4', value: this.group_id }}
204
+ id = ${{ type: 'int4', value: this.replicationStreamId }}
200
205
  `
201
206
  .decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op', 'snapshot_lsn']))
202
207
  .first();
@@ -206,9 +211,9 @@ export class PostgresSyncRulesStorage
206
211
  const writer = new PostgresBucketBatch({
207
212
  logger: options.logger ?? this.logger,
208
213
  db: this.db,
209
- sync_rules: this.sync_rules.parsed(options).hydratedSyncConfig(),
210
- group_id: this.group_id,
211
- slot_name: this.slot_name,
214
+ sync_rules: this.getParsedSyncRules(options),
215
+ replicationStreamId: this.replicationStreamId,
216
+ replicationStreamName: this.replicationStreamName,
212
217
  last_checkpoint_lsn: checkpoint_lsn,
213
218
  keep_alive_op: syncRules?.keepalive_op,
214
219
  resumeFromLsn: maxLsn(syncRules?.snapshot_lsn, checkpoint_lsn),
@@ -254,7 +259,7 @@ export class PostgresSyncRulesStorage
254
259
  value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
255
260
  }}) WITH ORDINALITY AS requested (value, index)
256
261
  WHERE
257
- group_id = ${{ type: 'int4', value: this.group_id }}
262
+ group_id = ${{ type: 'int4', value: this.replicationStreamId }}
258
263
  AND lookup = decode((requested.value ->> 0)::text, 'hex') -- Decode the hex string to bytea
259
264
  AND id <= ${{ type: 'int8', value: checkpoint.checkpoint }}
260
265
  ORDER BY
@@ -381,7 +386,7 @@ export class PostgresSyncRulesStorage
381
386
  LIMIT
382
387
  $3;`,
383
388
  params: [
384
- { type: 'int4', value: this.group_id },
389
+ { type: 'int4', value: this.replicationStreamId },
385
390
  { type: 'int8', value: end },
386
391
  { type: 'int4', value: batchRowLimit },
387
392
  ...filters.flatMap((f) => [
@@ -497,26 +502,23 @@ export class PostgresSyncRulesStorage
497
502
  state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }},
498
503
  snapshot_done = ${{ type: 'bool', value: false }}
499
504
  WHERE
500
- id = ${{ type: 'int4', value: this.group_id }}
505
+ id = ${{ type: 'int4', value: this.replicationStreamId }}
501
506
  `.execute();
502
507
  }
503
508
 
504
- async getStatus(): Promise<storage.SyncRuleStatus> {
509
+ async getStatus(): Promise<storage.ReplicationStreamStatus> {
505
510
  const syncRulesRow = await this.db.sql`
506
511
  SELECT
507
512
  snapshot_done,
508
513
  snapshot_lsn,
509
514
  last_checkpoint_lsn,
510
- state,
511
- keepalive_op
515
+ state
512
516
  FROM
513
517
  sync_rules
514
518
  WHERE
515
- id = ${{ type: 'int4', value: this.group_id }}
519
+ id = ${{ type: 'int4', value: this.replicationStreamId }}
516
520
  `
517
- .decoded(
518
- pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state', 'snapshot_lsn', 'keepalive_op'])
519
- )
521
+ .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'snapshot_lsn']))
520
522
  .first();
521
523
 
522
524
  if (syncRulesRow == null) {
@@ -524,11 +526,8 @@ export class PostgresSyncRulesStorage
524
526
  }
525
527
 
526
528
  return {
527
- snapshot_done: syncRulesRow.snapshot_done,
528
- active: syncRulesRow.state == storage.SyncRuleState.ACTIVE,
529
- checkpoint_lsn: syncRulesRow.last_checkpoint_lsn ?? null,
530
- snapshot_lsn: syncRulesRow.snapshot_lsn ?? null,
531
- keepalive_op: syncRulesRow.keepalive_op ?? null
529
+ snapshotDone: syncRulesRow.snapshot_done && syncRulesRow.last_checkpoint_lsn != null,
530
+ resumeLsn: maxLsn(syncRulesRow.snapshot_lsn, syncRulesRow.last_checkpoint_lsn)
532
531
  };
533
532
  }
534
533
 
@@ -542,27 +541,27 @@ export class PostgresSyncRulesStorage
542
541
  last_checkpoint = NULL,
543
542
  no_checkpoint_before = NULL
544
543
  WHERE
545
- id = ${{ type: 'int4', value: this.group_id }}
544
+ id = ${{ type: 'int4', value: this.replicationStreamId }}
546
545
  `.execute();
547
546
 
548
547
  await this.db.sql`
549
548
  DELETE FROM bucket_data
550
549
  WHERE
551
- group_id = ${{ type: 'int4', value: this.group_id }}
550
+ group_id = ${{ type: 'int4', value: this.replicationStreamId }}
552
551
  `.execute();
553
552
 
554
553
  await this.db.sql`
555
554
  DELETE FROM bucket_parameters
556
555
  WHERE
557
- group_id = ${{ type: 'int4', value: this.group_id }}
556
+ group_id = ${{ type: 'int4', value: this.replicationStreamId }}
558
557
  `.execute();
559
558
 
560
- await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.group_id });
559
+ await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.replicationStreamId });
561
560
 
562
561
  await this.db.sql`
563
562
  DELETE FROM source_tables
564
563
  WHERE
565
- group_id = ${{ type: 'int4', value: this.group_id }}
564
+ group_id = ${{ type: 'int4', value: this.replicationStreamId }}
566
565
  `.execute();
567
566
  }
568
567
 
@@ -603,7 +602,7 @@ export class PostgresSyncRulesStorage
603
602
  AND b.op_id > f.start_op_id
604
603
  AND b.op_id <= f.end_op_id
605
604
  WHERE
606
- b.group_id = ${{ type: 'int4', value: this.group_id }}
605
+ b.group_id = ${{ type: 'int4', value: this.replicationStreamId }}
607
606
  GROUP BY
608
607
  b.bucket_name;
609
608
  `.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>();
@@ -38,8 +38,8 @@ export interface PostgresBucketBatchOptions {
38
38
  logger: Logger;
39
39
  db: lib_postgres.DatabaseClient;
40
40
  sync_rules: sync_rules.HydratedSyncConfig;
41
- group_id: number;
42
- slot_name: string;
41
+ replicationStreamId: number;
42
+ replicationStreamName: string;
43
43
  last_checkpoint_lsn: string | null;
44
44
  store_current_data: boolean;
45
45
  keep_alive_op?: InternalOpId | null;
@@ -112,7 +112,7 @@ export class PostgresBucketBatch
112
112
  super();
113
113
  this.logger = options.logger;
114
114
  this.db = options.db;
115
- this.group_id = options.group_id;
115
+ this.group_id = options.replicationStreamId;
116
116
  this.last_checkpoint_lsn = options.last_checkpoint_lsn;
117
117
  this.resumeFromLsn = options.resumeFromLsn;
118
118
  this.skipExistingRows = options.skip_existing_rows;
@@ -129,10 +129,6 @@ export class PostgresBucketBatch
129
129
  }
130
130
  }
131
131
 
132
- get lastCheckpointLsn() {
133
- return this.last_checkpoint_lsn;
134
- }
135
-
136
132
  async [Symbol.asyncDispose]() {
137
133
  if (this.batch != null || this.write_checkpoint_batch.length > 0) {
138
134
  // We don't error here, since:
@@ -148,7 +144,7 @@ export class PostgresBucketBatch
148
144
  }
149
145
 
150
146
  async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
151
- const syncRules = options.syncRules ?? this.sync_rules;
147
+ const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
152
148
  const { connection_id, source } = options;
153
149
  const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
154
150
 
@@ -362,19 +358,22 @@ export class PostgresBucketBatch
362
358
  // TODO maybe share with abstract class
363
359
  const { after, before, sourceTable, tag } = record;
364
360
  const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
365
- for (const event of this.getTableEvents(sourceTable)) {
366
- this.iterateListeners((cb) =>
367
- cb.replicationEvent?.({
368
- batch: this,
369
- table: sourceTable,
370
- data: {
371
- op: tag,
372
- after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
373
- before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
374
- },
375
- event
376
- })
377
- );
361
+ // Only the table designated as event carrier by resolveTables may fire events.
362
+ if (sourceTable.syncEvent) {
363
+ for (const event of this.getTableEvents(sourceTable)) {
364
+ this.iterateListeners((cb) =>
365
+ cb.replicationEvent?.({
366
+ batch: this,
367
+ table: sourceTable,
368
+ data: {
369
+ op: tag,
370
+ after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
371
+ before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
372
+ },
373
+ event
374
+ })
375
+ );
376
+ }
378
377
  }
379
378
  /**
380
379
  * Return if the table is just an event table
@@ -761,23 +760,22 @@ export class PostgresBucketBatch
761
760
 
762
761
  async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
763
762
  await this.db.transaction(async (db) => {
764
- const snapshotRequiredCount = await db.sql`
763
+ const blockingTables = await db.sql`
765
764
  SELECT
766
- COUNT(*) AS count
765
+ schema_name,
766
+ table_name
767
767
  FROM
768
768
  source_tables
769
769
  WHERE
770
770
  group_id = ${{ type: 'int4', value: this.group_id }}
771
771
  AND snapshot_done = FALSE
772
772
  `
773
- .decoded(t.object({ count: bigint }))
774
- .first();
775
- if ((snapshotRequiredCount?.count ?? 0n) > 0n) {
773
+ .decoded(t.object({ schema_name: t.string, table_name: t.string }))
774
+ .rows();
775
+ if (blockingTables.length > 0) {
776
776
  if (options?.throwOnConflict ?? true) {
777
777
  throw new ReplicationAssertionError(
778
- `Cannot mark snapshot done while ${snapshotRequiredCount?.count} source table${
779
- snapshotRequiredCount?.count == 1n ? '' : 's'
780
- } still require snapshotting`
778
+ `Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
781
779
  );
782
780
  } else {
783
781
  return;
@@ -1134,7 +1132,7 @@ export class PostgresBucketBatch
1134
1132
  {
1135
1133
  level: errors.ErrorSeverity.WARNING,
1136
1134
  metadata: {
1137
- replication_slot: this.options.slot_name,
1135
+ replication_slot: this.options.replicationStreamName,
1138
1136
  table: record.sourceTable.qualifiedName
1139
1137
  }
1140
1138
  }
@@ -1190,7 +1188,7 @@ export class PostgresBucketBatch
1190
1188
  {
1191
1189
  level: errors.ErrorSeverity.WARNING,
1192
1190
  metadata: {
1193
- replication_slot: this.options.slot_name,
1191
+ replication_slot: this.options.replicationStreamName,
1194
1192
  table: record.sourceTable.qualifiedName
1195
1193
  }
1196
1194
  }
@@ -1231,7 +1229,7 @@ export class PostgresBucketBatch
1231
1229
  {
1232
1230
  level: errors.ErrorSeverity.WARNING,
1233
1231
  metadata: {
1234
- replication_slot: this.options.slot_name,
1232
+ replication_slot: this.options.replicationStreamName,
1235
1233
  table: record.sourceTable.qualifiedName
1236
1234
  }
1237
1235
  }
@@ -1,4 +1,5 @@
1
1
  export * from './PostgresBucketStorageFactory.js';
2
2
  export * from './PostgresCompactor.js';
3
+ export * from './PostgresReportStorage.js';
3
4
  export * from './PostgresStorageProvider.js';
4
5
  export * from './PostgresSyncRulesStorage.js';
@@ -0,0 +1,98 @@
1
+ import * as lib_postgres from '@powersync/lib-service-postgres';
2
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
3
+ import { storage } from '@powersync/service-core';
4
+ import { models } from '../../types/types.js';
5
+
6
+ export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
7
+ constructor(
8
+ private db: lib_postgres.DatabaseClient,
9
+ row: models.SyncRulesDecoded
10
+ ) {
11
+ super({
12
+ replicationStreamId: Number(row.id),
13
+ sync_rules_content: row.content,
14
+ compiled_plan: row.sync_plan,
15
+ replicationStreamName: row.slot_name,
16
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
17
+ syncConfigState: row.state as storage.SyncRuleState
18
+ });
19
+ }
20
+
21
+ async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
22
+ const row = await this.db.sql`
23
+ SELECT
24
+ *
25
+ FROM
26
+ sync_rules
27
+ WHERE
28
+ id = ${{ value: this.replicationStreamId, type: 'int4' }}
29
+ `
30
+ .decoded(models.SyncRules)
31
+ .first();
32
+
33
+ return row == null ? null : syncConfigStatusFromRow(row);
34
+ }
35
+ }
36
+
37
+ function syncConfigStatusFromRow(row: models.SyncRulesDecoded): storage.PersistedSyncConfigStatus {
38
+ return {
39
+ id: String(row.id),
40
+ replicationStreamId: Number(row.id),
41
+ state: row.state as storage.SyncRuleState,
42
+ snapshot_done: row.snapshot_done,
43
+ last_checkpoint_lsn: row.last_checkpoint_lsn,
44
+ last_fatal_error: row.last_fatal_error,
45
+ last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
46
+ last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
47
+ };
48
+ }
49
+
50
+ export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
51
+ current_lock: storage.ReplicationLock | null = null;
52
+ readonly syncConfigContent: readonly PostgresPersistedSyncConfigContent[];
53
+
54
+ constructor(
55
+ private db: lib_postgres.DatabaseClient,
56
+ private readonly row: models.SyncRulesDecoded
57
+ ) {
58
+ super({
59
+ replicationStreamId: Number(row.id),
60
+ replicationStreamName: row.slot_name,
61
+ state: row.state as storage.SyncRuleState,
62
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
63
+ });
64
+ this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
65
+ }
66
+
67
+ parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
68
+ return this.syncConfigContent[0].parsed(options);
69
+ }
70
+
71
+ async lock(): Promise<storage.ReplicationLock> {
72
+ const manager = new lib_postgres.PostgresLockManager({
73
+ db: this.db,
74
+ name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
75
+ });
76
+ const lockHandle = await manager.acquire();
77
+ if (!lockHandle) {
78
+ throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
79
+ }
80
+
81
+ const interval = setInterval(async () => {
82
+ try {
83
+ await lockHandle.refresh();
84
+ } catch (e) {
85
+ this.logger.error('Failed to refresh lock', e);
86
+ clearInterval(interval);
87
+ }
88
+ }, 30_130);
89
+
90
+ return (this.current_lock = {
91
+ sync_rules_id: this.replicationStreamId,
92
+ release: async () => {
93
+ clearInterval(interval);
94
+ return lockHandle.release();
95
+ }
96
+ });
97
+ }
98
+ }
@@ -73,7 +73,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
73
73
 
74
74
  await using storageFactory = new PostgresBucketStorageFactory({
75
75
  config: TEST_CONNECTION_OPTIONS,
76
- slot_name_prefix: 'test_'
76
+ replicationStreamNamePrefix: 'test_'
77
77
  });
78
78
  await truncateTables(storageFactory.db);
79
79
  };
@@ -102,7 +102,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
102
102
 
103
103
  return new PostgresBucketStorageFactory({
104
104
  config: TEST_CONNECTION_OPTIONS,
105
- slot_name_prefix: 'test_'
105
+ replicationStreamNamePrefix: 'test_'
106
106
  });
107
107
  } catch (ex) {
108
108
  // Vitest does not display these errors nicely when using the `await using` syntx
@@ -18,7 +18,7 @@ describe('Migrations', () => {
18
18
  const setup = POSTGRES_STORAGE_SETUP;
19
19
  await using factory = new PostgresBucketStorageFactory({
20
20
  config: TEST_CONNECTION_OPTIONS,
21
- slot_name_prefix: 'test_'
21
+ replicationStreamNamePrefix: 'test_'
22
22
  });
23
23
 
24
24
  await dropTables(factory.db);
@@ -10,7 +10,11 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
10
10
  register.registerDataStorageParameterTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
11
11
 
12
12
  describe(`Postgres Sync Bucket Storage - Data - v${storageVersion}`, () =>
13
- register.registerDataStorageDataTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
13
+ register.registerDataStorageDataTests({
14
+ ...POSTGRES_STORAGE_FACTORY,
15
+ storageVersion,
16
+ compressedBucketStorage: false
17
+ }));
14
18
 
15
19
  describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
16
20
  register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
@@ -39,7 +43,8 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
39
43
  )
40
44
  );
41
45
  const bucketStorage = factory.getInstance(syncRules);
42
- const globalBucket = bucketRequest(syncRules, 'global[]');
46
+ const syncRulesContent = syncRules.syncConfigContent[0];
47
+ const globalBucket = bucketRequest(syncRulesContent, 'global[]');
43
48
 
44
49
  const result = await (async () => {
45
50
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -17,6 +17,7 @@ bucket_definitions:
17
17
  `)
18
18
  );
19
19
  const bucketStorage = factory.getInstance(syncRules);
20
+ const syncRulesContent = syncRules.syncConfigContent[0];
20
21
 
21
22
  const result = await (async () => {
22
23
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -44,12 +45,12 @@ bucket_definitions:
44
45
  // Compact with an explicit bucket name — exercises the this.buckets
45
46
  // iteration path, NOT the compactAllBuckets discovery path.
46
47
  await bucketStorage.compact({
47
- compactBuckets: [bucketRequest(syncRules, 'global[]').bucket],
48
+ compactBuckets: [bucketRequest(syncRulesContent, 'global[]').bucket],
48
49
  minBucketChanges: 1
49
50
  });
50
51
 
51
52
  const batch = await test_utils.oneFromAsync(
52
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
53
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
53
54
  );
54
55
 
55
56
  expect(batch.chunkData.data).toMatchObject([
@@ -70,7 +71,8 @@ bucket_definitions:
70
71
  `)
71
72
  );
72
73
  const bucketStorage = factory.getInstance(syncRules);
73
- const request = bucketRequest(syncRules, 'global[]');
74
+ const syncRulesContent = syncRules.syncConfigContent[0];
75
+ const request = bucketRequest(syncRulesContent, 'global[]');
74
76
 
75
77
  const result = await (async () => {
76
78
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -110,7 +112,7 @@ bucket_definitions:
110
112
  const dataBefore = test_utils.getBatchData(rowsBefore);
111
113
  const clearToOpId = BigInt(dataBefore[2].op_id);
112
114
 
113
- const compactor = new PostgresCompactor(factory.db, bucketStorage.group_id, {});
115
+ const compactor = new PostgresCompactor(factory.db, bucketStorage.replicationStreamId, {});
114
116
  // Trigger the private method directly
115
117
  await expect(compactor.clearBucketForTests(request.bucket, clearToOpId)).rejects.toThrow(
116
118
  /Unexpected PUT operation/
@@ -34,7 +34,8 @@ function registerStorageVersionTests(storageVersion: number) {
34
34
  )
35
35
  );
36
36
  const bucketStorage = factory.getInstance(syncRules);
37
- const globalBucket = bucketRequest(syncRules, 'global[]');
37
+ const syncRulesContent = syncRules.syncConfigContent[0];
38
+ const globalBucket = bucketRequest(syncRulesContent, 'global[]');
38
39
 
39
40
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
40
41
 
@@ -1,8 +1,6 @@
1
1
  {
2
2
  "extends": "../../../tsconfig.tests.json",
3
3
  "compilerOptions": {
4
- "declarationDir": "dist/@types",
5
- "tsBuildInfoFile": "dist/.tsbuildinfo",
6
4
  "lib": ["ES2022", "esnext.disposable"],
7
5
  "rootDir": "src"
8
6
  },
package/tsconfig.json CHANGED
@@ -1,11 +1,8 @@
1
1
  {
2
2
  "extends": "../../tsconfig.base.json",
3
3
  "compilerOptions": {
4
- "outDir": "dist",
5
- "declarationDir": "dist/@types",
6
- "tsBuildInfoFile": "dist/.tsbuildinfo",
7
4
  "rootDir": "src",
8
- "skipLibCheck": true
5
+ "outDir": "dist"
9
6
  },
10
7
  "include": ["src"],
11
8
  "references": [