@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +101 -6
  2. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
  3. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
  4. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
  5. package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +13 -12
  6. package/dist/storage/PostgresBucketStorageFactory.js +59 -55
  7. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  8. package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +3 -1
  9. package/dist/storage/PostgresCompactor.js +21 -1
  10. package/dist/storage/PostgresCompactor.js.map +1 -1
  11. package/dist/storage/PostgresReportStorage.js.map +1 -1
  12. package/dist/storage/PostgresStorageProvider.js +2 -1
  13. package/dist/storage/PostgresStorageProvider.js.map +1 -1
  14. package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +36 -11
  15. package/dist/storage/PostgresSyncRulesStorage.js +250 -177
  16. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  17. package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +8 -24
  18. package/dist/storage/batch/PostgresBucketBatch.js +229 -194
  19. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  20. package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
  21. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +161 -24
  22. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
  23. package/dist/{@types/storage → storage}/current-data-store.d.ts +8 -2
  24. package/dist/storage/current-data-store.js +66 -11
  25. package/dist/storage/current-data-store.js.map +1 -1
  26. package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
  27. package/dist/storage/storage-index.js +1 -0
  28. package/dist/storage/storage-index.js.map +1 -1
  29. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
  30. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +90 -0
  31. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +2 -0
  33. package/dist/types/models/WriteCheckpoint.js +5 -2
  34. package/dist/types/models/WriteCheckpoint.js.map +1 -1
  35. package/dist/utils/checkpoints.d.ts +9 -0
  36. package/dist/utils/checkpoints.js +26 -0
  37. package/dist/utils/checkpoints.js.map +1 -0
  38. package/dist/utils/test-utils.js +2 -2
  39. package/dist/utils/test-utils.js.map +1 -1
  40. package/package.json +10 -22
  41. package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
  42. package/src/storage/PostgresBucketStorageFactory.ts +67 -58
  43. package/src/storage/PostgresCompactor.ts +25 -1
  44. package/src/storage/PostgresReportStorage.ts +1 -2
  45. package/src/storage/PostgresStorageProvider.ts +2 -1
  46. package/src/storage/PostgresSyncRulesStorage.ts +275 -191
  47. package/src/storage/batch/PostgresBucketBatch.ts +256 -214
  48. package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +174 -24
  49. package/src/storage/current-data-store.ts +66 -11
  50. package/src/storage/storage-index.ts +1 -0
  51. package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +98 -0
  52. package/src/types/models/WriteCheckpoint.ts +5 -2
  53. package/src/utils/checkpoints.ts +31 -0
  54. package/src/utils/test-utils.ts +2 -2
  55. package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -582
  56. package/test/src/checkpoint_notifications.test.ts +522 -0
  57. package/test/src/migrations.test.ts +1 -1
  58. package/test/src/storage.test.ts +226 -10
  59. package/test/src/storage_compacting.test.ts +15 -7
  60. package/test/src/storage_sync.test.ts +11 -8
  61. package/test/tsconfig.json +1 -3
  62. package/tsconfig.json +1 -4
  63. package/tsconfig.tsbuildinfo +1 -0
  64. package/dist/.tsbuildinfo +0 -1
  65. package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
  66. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
  67. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
  68. package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
  69. /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
  70. /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
  71. /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
  72. /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
  73. /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
  74. /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
  75. /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
  76. /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
  77. /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
  78. /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
  79. /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
  80. /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
  81. /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
  82. /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
  83. /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
  84. /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
  85. /package/dist/{@types/types → types}/codecs.d.ts +0 -0
  86. /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
  87. /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
  88. /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
  89. /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
  90. /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
  91. /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
  92. /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
  93. /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
  94. /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
  95. /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
  96. /package/dist/{@types/types → types}/models/json.d.ts +0 -0
  97. /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
  98. /package/dist/{@types/types → types}/types.d.ts +0 -0
  99. /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
  100. /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
  101. /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
  102. /package/dist/{@types/utils → utils}/db.d.ts +0 -0
  103. /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
  104. /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
  105. /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
@@ -0,0 +1,90 @@
1
+ import * as lib_postgres from '@powersync/lib-service-postgres';
2
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
3
+ import { storage } from '@powersync/service-core';
4
+ import { models } from '../../types/types.js';
5
+ export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
6
+ db;
7
+ constructor(db, row) {
8
+ super({
9
+ replicationStreamId: Number(row.id),
10
+ sync_rules_content: row.content,
11
+ compiled_plan: row.sync_plan,
12
+ replicationStreamName: row.slot_name,
13
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
14
+ syncConfigState: row.state
15
+ });
16
+ this.db = db;
17
+ }
18
+ async getSyncConfigStatus() {
19
+ const row = await this.db.sql `
20
+ SELECT
21
+ *
22
+ FROM
23
+ sync_rules
24
+ WHERE
25
+ id = ${{ value: this.replicationStreamId, type: 'int4' }}
26
+ `
27
+ .decoded(models.SyncRules)
28
+ .first();
29
+ return row == null ? null : syncConfigStatusFromRow(row);
30
+ }
31
+ }
32
+ function syncConfigStatusFromRow(row) {
33
+ return {
34
+ id: String(row.id),
35
+ replicationStreamId: Number(row.id),
36
+ state: row.state,
37
+ snapshot_done: row.snapshot_done,
38
+ last_checkpoint_lsn: row.last_checkpoint_lsn,
39
+ last_fatal_error: row.last_fatal_error,
40
+ last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
41
+ last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
42
+ };
43
+ }
44
+ export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
45
+ db;
46
+ row;
47
+ current_lock = null;
48
+ syncConfigContent;
49
+ constructor(db, row) {
50
+ super({
51
+ replicationStreamId: Number(row.id),
52
+ replicationStreamName: row.slot_name,
53
+ state: row.state,
54
+ storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
55
+ });
56
+ this.db = db;
57
+ this.row = row;
58
+ this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
59
+ }
60
+ parsed(options) {
61
+ return this.syncConfigContent[0].parsed(options);
62
+ }
63
+ async lock() {
64
+ const manager = new lib_postgres.PostgresLockManager({
65
+ db: this.db,
66
+ name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
67
+ });
68
+ const lockHandle = await manager.acquire();
69
+ if (!lockHandle) {
70
+ throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
71
+ }
72
+ const interval = setInterval(async () => {
73
+ try {
74
+ await lockHandle.refresh();
75
+ }
76
+ catch (e) {
77
+ this.logger.error('Failed to refresh lock', e);
78
+ clearInterval(interval);
79
+ }
80
+ }, 30_130);
81
+ return (this.current_lock = {
82
+ sync_rules_id: this.replicationStreamId,
83
+ release: async () => {
84
+ clearInterval(interval);
85
+ return lockHandle.release();
86
+ }
87
+ });
88
+ }
89
+ }
90
+ //# sourceMappingURL=PostgresPersistedSyncConfigContent.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"PostgresPersistedSyncConfigContent.js","sourceRoot":"","sources":["../../../src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,MAAM,OAAO,kCAAmC,SAAQ,OAAO,CAAC,0BAA0B;IAE9E;IADV,YACU,EAA+B,EACvC,GAA4B;QAE5B,KAAK,CAAC;YACJ,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,kBAAkB,EAAE,GAAG,CAAC,OAAO;YAC/B,aAAa,EAAE,GAAG,CAAC,SAAS;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,sBAAsB;YACrE,eAAe,EAAE,GAAG,CAAC,KAA8B;SACpD,CAAC,CAAC;QAVK,OAAE,GAAF,EAAE,CAA6B;IAWzC,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;eAMlB,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;KAC3D;aACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aACzB,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;CACF;AAED,SAAS,uBAAuB,CAAC,GAA4B;IAC3D,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,EAAE,GAAG,CAAC,KAA8B;QACzC,aAAa,EAAE,GAAG,CAAC,aAAa;QAChC,mBAAmB,EAAE,GAAG,CAAC,mBAAmB;QAC5C,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI;QACjF,kBAAkB,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;KACrF,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,kCAAmC,SAAQ,OAAO,CAAC,0BAA0B;IAK9E;IACS;IALnB,YAAY,GAAmC,IAAI,CAAC;IAC3C,iBAAiB,CAAgD;IAE1E,YACU,EAA+B,EACtB,GAA4B;QAE7C,KAAK,CAAC;YACJ,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,KAAK,EAAE,GAAG,CAAC,KAA8B;YACzC,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,sBAAsB;SACtE,CAAC,CAAC;QARK,OAAE,GAAF,EAAE,CAA6B;QACtB,QAAG,GAAH,GAAG,CAAyB;QAQ7C,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,kCAAkC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,CAAC,OAAuC;QAC5C,OAAO,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC;YACnD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,cAAc,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,qBAAqB,EAAE;SAC7E,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,+DAA+D,CAAC,CAAC;QACjH,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC7B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;gBAC/C,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,EAAE,MAAM,CAAC,CAAC;QAEX,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG;YAC1B,aAAa,EAAE,IAAI,CAAC,mBAAmB;YACvC,OAAO,EAAE,KAAK,IAAI,EAAE;gBAClB,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACxB,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC;SACF,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -3,6 +3,7 @@ export declare const WriteCheckpoint: t.ObjectCodec<{
3
3
  user_id: t.IdentityCodec<t.CodecType.String>;
4
4
  lsns: t.Codec<Record<string, string>, string, string, t.CodecProps>;
5
5
  write_checkpoint: t.Codec<bigint, string | number, string, t.CodecProps>;
6
+ checkpoint_requested_at: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<Date, string | import("@powersync/service-sync-rules").DateTimeValue, string, t.CodecProps>>;
6
7
  }>;
7
8
  export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
8
9
  export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
@@ -10,6 +11,7 @@ export declare const CustomWriteCheckpoint: t.ObjectCodec<{
10
11
  user_id: t.IdentityCodec<t.CodecType.String>;
11
12
  write_checkpoint: t.Codec<bigint, string | number, string, t.CodecProps>;
12
13
  sync_rules_id: t.Codec<bigint, string | number, string, t.CodecProps>;
14
+ checkpoint_requested_at: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<Date, string | import("@powersync/service-sync-rules").DateTimeValue, string, t.CodecProps>>;
13
15
  }>;
14
16
  export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;
15
17
  export type CustomWriteCheckpointDecoded = t.Decoded<typeof CustomWriteCheckpoint>;
@@ -1,13 +1,16 @@
1
+ import { framework } from '@powersync/service-core';
1
2
  import * as t from 'ts-codec';
2
3
  import { bigint, jsonb } from '../codecs.js';
3
4
  export const WriteCheckpoint = t.object({
4
5
  user_id: t.string,
5
6
  lsns: jsonb(t.record(t.string)),
6
- write_checkpoint: bigint
7
+ write_checkpoint: bigint,
8
+ checkpoint_requested_at: t.Null.or(framework.codecs.date)
7
9
  });
8
10
  export const CustomWriteCheckpoint = t.object({
9
11
  user_id: t.string,
10
12
  write_checkpoint: bigint,
11
- sync_rules_id: bigint
13
+ sync_rules_id: bigint,
14
+ checkpoint_requested_at: t.Null.or(framework.codecs.date)
12
15
  });
13
16
  //# sourceMappingURL=WriteCheckpoint.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;CACzB,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;CACtB,CAAC,CAAC"}
1
+ {"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;IACxB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;IACrB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC"}
@@ -0,0 +1,9 @@
1
+ import * as lib_postgres from '@powersync/lib-service-postgres';
2
+ import { models } from '../types/types.js';
3
+ /**
4
+ * Gets the latest active checkpoint document.
5
+ * This is mainly exported for mocking in tests.
6
+ */
7
+ export declare function getActiveCheckpointDocument({ db }: {
8
+ db: lib_postgres.DatabaseClient;
9
+ }): Promise<models.ActiveCheckpointDecoded | null>;
@@ -0,0 +1,26 @@
1
+ import { storage } from '@powersync/service-core';
2
+ import { models } from '../types/types.js';
3
+ /**
4
+ * Gets the latest active checkpoint document.
5
+ * This is mainly exported for mocking in tests.
6
+ */
7
+ export async function getActiveCheckpointDocument({ db }) {
8
+ return db.sql `
9
+ SELECT
10
+ id,
11
+ last_checkpoint,
12
+ last_checkpoint_lsn
13
+ FROM
14
+ sync_rules
15
+ WHERE
16
+ state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
17
+ OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
18
+ ORDER BY
19
+ id DESC
20
+ LIMIT
21
+ 1
22
+ `
23
+ .decoded(models.ActiveCheckpoint)
24
+ .first();
25
+ }
26
+ //# sourceMappingURL=checkpoints.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../../src/utils/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAChD,EAAE,EAGH;IACC,OAAO,EAAE,CAAC,GAAG,CAAA;;;;;;;;gBAQC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;mBACrD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;;;;GAKzE;SACE,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;SAChC,KAAK,EAAE,CAAC;AACb,CAAC"}
@@ -54,7 +54,7 @@ export function postgresTestSetup(factoryOptions) {
54
54
  await runMigrations({ down: false, up: true });
55
55
  await using storageFactory = new PostgresBucketStorageFactory({
56
56
  config: TEST_CONNECTION_OPTIONS,
57
- slot_name_prefix: 'test_'
57
+ replicationStreamNamePrefix: 'test_'
58
58
  });
59
59
  await truncateTables(storageFactory.db);
60
60
  };
@@ -81,7 +81,7 @@ export function postgresTestSetup(factoryOptions) {
81
81
  }
82
82
  return new PostgresBucketStorageFactory({
83
83
  config: TEST_CONNECTION_OPTIONS,
84
- slot_name_prefix: 'test_'
84
+ replicationStreamNamePrefix: 'test_'
85
85
  });
86
86
  }
87
87
  catch (ex) {
@@ -1 +1 @@
1
- {"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAWzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,gBAAgB,EAAE,OAAO;SAC1B,CAAC,CAAC;QACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,gBAAgB,EAAE,OAAO;iBAC1B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAWzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,2BAA2B,EAAE,OAAO;SACrC,CAAC,CAAC;QACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,2BAA2B,EAAE,OAAO;iBACrC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}
package/package.json CHANGED
@@ -1,44 +1,32 @@
1
1
  {
2
2
  "name": "@powersync/service-module-postgres-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
- "types": "dist/@types/index.d.ts",
5
- "version": "0.0.0-dev-20260602073133",
6
- "main": "dist/index.js",
4
+ "version": "0.0.0-dev-20260803144812",
7
5
  "license": "FSL-1.1-ALv2",
8
6
  "type": "module",
9
7
  "publishConfig": {
10
8
  "access": "public"
11
9
  },
12
10
  "exports": {
13
- ".": {
14
- "types": "./dist/@types/index.d.ts",
15
- "import": "./dist/index.js",
16
- "require": "./dist/index.js",
17
- "default": "./dist/index.js"
18
- },
19
- "./types": {
20
- "types": "./dist/@types/index.d.ts",
21
- "import": "./dist/types/types.js",
22
- "require": "./dist/types/types.js",
23
- "default": "./dist/types/types.js"
24
- }
11
+ ".": "./dist/index.js",
12
+ "./types": "./dist/types/types.js"
25
13
  },
26
14
  "dependencies": {
27
15
  "ix": "^5.0.0",
28
16
  "lru-cache": "^10.2.2",
29
17
  "ts-codec": "^1.3.0",
30
18
  "uuid": "^14.0.0",
31
- "@powersync/lib-service-postgres": "0.0.0-dev-20260602073133",
32
- "@powersync/lib-services-framework": "0.0.0-dev-20260602073133",
33
- "@powersync/service-core": "0.0.0-dev-20260602073133",
34
- "@powersync/service-types": "0.15.2",
35
- "@powersync/service-jpgwire": "0.0.0-dev-20260602073133",
19
+ "@powersync/lib-service-postgres": "0.0.0-dev-20260803144812",
20
+ "@powersync/lib-services-framework": "0.0.0-dev-20260803144812",
21
+ "@powersync/service-core": "0.0.0-dev-20260803144812",
22
+ "@powersync/service-types": "0.0.0-dev-20260803144812",
23
+ "@powersync/service-jpgwire": "0.0.0-dev-20260803144812",
36
24
  "@powersync/service-jsonbig": "0.17.13",
37
- "@powersync/service-sync-rules": "0.0.0-dev-20260602073133"
25
+ "@powersync/service-sync-rules": "0.0.0-dev-20260803144812"
38
26
  },
39
27
  "devDependencies": {
40
28
  "typescript": "~6.0.3",
41
- "@powersync/service-core-tests": "0.0.0-dev-20260602073133"
29
+ "@powersync/service-core-tests": "0.0.0-dev-20260803144812"
42
30
  },
43
31
  "scripts": {
44
32
  "build": "tsc -b",
@@ -0,0 +1,51 @@
1
+ import { migrations } from '@powersync/service-core';
2
+
3
+ import { openMigrationDB } from '../migration-utils.js';
4
+
5
+ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
6
+ const {
7
+ service_context: { configuration }
8
+ } = context;
9
+ await using client = openMigrationDB(configuration.storage);
10
+
11
+ await client.transaction(async (db) => {
12
+ await db.sql`
13
+ ALTER TABLE write_checkpoints
14
+ ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
15
+ `.execute();
16
+ await db.sql`
17
+ ALTER TABLE custom_write_checkpoints
18
+ ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
19
+ `.execute();
20
+ await db.sql`
21
+ CREATE INDEX write_checkpoints_checkpoint_requested_at ON write_checkpoints (checkpoint_requested_at)
22
+ WHERE
23
+ checkpoint_requested_at IS NOT NULL
24
+ `.execute();
25
+ await db.sql`
26
+ CREATE INDEX custom_write_checkpoints_checkpoint_requested_at ON custom_write_checkpoints (checkpoint_requested_at)
27
+ WHERE
28
+ checkpoint_requested_at IS NOT NULL
29
+ `.execute();
30
+ });
31
+ };
32
+
33
+ export const down: migrations.PowerSyncMigrationFunction = async (context) => {
34
+ const {
35
+ service_context: { configuration }
36
+ } = context;
37
+ await using client = openMigrationDB(configuration.storage);
38
+
39
+ await client.transaction(async (db) => {
40
+ // Also drops the write_checkpoints_checkpoint_requested_at index.
41
+ await db.sql`
42
+ ALTER TABLE write_checkpoints
43
+ DROP COLUMN checkpoint_requested_at
44
+ `.execute();
45
+ // Also drops the custom_write_checkpoints_checkpoint_requested_at index.
46
+ await db.sql`
47
+ ALTER TABLE custom_write_checkpoints
48
+ DROP COLUMN checkpoint_requested_at
49
+ `.execute();
50
+ });
51
+ };
@@ -1,4 +1,4 @@
1
- import { framework, GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core';
1
+ import { framework, GetIntanceOptions, storage } from '@powersync/service-core';
2
2
  import * as pg_wire from '@powersync/service-jpgwire';
3
3
  import crypto from 'crypto';
4
4
  import * as uuid from 'uuid';
@@ -8,19 +8,19 @@ import { models, NormalizedPostgresStorageConfig } from '../types/types.js';
8
8
 
9
9
  import { getStorageApplicationName } from '../utils/application-name.js';
10
10
  import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
11
- import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
12
11
  import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
13
- import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js';
12
+ import { PostgresPersistedReplicationStream } from './sync-rules/PostgresPersistedSyncConfigContent.js';
14
13
 
15
14
  export type PostgresBucketStorageOptions = {
16
15
  config: NormalizedPostgresStorageConfig;
17
- slot_name_prefix: string;
16
+ replicationStreamNamePrefix: string;
17
+ checksumCacheTtlMs?: number;
18
18
  };
19
19
 
20
20
  export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
21
21
  [framework.DO_NOT_LOG] = true;
22
22
  readonly db: lib_postgres.DatabaseClient;
23
- public readonly slot_name_prefix: string;
23
+ public readonly replicationStreamNamePrefix: string;
24
24
 
25
25
  private activeStorageCache: storage.SyncRulesBucketStorage | undefined;
26
26
 
@@ -32,7 +32,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
32
32
  notificationChannels: [NOTIFICATION_CHANNEL],
33
33
  applicationName: getStorageApplicationName()
34
34
  });
35
- this.slot_name_prefix = options.slot_name_prefix;
35
+ this.replicationStreamNamePrefix = options.replicationStreamNamePrefix;
36
36
 
37
37
  this.db.registerListener({
38
38
  connectionCreated: async (connection) => this.prepareStatements(connection)
@@ -49,31 +49,32 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
49
49
  }
50
50
 
51
51
  getInstance(
52
- syncRules: storage.PersistedSyncRulesContent,
52
+ replicationStream: storage.PersistedReplicationStream,
53
53
  options?: GetIntanceOptions
54
54
  ): storage.SyncRulesBucketStorage {
55
- const storage = new PostgresSyncRulesStorage({
55
+ const syncRuleStorage = new PostgresSyncRulesStorage({
56
56
  factory: this,
57
57
  db: this.db,
58
- sync_rules: syncRules,
59
- batchLimits: this.options.config.batch_limits
58
+ replicationStream,
59
+ batchLimits: this.options.config.batch_limits,
60
+ checksumCacheTtlMs: this.options.checksumCacheTtlMs
60
61
  });
61
62
  if (!options?.skipLifecycleHooks) {
62
- this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
63
+ this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage));
63
64
  }
64
- storage.registerListener({
65
+ syncRuleStorage.registerListener({
65
66
  batchStarted: (batch) => {
66
67
  batch.registerListener({
67
68
  replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
68
69
  });
69
70
  }
70
71
  });
71
- return storage;
72
+ return syncRuleStorage;
72
73
  }
73
74
 
74
75
  async getStorageMetrics(): Promise<storage.StorageMetrics> {
75
- const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
76
- if (active_sync_rules == null) {
76
+ const activeSyncConfig = await this.getActiveSyncConfig();
77
+ if (activeSyncConfig == null) {
77
78
  return {
78
79
  operations_size_bytes: 0,
79
80
  parameters_size_bytes: 0,
@@ -153,7 +154,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
153
154
  };
154
155
  }
155
156
 
156
- async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedSyncRulesContent> {
157
+ async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
157
158
  const storageVersion =
158
159
  options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
159
160
  const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
@@ -200,7 +201,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
200
201
  ${{ type: 'json', value: options.config.plan }},
201
202
  ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
202
203
  CONCAT(
203
- ${{ type: 'varchar', value: this.slot_name_prefix }},
204
+ ${{ type: 'varchar', value: this.replicationStreamNamePrefix }},
204
205
  (
205
206
  SELECT
206
207
  id
@@ -218,9 +219,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
218
219
  .decoded(models.SyncRules)
219
220
  .first();
220
221
 
221
- await notifySyncRulesUpdate(this.db, newSyncRulesRow!);
222
-
223
- return new PostgresPersistedSyncRulesContent(this.db, newSyncRulesRow!);
222
+ return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
224
223
  });
225
224
  }
226
225
 
@@ -252,28 +251,28 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
252
251
  `.execute();
253
252
  }
254
253
 
255
- async restartReplication(sync_rules_group_id: number): Promise<void> {
256
- const next = await this.getNextSyncRulesContent();
257
- const active = await this.getActiveSyncRulesContent();
254
+ async restartReplication(replicationStreamId: number): Promise<void> {
255
+ const next = await this.getDeployingSyncConfig();
256
+ const active = await this.getActiveSyncConfig();
258
257
 
259
258
  // In both the below cases, we create a new replication stream.
260
259
  // The current one will continue serving sync requests until the next one has finished processing.
261
- if (next != null && next.id == sync_rules_group_id) {
260
+ if (next != null && next.content.replicationStreamId == replicationStreamId) {
262
261
  // We need to redo the "next" replication stream
263
262
 
264
- await this.updateSyncRules(next.asUpdateOptions());
263
+ await this.updateSyncRules(next.content.asUpdateOptions());
265
264
  // Pro-actively stop replicating
266
265
  await this.db.sql`
267
266
  UPDATE sync_rules
268
267
  SET
269
268
  state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }}
270
269
  WHERE
271
- id = ${{ value: next.id, type: 'int4' }}
270
+ id = ${{ value: next.content.replicationStreamId, type: 'int4' }}
272
271
  AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }}
273
272
  `.execute();
274
- } else if (next == null && active?.id == sync_rules_group_id) {
273
+ } else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
275
274
  // Slot removed for "active" replication stream, while there is no "next" one.
276
- await this.updateSyncRules(active.asUpdateOptions());
275
+ await this.updateSyncRules(active.content.asUpdateOptions());
277
276
 
278
277
  // Pro-actively stop replicating, but still serve clients with existing data
279
278
  await this.db.sql`
@@ -281,10 +280,10 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
281
280
  SET
282
281
  state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
283
282
  WHERE
284
- id = ${{ value: active.id, type: 'int4' }}
283
+ id = ${{ value: active.content.replicationStreamId, type: 'int4' }}
285
284
  AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
286
285
  `.execute();
287
- } else if (next != null && active?.id == sync_rules_group_id) {
286
+ } else if (next != null && active?.content.replicationStreamId == replicationStreamId) {
288
287
  // Already have "next" replication stream - don't update any.
289
288
 
290
289
  // Pro-actively stop replicating, but still serve clients with existing data
@@ -293,13 +292,13 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
293
292
  SET
294
293
  state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
295
294
  WHERE
296
- id = ${{ value: active.id, type: 'int4' }}
295
+ id = ${{ value: active.content.replicationStreamId, type: 'int4' }}
297
296
  AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
298
297
  `.execute();
299
298
  }
300
299
  }
301
300
 
302
- async getActiveSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null> {
301
+ async getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
303
302
  const activeRow = await this.db.sql`
304
303
  SELECT
305
304
  *
@@ -319,11 +318,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
319
318
  return null;
320
319
  }
321
320
 
322
- return new PostgresPersistedSyncRulesContent(this.db, activeRow);
321
+ return this.resolvedSyncConfigFromRow(activeRow, { cacheActiveStorage: true });
323
322
  }
324
323
 
325
- async getNextSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null> {
326
- const nextRow = await this.db.sql`
324
+ async getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
325
+ const row = await this.db.sql`
327
326
  SELECT
328
327
  *
329
328
  FROM
@@ -337,14 +336,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
337
336
  `
338
337
  .decoded(models.SyncRules)
339
338
  .first();
340
- if (!nextRow) {
341
- return null;
342
- }
343
339
 
344
- return new PostgresPersistedSyncRulesContent(this.db, nextRow);
340
+ return row == null ? null : this.resolvedSyncConfigFromRow(row);
345
341
  }
346
342
 
347
- async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
343
+ async getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
348
344
  const rows = await this.db.sql`
349
345
  SELECT
350
346
  *
@@ -357,10 +353,10 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
357
353
  .decoded(models.SyncRules)
358
354
  .rows();
359
355
 
360
- return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row));
356
+ return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
361
357
  }
362
358
 
363
- async getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
359
+ async getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
364
360
  const rows = await this.db.sql`
365
361
  SELECT
366
362
  *
@@ -372,24 +368,37 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
372
368
  .decoded(models.SyncRules)
373
369
  .rows();
374
370
 
375
- return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row));
371
+ return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
376
372
  }
377
373
 
378
- async getActiveStorage(): Promise<SyncRulesBucketStorage | null> {
379
- const content = await this.getActiveSyncRulesContent();
380
- if (content == null) {
381
- return null;
382
- }
374
+ private resolvedSyncConfigFromRow(
375
+ row: models.SyncRulesDecoded,
376
+ options: { cacheActiveStorage?: boolean } = {}
377
+ ): storage.ResolvedSyncConfig {
378
+ const stream = new PostgresPersistedReplicationStream(this.db, row);
379
+ const content = stream.syncConfigContent[0];
380
+ const thisFactory = this;
383
381
 
384
- // It is important that this instance is cached.
385
- // Not for the instance construction itself, but to ensure that internal caches on the instance
386
- // are re-used properly.
387
- if (this.activeStorageCache?.group_id == content.id) {
388
- return this.activeStorageCache;
389
- } else {
390
- const instance = this.getInstance(content);
391
- this.activeStorageCache = instance;
392
- return instance;
393
- }
382
+ return {
383
+ content,
384
+ replicationStream: stream,
385
+ get storage() {
386
+ // It is important that this instance is cached.
387
+ // Not for the instance construction itself, but to ensure that internal caches on the instance
388
+ // are re-used properly.
389
+ if (options.cacheActiveStorage) {
390
+ const activeStorageCache = thisFactory.activeStorageCache;
391
+ if (activeStorageCache?.replicationStreamId == stream.replicationStreamId) {
392
+ return activeStorageCache;
393
+ }
394
+ }
395
+
396
+ const instance = thisFactory.getInstance(stream);
397
+ if (options.cacheActiveStorage) {
398
+ thisFactory.activeStorageCache = instance;
399
+ }
400
+ return instance;
401
+ }
402
+ };
394
403
  }
395
404
  }
@@ -53,6 +53,7 @@ export class PostgresCompactor {
53
53
  private clearBatchLimit: number;
54
54
  private maxOpId: InternalOpId;
55
55
  private buckets: string[] | undefined;
56
+ private deleteCheckpointRequestsBefore: Date | undefined;
56
57
 
57
58
  constructor(
58
59
  private db: lib_postgres.DatabaseClient,
@@ -65,14 +66,17 @@ export class PostgresCompactor {
65
66
  this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
66
67
  this.maxOpId = options.maxOpId ?? 0n;
67
68
  this.buckets = options.compactBuckets;
69
+ this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
68
70
  }
69
71
 
70
72
  /**
71
73
  * Compact buckets by converting operations into MOVE and/or CLEAR operations.
72
74
  *
73
- * See /docs/compacting-operations.md for details.
75
+ * See /docs/storage/compacting-operations.md for details.
74
76
  */
75
77
  async compact() {
78
+ await this.deleteOldCheckpointRequests();
79
+
76
80
  if (this.buckets) {
77
81
  for (let bucket of this.buckets) {
78
82
  await this.compactSingleBucket(bucket);
@@ -82,6 +86,26 @@ export class PostgresCompactor {
82
86
  }
83
87
  }
84
88
 
89
+ private async deleteOldCheckpointRequests() {
90
+ if (this.deleteCheckpointRequestsBefore == null) {
91
+ return;
92
+ }
93
+
94
+ await this.db.sql`
95
+ DELETE FROM write_checkpoints
96
+ WHERE
97
+ checkpoint_requested_at IS NOT NULL
98
+ AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
99
+ `.execute();
100
+
101
+ await this.db.sql`
102
+ DELETE FROM custom_write_checkpoints
103
+ WHERE
104
+ checkpoint_requested_at IS NOT NULL
105
+ AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
106
+ `.execute();
107
+ }
108
+
85
109
  private async compactAllBuckets() {
86
110
  const DISCOVERY_BATCH_SIZE = 200;
87
111
  let lastBucket = '';