@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +152 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +36 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +30 -22
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/metrics/metrics-interfaces.d.ts +5 -1
  9. package/dist/metrics/metrics-interfaces.js.map +1 -1
  10. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
  11. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
  12. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
  13. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
  14. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
  15. package/dist/metrics/open-telemetry/util.js +7 -3
  16. package/dist/metrics/open-telemetry/util.js.map +1 -1
  17. package/dist/modules/AbstractModule.d.ts +6 -2
  18. package/dist/modules/AbstractModule.js.map +1 -1
  19. package/dist/replication/AbstractReplicator.d.ts +31 -7
  20. package/dist/replication/AbstractReplicator.js +168 -74
  21. package/dist/replication/AbstractReplicator.js.map +1 -1
  22. package/dist/replication/ReplicationModule.d.ts +3 -3
  23. package/dist/replication/ReplicationModule.js.map +1 -1
  24. package/dist/routes/RouterEngine.d.ts +1 -0
  25. package/dist/routes/RouterEngine.js +6 -1
  26. package/dist/routes/RouterEngine.js.map +1 -1
  27. package/dist/routes/configure-fastify.d.ts +104 -56
  28. package/dist/routes/configure-fastify.js +15 -2
  29. package/dist/routes/configure-fastify.js.map +1 -1
  30. package/dist/routes/endpoints/admin.d.ts +42 -24
  31. package/dist/routes/endpoints/admin.js +49 -36
  32. package/dist/routes/endpoints/admin.js.map +1 -1
  33. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  34. package/dist/routes/endpoints/checkpointing.js +65 -7
  35. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  36. package/dist/routes/endpoints/socket-route.js +3 -2
  37. package/dist/routes/endpoints/socket-route.js.map +1 -1
  38. package/dist/routes/endpoints/sync-rules.js +25 -14
  39. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  40. package/dist/routes/endpoints/sync-stream.js +1 -1
  41. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  42. package/dist/runner/teardown.js +8 -8
  43. package/dist/runner/teardown.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +64 -17
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/BucketStorageFactory.d.ts +50 -31
  47. package/dist/storage/BucketStorageFactory.js +10 -20
  48. package/dist/storage/BucketStorageFactory.js.map +1 -1
  49. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  50. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  51. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  52. package/dist/storage/ChecksumCache.d.ts +11 -6
  53. package/dist/storage/ChecksumCache.js +5 -4
  54. package/dist/storage/ChecksumCache.js.map +1 -1
  55. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  56. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  57. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  58. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  59. package/dist/storage/PersistedReplicationStream.js +26 -0
  60. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  61. package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
  62. package/dist/storage/PersistedSyncConfigContent.js +138 -0
  63. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  64. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  65. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  66. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  67. package/dist/storage/ReplicationEventPayload.d.ts +6 -1
  68. package/dist/storage/SourceEntity.d.ts +13 -2
  69. package/dist/storage/SourceTable.d.ts +53 -4
  70. package/dist/storage/SourceTable.js +62 -7
  71. package/dist/storage/SourceTable.js.map +1 -1
  72. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  73. package/dist/storage/SourceTableReconciler.js +95 -0
  74. package/dist/storage/SourceTableReconciler.js.map +1 -0
  75. package/dist/storage/StorageVersionConfig.d.ts +4 -0
  76. package/dist/storage/StorageVersionConfig.js +10 -1
  77. package/dist/storage/StorageVersionConfig.js.map +1 -1
  78. package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
  79. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  80. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  81. package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
  82. package/dist/storage/WriteCheckpointAPI.js +33 -0
  83. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  84. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
  85. package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
  86. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  87. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  88. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  89. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  90. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  91. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
  92. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  93. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  94. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  95. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  96. package/dist/storage/storage-index.d.ts +10 -1
  97. package/dist/storage/storage-index.js +10 -1
  98. package/dist/storage/storage-index.js.map +1 -1
  99. package/dist/storage/storage-metrics.js +120 -4
  100. package/dist/storage/storage-metrics.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +12 -2
  102. package/dist/sync/BucketChecksumState.js +69 -21
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.d.ts +7 -0
  105. package/dist/sync/RequestTracker.js +12 -0
  106. package/dist/sync/RequestTracker.js.map +1 -1
  107. package/dist/sync/sync.js +184 -43
  108. package/dist/sync/sync.js.map +1 -1
  109. package/dist/sync/util.d.ts +5 -0
  110. package/dist/sync/util.js +19 -1
  111. package/dist/sync/util.js.map +1 -1
  112. package/dist/system/ServiceContext.d.ts +2 -0
  113. package/dist/system/ServiceContext.js +2 -0
  114. package/dist/system/ServiceContext.js.map +1 -1
  115. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  116. package/dist/tracing/PerformanceTracer.js +11 -0
  117. package/dist/tracing/PerformanceTracer.js.map +1 -1
  118. package/dist/util/checkpointing.d.ts +4 -8
  119. package/dist/util/checkpointing.js +1 -16
  120. package/dist/util/checkpointing.js.map +1 -1
  121. package/dist/util/config/compound-config-collector.js +23 -2
  122. package/dist/util/config/compound-config-collector.js.map +1 -1
  123. package/dist/util/config/defaults.d.ts +2 -0
  124. package/dist/util/config/defaults.js +2 -0
  125. package/dist/util/config/defaults.js.map +1 -1
  126. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  127. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  128. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  130. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  131. package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
  132. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  133. package/dist/util/config/types.d.ts +4 -1
  134. package/dist/util/lsn.d.ts +4 -0
  135. package/dist/util/lsn.js +10 -0
  136. package/dist/util/lsn.js.map +1 -1
  137. package/dist/util/protocol-types.d.ts +4 -4
  138. package/dist/util/protocol-types.js +3 -2
  139. package/dist/util/protocol-types.js.map +1 -1
  140. package/dist/util/util-index.d.ts +1 -0
  141. package/dist/util/util-index.js +1 -0
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +3 -2
  144. package/dist/util/utils.js +17 -2
  145. package/dist/util/utils.js.map +1 -1
  146. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  147. package/dist/util/write-checkpoint-batcher.js +100 -0
  148. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  149. package/package.json +7 -7
  150. package/src/api/RouteAPI.ts +43 -14
  151. package/src/api/diagnostics.ts +45 -26
  152. package/src/entry/commands/compact-action.ts +37 -23
  153. package/src/metrics/metrics-interfaces.ts +6 -1
  154. package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
  155. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
  156. package/src/metrics/open-telemetry/util.ts +7 -3
  157. package/src/modules/AbstractModule.ts +6 -2
  158. package/src/replication/AbstractReplicator.ts +212 -78
  159. package/src/replication/ReplicationModule.ts +2 -3
  160. package/src/routes/RouterEngine.ts +7 -1
  161. package/src/routes/configure-fastify.ts +25 -2
  162. package/src/routes/endpoints/admin.ts +75 -51
  163. package/src/routes/endpoints/checkpointing.ts +79 -8
  164. package/src/routes/endpoints/socket-route.ts +3 -2
  165. package/src/routes/endpoints/sync-rules.ts +26 -14
  166. package/src/routes/endpoints/sync-stream.ts +1 -1
  167. package/src/runner/teardown.ts +8 -8
  168. package/src/storage/BucketStorageBatch.ts +65 -18
  169. package/src/storage/BucketStorageFactory.ts +73 -46
  170. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  171. package/src/storage/ChecksumCache.ts +27 -14
  172. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  173. package/src/storage/PersistedReplicationStream.ts +55 -0
  174. package/src/storage/PersistedSyncConfigContent.ts +207 -0
  175. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  176. package/src/storage/ReplicationEventPayload.ts +6 -1
  177. package/src/storage/SourceEntity.ts +12 -2
  178. package/src/storage/SourceTable.ts +96 -8
  179. package/src/storage/SourceTableReconciler.ts +188 -0
  180. package/src/storage/StorageVersionConfig.ts +11 -1
  181. package/src/storage/SyncRulesBucketStorage.ts +189 -27
  182. package/src/storage/WriteCheckpointAPI.ts +140 -6
  183. package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
  184. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  185. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
  186. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  187. package/src/storage/storage-index.ts +11 -1
  188. package/src/storage/storage-metrics.ts +154 -8
  189. package/src/sync/BucketChecksumState.ts +105 -21
  190. package/src/sync/RequestTracker.ts +20 -0
  191. package/src/sync/sync.ts +214 -50
  192. package/src/sync/util.ts +33 -8
  193. package/src/system/ServiceContext.ts +7 -0
  194. package/src/tracing/PerformanceTracer.ts +41 -1
  195. package/src/util/checkpointing.ts +4 -22
  196. package/src/util/config/compound-config-collector.ts +31 -1
  197. package/src/util/config/defaults.ts +2 -0
  198. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  199. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  200. package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
  201. package/src/util/config/types.ts +4 -1
  202. package/src/util/lsn.ts +9 -0
  203. package/src/util/protocol-types.ts +3 -2
  204. package/src/util/util-index.ts +1 -0
  205. package/src/util/utils.ts +16 -2
  206. package/src/util/write-checkpoint-batcher.ts +142 -0
  207. package/test/src/AbstractReplicator.test.ts +177 -0
  208. package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
  209. package/test/src/PersistedSyncConfigContent.test.ts +152 -0
  210. package/test/src/checksum_cache.test.ts +22 -0
  211. package/test/src/config.test.ts +142 -0
  212. package/test/src/diagnostics.test.ts +87 -34
  213. package/test/src/module-loader.test.ts +33 -1
  214. package/test/src/routes/admin.test.ts +188 -18
  215. package/test/src/routes/checkpointing.test.ts +54 -0
  216. package/test/src/routes/mocks.ts +6 -2
  217. package/test/src/routes/stream.test.ts +123 -2
  218. package/test/src/source-table-reconciler.test.ts +244 -0
  219. package/test/src/storage/SourceTable.test.ts +92 -0
  220. package/test/src/storage-metrics.test.ts +148 -0
  221. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  222. package/test/src/sync/util.test.ts +28 -1
  223. package/test/src/util/checkpointing.test.ts +249 -0
  224. package/test/src/util/utils.test.ts +23 -0
  225. package/tsconfig.json +1 -4
  226. package/tsconfig.tsbuildinfo +1 -1
  227. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  228. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  229. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  230. package/src/storage/PersistedSyncRulesContent.ts +0 -174
@@ -12,6 +12,15 @@ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptio
12
12
  oldestUncommittedChange: null
13
13
  };
14
14
 
15
+ /**
16
+ * Storage writer for a source-specific unit of replication work.
17
+ *
18
+ * A batch evaluates source row changes against the hydrated sync config,
19
+ * persists bucket operations and parameter lookup rows, tracks source table and
20
+ * snapshot state, records the committed source position for visible
21
+ * checkpoints, and stores restart cursors for sources with PowerSync-managed
22
+ * resume state.
23
+ */
15
24
  export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
16
25
  /**
17
26
  * Alias for [Symbol.asyncDispose]
@@ -29,21 +38,28 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
29
38
  readonly skipExistingRows: boolean;
30
39
 
31
40
  /**
32
- * Save an op, and potentially flush.
41
+ * Queue one normalized source row operation and potentially flush.
33
42
  *
34
- * This can be an insert, update or delete op.
43
+ * When flushed, the operation is evaluated against data and parameter
44
+ * queries. That may produce bucket operations, parameter lookup rows, and
45
+ * current-data updates. `save()` may trigger that flush automatically when
46
+ * the batch reaches its size limits, but a flush does not create a checkpoint
47
+ * by itself.
35
48
  */
36
49
  save(record: SaveOptions): Promise<FlushedResult | null>;
37
50
 
38
51
  /**
39
- * Replicate a truncate op - deletes all data in the specified tables.
52
+ * Replicate a truncate operation by removing all currently replicated rows
53
+ * for the specified source tables.
40
54
  */
41
55
  truncate(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
42
56
 
43
57
  /**
44
- * Drop one or more tables.
58
+ * Drop one or more source table mappings.
45
59
  *
46
- * This is the same as truncate, but additionally removes the SourceTable record.
60
+ * This performs the same logical row removal as `truncate()`, then removes
61
+ * the stored `SourceTable` record. It is used for table drops, renames,
62
+ * replica identity changes, and superseded mappings.
47
63
  */
48
64
  drop(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
49
65
 
@@ -58,21 +74,26 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
58
74
  flush(options?: BatchBucketFlushOptions): Promise<FlushedResult | null>;
59
75
 
60
76
  /**
61
- * Flush and commit any saved ops. This creates a new checkpoint by default.
77
+ * Flush saved ops and advance the committed source checkpoint position.
62
78
  *
63
- * Only call this after a transaction.
79
+ * Only call this at a source transaction, page, snapshot, or other boundary
80
+ * where it is valid for clients to observe all flushed changes together. A
81
+ * single commit may include multiple complete source transactions, but must
82
+ * not split one source transaction across checkpoint boundaries.
64
83
  */
65
84
  commit(lsn: string, options?: BucketBatchCommitOptions): Promise<CheckpointResult>;
66
85
 
67
86
  /**
68
- * Advance the checkpoint LSN position, without any associated op.
87
+ * Advance the committed source checkpoint position without associated row
88
+ * operations.
69
89
  *
70
- * This must only be called when not inside a transaction.
90
+ * Use this for source heartbeats, marker-only events, and idle-source write
91
+ * checkpoint progress. This must only be called outside a source transaction.
71
92
  */
72
93
  keepalive(lsn: string): Promise<CheckpointResult>;
73
94
 
74
95
  /**
75
- * Set the LSN that replication should resume from.
96
+ * Set the source position that replication should resume from after restart.
76
97
  *
77
98
  * This can be used for:
78
99
  * 1. Setting the LSN for a snapshot, before starting replication.
@@ -80,14 +101,13 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
80
101
  *
81
102
  * Not required if the source database keeps track of this, for example with
82
103
  * PostgreSQL logical replication slots.
104
+ *
105
+ * This is for restart safety only. If the same source position should become
106
+ * visible to clients, also call `commit()` or `keepalive()` at a valid source
107
+ * boundary.
83
108
  */
84
109
  setResumeLsn(lsn: string): Promise<void>;
85
110
 
86
- /**
87
- * Get the last checkpoint LSN, from either commit or keepalive.
88
- */
89
- lastCheckpointLsn: string | null;
90
-
91
111
  /**
92
112
  * LSN to resume from.
93
113
  *
@@ -95,7 +115,21 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
95
115
  */
96
116
  resumeFromLsn: string | null;
97
117
 
118
+ /**
119
+ * Mark table snapshots complete and record the source position that
120
+ * checkpoints must not precede for those tables.
121
+ */
98
122
  markTableSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn?: string): Promise<SourceTable[]>;
123
+
124
+ /**
125
+ * Mark a source table as requiring snapshot work again.
126
+ *
127
+ * Use this when a table that was previously considered snapshot-complete
128
+ * must be copied again before it can safely contribute to checkpoints. Common
129
+ * cases include detected schema or replica identity changes, interrupted
130
+ * re-snapshot work, or other source metadata changes that make the existing
131
+ * replicated rows no longer trustworthy.
132
+ */
99
133
  markTableSnapshotRequired(table: SourceTable): Promise<void>;
100
134
 
101
135
  /**
@@ -118,6 +152,10 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
118
152
  */
119
153
  markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void>;
120
154
 
155
+ /**
156
+ * Persist resumable table snapshot progress such as row counts or source
157
+ * cursors.
158
+ */
121
159
  updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
122
160
 
123
161
  /**
@@ -125,10 +163,21 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
125
163
  */
126
164
  getSourceTableStatus(table: SourceTable): Promise<SourceTable | null>;
127
165
 
166
+ /**
167
+ * Resolve discovered source metadata against the active sync config and
168
+ * persisted source table state.
169
+ *
170
+ * @returns Current mappings that should receive replicated rows, plus
171
+ * outdated mappings that the source connector should drop.
172
+ */
128
173
  resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
129
174
 
130
175
  /**
131
- * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
176
+ * Queue a custom checkpoint request to be persisted after operations are
177
+ * flushed. Set `checkpoint_requested_at` when the custom checkpoint came from
178
+ * a client checkpoint request and should be cleaned up by request-retention
179
+ * compaction; omit it for persistent source-owned checkpoints. Incremental
180
+ * event handlers should copy `event_id` from their {@link ReplicationEventPayload}.
132
181
  */
133
182
  addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
134
183
  }
@@ -216,8 +265,6 @@ export interface CheckpointResult {
216
265
  /**
217
266
  * True if a checkpoint was actually created by this operation. This can be false even if checkpointBlocked is false,
218
267
  * if the checkpoint was empty.
219
- *
220
- * This is primarily used for testing.
221
268
  */
222
269
  checkpointCreated: boolean;
223
270
  }
@@ -1,6 +1,7 @@
1
1
  import { BaseObserver, logger } from '@powersync/lib-services-framework';
2
2
  import {
3
3
  PrecompiledSyncConfig,
4
+ SerializedSyncPlan as RawSerializedSyncPlan,
4
5
  SerializedCompatibilityContext,
5
6
  serializeSyncPlan,
6
7
  SqlSyncRules,
@@ -8,7 +9,8 @@ import {
8
9
  } from '@powersync/service-sync-rules';
9
10
  import { ReplicationError } from '@powersync/service-types';
10
11
  import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
11
- import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
12
+ import { PersistedReplicationStream } from './PersistedReplicationStream.js';
13
+ import { PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
12
14
  import { ReplicationEventPayload } from './ReplicationEventPayload.js';
13
15
  import { ReplicationLock } from './ReplicationLock.js';
14
16
  import { ReportStorage } from './ReportStorage.js';
@@ -31,14 +33,21 @@ export abstract class BucketStorageFactory
31
33
  */
32
34
  async configureSyncRules(
33
35
  options: UpdateSyncRulesOptions
34
- ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }> {
35
- const next = await this.getNextSyncRulesContent();
36
- const active = await this.getActiveSyncRulesContent();
37
-
38
- if (next?.sync_rules_content == options.config.yaml) {
36
+ ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedReplicationStream; lock?: ReplicationLock }> {
37
+ const deploying = await this.getDeployingSyncConfig();
38
+ const active = await this.getActiveSyncConfig();
39
+
40
+ if (
41
+ deploying?.content.sync_rules_content == options.config.yaml &&
42
+ deploying.content.version_label == options.version_label
43
+ ) {
39
44
  logger.info('Sync config unchanged');
40
45
  return { updated: false };
41
- } else if (next == null && active?.sync_rules_content == options.config.yaml) {
46
+ } else if (
47
+ deploying == null &&
48
+ active?.content.sync_rules_content == options.config.yaml &&
49
+ active.content.version_label == options.version_label
50
+ ) {
42
51
  logger.info('Sync config unchanged');
43
52
  return { updated: false };
44
53
  } else {
@@ -51,12 +60,15 @@ export abstract class BucketStorageFactory
51
60
  /**
52
61
  * Get a storage instance to query sync data for specific sync config.
53
62
  */
54
- abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
63
+ abstract getInstance(
64
+ replicationStream: PersistedReplicationStream,
65
+ options?: GetIntanceOptions
66
+ ): SyncRulesBucketStorage;
55
67
 
56
68
  /**
57
69
  * Deploy new sync config.
58
70
  */
59
- abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
71
+ abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedReplicationStream>;
60
72
 
61
73
  /**
62
74
  * Indicate that a slot was removed, and we should re-sync by creating
@@ -68,48 +80,27 @@ export abstract class BucketStorageFactory
68
80
  *
69
81
  * Replication should be restarted after this.
70
82
  */
71
- abstract restartReplication(sync_rules_group_id: number): Promise<void>;
83
+ abstract restartReplication(replicationStreamId: number): Promise<void>;
72
84
 
73
85
  /**
74
- * Get the sync config used for querying.
86
+ * Get the sync config and storage used for querying.
75
87
  */
76
- async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
77
- const content = await this.getActiveSyncRulesContent();
78
- return content?.parsed(options) ?? null;
79
- }
88
+ abstract getActiveSyncConfig(): Promise<ResolvedSyncConfig | null>;
80
89
 
81
90
  /**
82
- * Get the sync config used for querying.
91
+ * Get the sync config and storage that is still deploying.
83
92
  */
84
- abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
93
+ abstract getDeployingSyncConfig(): Promise<ResolvedSyncConfig | null>;
85
94
 
86
95
  /**
87
- * Get the sync config that will be active next once done with initial replicatino.
96
+ * Get all replication streams currently replicating.
88
97
  */
89
- async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
90
- const content = await this.getNextSyncRulesContent();
91
- return content?.parsed(options) ?? null;
92
- }
98
+ abstract getReplicatingReplicationStreams(): Promise<PersistedReplicationStream[]>;
93
99
 
94
100
  /**
95
- * Get the sync config that will be active next once done with initial replicatino.
101
+ * Get all replication streams stopped but not terminated yet.
96
102
  */
97
- abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
98
-
99
- /**
100
- * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
101
- */
102
- abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
103
-
104
- /**
105
- * Get all sync config stopped but not terminated yet.
106
- */
107
- abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
108
-
109
- /**
110
- * Get the active storage instance.
111
- */
112
- abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
103
+ abstract getStoppedReplicationStreams(): Promise<PersistedReplicationStream[]>;
113
104
 
114
105
  /**
115
106
  * Get storage size of active replication stream.
@@ -134,6 +125,12 @@ export interface BucketStorageFactoryListener {
134
125
  replicationEvent: (event: ReplicationEventPayload) => void;
135
126
  }
136
127
 
128
+ export interface ResolvedSyncConfig {
129
+ content: PersistedSyncConfigContent;
130
+ replicationStream: PersistedReplicationStream;
131
+ storage: SyncRulesBucketStorage;
132
+ }
133
+
137
134
  export interface StorageMetrics {
138
135
  /**
139
136
  * Size of operations (bucket_data)
@@ -151,9 +148,32 @@ export interface StorageMetrics {
151
148
  * Size of current_data.
152
149
  */
153
150
  replication_size_bytes: number;
151
+
152
+ /**
153
+ * Size of active object-storage references, such as S3 bucket-data objects.
154
+ */
155
+ object_storage_size_bytes?: number;
156
+
157
+ /**
158
+ * Per-sync-config storage sizes, when the storage backend can provide them.
159
+ */
160
+ sync_config_metrics?: StorageSyncConfigMetrics[];
161
+ }
162
+
163
+ export interface StorageSyncConfigMetrics {
164
+ sync_config_id: string;
165
+ sync_config_state: string;
166
+ /** Optional operator-supplied label identifying this sync config version. */
167
+ version_label?: string;
168
+ attributed_bucket_data_bytes: number;
169
+ attributed_parameter_indexes_bytes: number;
170
+ attributed_source_records_bytes: number;
171
+ attributed_object_storage_bytes: number;
154
172
  }
155
173
 
156
174
  export interface UpdateSyncRulesOptions {
175
+ /** Optional operator-supplied label identifying this deployed sync config version. */
176
+ version_label?: string;
157
177
  config: {
158
178
  yaml: string;
159
179
  /**
@@ -171,6 +191,12 @@ export interface UpdateSyncRulesOptions {
171
191
  lock?: boolean;
172
192
  storageVersion?: number;
173
193
 
194
+ /**
195
+ * Start a replacement replication stream instead of incrementally reprocessing a
196
+ * compatible active stream. Used when the current stream needs a full reprocess.
197
+ */
198
+ forceNewReplicationStream?: boolean;
199
+
174
200
  /**
175
201
  * Only relevant if the result is used. This does not affect the persisted config.
176
202
  */
@@ -181,14 +207,13 @@ export interface SerializedSyncPlan {
181
207
  /**
182
208
  * The serialized plan, from {@link serializeSyncPlan}.
183
209
  */
184
- plan: unknown;
210
+ plan: RawSerializedSyncPlan;
185
211
  compatibility: SerializedCompatibilityContext;
186
212
  /**
187
- * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
188
- * yet.
213
+ * Raw event SQL persisted as a compatibility mirror for compiled {@link plan} events.
189
214
  *
190
- * We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore
191
- * them.
215
+ * Compiled events are an additive plan field. Older services ignore that field and restore these descriptors through
216
+ * the legacy evaluator. Keep dual-writing this field until a future plan version explicitly removes that support.
192
217
  */
193
218
  eventDescriptors: Record<string, string[]>;
194
219
  errors?: ReplicationError[];
@@ -216,13 +241,14 @@ export function updateSyncRulesFromConfig(
216
241
  const { config, errors } = parsed;
217
242
  if (config instanceof PrecompiledSyncConfig) {
218
243
  const eventDescriptors: Record<string, string[]> = {};
219
- for (const event of config.eventDescriptors) {
220
- eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
244
+ for (const event of config.plan.events) {
245
+ eventDescriptors[event.name] = event.sourceQueries.map((query) => query.sql);
221
246
  }
222
247
 
223
248
  plan = {
224
249
  compatibility: config.compatibility.serialize(),
225
250
  plan: serializeSyncPlan(config.plan),
251
+ // Dual-write raw SQL so older services can ignore additive compiled plan events without losing event behavior.
226
252
  eventDescriptors,
227
253
  errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
228
254
  };
@@ -278,4 +304,5 @@ export interface TestStorageConfig {
278
304
  factory: TestStorageFactory;
279
305
  tableIdStrings: boolean;
280
306
  storageVersion?: number;
307
+ compressedBucketStorage?: boolean;
281
308
  }
@@ -0,0 +1,17 @@
1
+ import { InternalOpId } from '../util/util-index.js';
2
+
3
+ /**
4
+ * A checkpoint cannot be served because compaction rewrote a bucket-data
5
+ * document across its end boundary.
6
+ *
7
+ * The sync loop must skip this checkpoint before it sends its checkpoint line.
8
+ */
9
+ export class CheckpointChecksumInvalidatedError extends Error {
10
+ constructor(
11
+ public readonly checkpoint: InternalOpId,
12
+ public readonly bucket: string
13
+ ) {
14
+ super(`Checkpoint ${checkpoint} was invalidated by compaction in bucket ${bucket}`);
15
+ this.name = 'CheckpointChecksumInvalidatedError';
16
+ }
17
+ }
@@ -5,7 +5,7 @@ import { BucketChecksum } from '../util/protocol-types.js';
5
5
  import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
6
6
  import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
7
7
 
8
- interface ChecksumFetchContext {
8
+ interface ChecksumFetchContext<TContext> {
9
9
  fetch(bucket: string): Promise<BucketChecksum>;
10
10
  checkpoint: InternalOpId;
11
11
  }
@@ -20,26 +20,35 @@ export interface FetchPartialBucketChecksum {
20
20
  export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
21
21
  export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
22
22
 
23
- export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
23
+ export type FetchChecksums<TContext = undefined> = (
24
+ batch: FetchPartialBucketChecksum[],
25
+ context: TContext
26
+ ) => Promise<PartialChecksumMap>;
24
27
 
25
- export interface ChecksumCacheOptions {
28
+ export interface ChecksumCacheOptions<TContext = undefined> {
26
29
  /**
27
30
  * Upstream checksum implementation.
28
31
  *
29
32
  * This fetches a batch of either entire bucket checksums, or a partial range.
30
33
  */
31
- fetchChecksums: FetchChecksums;
34
+ fetchChecksums: FetchChecksums<TContext>;
32
35
 
33
36
  /**
34
37
  * Maximum number of cached checksums.
35
38
  */
36
39
  maxSize?: number;
40
+
41
+ /**
42
+ * How long to keep cached checksums before refreshing them.
43
+ */
44
+ ttlMs?: number;
37
45
  }
38
46
 
39
47
  // Approximately 5MB of memory, if we assume 50 bytes per entry
40
48
  const DEFAULT_MAX_SIZE = 100_000;
41
49
 
42
- const TTL_MS = 3_600_000;
50
+ // Default to 1h
51
+ export const DEFAULT_CHECKSUM_CACHE_TTL_MS = 1 * 3_600_000;
43
52
 
44
53
  /**
45
54
  * Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
@@ -50,22 +59,22 @@ const TTL_MS = 3_600_000;
50
59
  *
51
60
  * We use the LRUCache fetchMethod to deduplicate in-progress requests.
52
61
  */
53
- export class ChecksumCache {
62
+ export class ChecksumCache<TContext = undefined> {
54
63
  /**
55
64
  * The primary checksum cache, with key of `${checkpoint}/${bucket}`.
56
65
  */
57
- private cache: LRUCache<string, BucketChecksum, ChecksumFetchContext>;
66
+ private cache: LRUCache<string, BucketChecksum, ChecksumFetchContext<TContext>>;
58
67
  /**
59
68
  * For each bucket, an ordered set of cached checkpoints.
60
69
  */
61
70
  private bucketCheckpoints = new Map<string, OrderedSet<bigint>>();
62
71
 
63
- private fetchChecksums: FetchChecksums;
72
+ private fetchChecksums: FetchChecksums<TContext>;
64
73
 
65
- constructor(options: ChecksumCacheOptions) {
74
+ constructor(options: ChecksumCacheOptions<TContext>) {
66
75
  this.fetchChecksums = options.fetchChecksums;
67
76
 
68
- this.cache = new LRUCache<string, BucketChecksum, ChecksumFetchContext>({
77
+ this.cache = new LRUCache<string, BucketChecksum, ChecksumFetchContext<TContext>>({
69
78
  max: options.maxSize ?? DEFAULT_MAX_SIZE,
70
79
  fetchMethod: async (cacheKey, _staleValue, options) => {
71
80
  // Called when this checksum hasn't been cached yet.
@@ -105,7 +114,7 @@ export class ChecksumCache {
105
114
  // We use a TTL so that counts can eventually be refreshed
106
115
  // after a compact. This only has effect if the bucket has
107
116
  // not been checked in the meantime.
108
- ttl: TTL_MS,
117
+ ttl: options.ttlMs ?? DEFAULT_CHECKSUM_CACHE_TTL_MS,
109
118
  ttlResolution: 1_000,
110
119
  allowStale: false
111
120
  });
@@ -129,7 +138,11 @@ export class ChecksumCache {
129
138
  *
130
139
  * @returns a Map with exactly one entry for each bucket requested
131
140
  */
132
- async getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
141
+ async getChecksumMap(
142
+ checkpoint: InternalOpId,
143
+ buckets: BucketChecksumRequest[],
144
+ fetchContext?: TContext
145
+ ): Promise<ChecksumMap> {
133
146
  // Buckets that don't have a cached checksum for this checkpoint yet
134
147
  let toFetch = new Set<string>();
135
148
 
@@ -147,7 +160,7 @@ export class ChecksumCache {
147
160
  // Accumulated results - both from cached checksums, and fetched checksums
148
161
  let finalResults = new Map<string, BucketChecksum>();
149
162
 
150
- const context: ChecksumFetchContext = {
163
+ const context: ChecksumFetchContext<TContext> = {
151
164
  async fetch(bucket) {
152
165
  await fetchPromise;
153
166
  if (!toFetch.has(bucket)) {
@@ -260,7 +273,7 @@ export class ChecksumCache {
260
273
  }
261
274
 
262
275
  // Fetch partial checksums from upstream
263
- const results = await this.fetchChecksums(bucketRequests);
276
+ const results = await this.fetchChecksums(bucketRequests, fetchContext as TContext);
264
277
 
265
278
  for (let bucket of toFetch) {
266
279
  const result = results.get(bucket);
@@ -0,0 +1,25 @@
1
+ import { HydratedSyncConfig, HydrationState, SyncConfigWithErrors } from '@powersync/service-sync-rules';
2
+
3
+ /**
4
+ * A parsed sync config set is the identity boundary for parsed sync config state.
5
+ *
6
+ * All values on a single instance - the parsed configs, hydration state and hydrated
7
+ * sync config - are derived from the same parse, and may be associated with each other
8
+ * by object identity. Values from two different instances must never be mixed, even if
9
+ * they were parsed from the same persisted content.
10
+ */
11
+ export interface ParsedSyncConfigSet {
12
+ readonly replicationStreamId: number;
13
+ readonly replicationStreamName: string;
14
+ readonly syncConfigs: SyncConfigWithErrors[];
15
+ /**
16
+ * For testing only.
17
+ */
18
+ readonly hydrationState: HydrationState;
19
+
20
+ /**
21
+ * The hydrated sync config for this parse. This is a stable property: repeated reads
22
+ * return the same instance.
23
+ */
24
+ readonly hydratedSyncConfig: HydratedSyncConfig;
25
+ }
@@ -0,0 +1,55 @@
1
+ import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
+ import { Logger } from 'winston';
3
+ import { SyncRuleState } from './BucketStorage.js';
4
+ import type { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
5
+ import type { ParseSyncConfigOptions, PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
6
+ import { ReplicationLock } from './ReplicationLock.js';
7
+ import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
8
+
9
+ export abstract class PersistedReplicationStream implements PersistedReplicationStreamData {
10
+ readonly replicationStreamId: number;
11
+ readonly replicationJobId: string;
12
+ readonly replicationStreamName: string;
13
+ readonly state: SyncRuleState;
14
+ readonly storageVersion: number;
15
+ readonly logger: Logger;
16
+
17
+ abstract readonly syncConfigContent: readonly PersistedSyncConfigContent[];
18
+ abstract readonly current_lock: ReplicationLock | null;
19
+
20
+ constructor(data: PersistedReplicationStreamData) {
21
+ this.replicationStreamId = data.replicationStreamId;
22
+ this.replicationJobId = data.replicationJobId ?? String(data.replicationStreamId);
23
+ this.replicationStreamName = data.replicationStreamName;
24
+ this.state = data.state;
25
+ this.storageVersion = data.storageVersion;
26
+ this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
27
+ }
28
+
29
+ getStorageConfig(): StorageVersionConfig {
30
+ const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
31
+ if (storageConfig == null) {
32
+ throw new ServiceError(
33
+ ErrorCode.PSYNC_S1005,
34
+ `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`
35
+ );
36
+ }
37
+ return storageConfig;
38
+ }
39
+
40
+ abstract lock(): Promise<ReplicationLock>;
41
+
42
+ abstract parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
43
+ }
44
+ export interface PersistedReplicationStreamData {
45
+ readonly replicationStreamId: number;
46
+ readonly replicationStreamName: string;
47
+ readonly state: SyncRuleState;
48
+ readonly storageVersion: number;
49
+ /**
50
+ * Uniquely identifies a job, as a combination of replication stream and processing sync configs.
51
+ *
52
+ * Used in-memory only.
53
+ */
54
+ readonly replicationJobId?: string;
55
+ }