@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +152 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +36 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +30 -22
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/metrics/metrics-interfaces.d.ts +5 -1
  9. package/dist/metrics/metrics-interfaces.js.map +1 -1
  10. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
  11. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
  12. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
  13. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
  14. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
  15. package/dist/metrics/open-telemetry/util.js +7 -3
  16. package/dist/metrics/open-telemetry/util.js.map +1 -1
  17. package/dist/modules/AbstractModule.d.ts +6 -2
  18. package/dist/modules/AbstractModule.js.map +1 -1
  19. package/dist/replication/AbstractReplicator.d.ts +31 -7
  20. package/dist/replication/AbstractReplicator.js +168 -74
  21. package/dist/replication/AbstractReplicator.js.map +1 -1
  22. package/dist/replication/ReplicationModule.d.ts +3 -3
  23. package/dist/replication/ReplicationModule.js.map +1 -1
  24. package/dist/routes/RouterEngine.d.ts +1 -0
  25. package/dist/routes/RouterEngine.js +6 -1
  26. package/dist/routes/RouterEngine.js.map +1 -1
  27. package/dist/routes/configure-fastify.d.ts +104 -56
  28. package/dist/routes/configure-fastify.js +15 -2
  29. package/dist/routes/configure-fastify.js.map +1 -1
  30. package/dist/routes/endpoints/admin.d.ts +42 -24
  31. package/dist/routes/endpoints/admin.js +49 -36
  32. package/dist/routes/endpoints/admin.js.map +1 -1
  33. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  34. package/dist/routes/endpoints/checkpointing.js +65 -7
  35. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  36. package/dist/routes/endpoints/socket-route.js +3 -2
  37. package/dist/routes/endpoints/socket-route.js.map +1 -1
  38. package/dist/routes/endpoints/sync-rules.js +25 -14
  39. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  40. package/dist/routes/endpoints/sync-stream.js +1 -1
  41. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  42. package/dist/runner/teardown.js +8 -8
  43. package/dist/runner/teardown.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +64 -17
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/BucketStorageFactory.d.ts +50 -31
  47. package/dist/storage/BucketStorageFactory.js +10 -20
  48. package/dist/storage/BucketStorageFactory.js.map +1 -1
  49. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  50. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  51. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  52. package/dist/storage/ChecksumCache.d.ts +11 -6
  53. package/dist/storage/ChecksumCache.js +5 -4
  54. package/dist/storage/ChecksumCache.js.map +1 -1
  55. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  56. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  57. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  58. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  59. package/dist/storage/PersistedReplicationStream.js +26 -0
  60. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  61. package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
  62. package/dist/storage/PersistedSyncConfigContent.js +138 -0
  63. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  64. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  65. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  66. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  67. package/dist/storage/ReplicationEventPayload.d.ts +6 -1
  68. package/dist/storage/SourceEntity.d.ts +13 -2
  69. package/dist/storage/SourceTable.d.ts +53 -4
  70. package/dist/storage/SourceTable.js +62 -7
  71. package/dist/storage/SourceTable.js.map +1 -1
  72. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  73. package/dist/storage/SourceTableReconciler.js +95 -0
  74. package/dist/storage/SourceTableReconciler.js.map +1 -0
  75. package/dist/storage/StorageVersionConfig.d.ts +4 -0
  76. package/dist/storage/StorageVersionConfig.js +10 -1
  77. package/dist/storage/StorageVersionConfig.js.map +1 -1
  78. package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
  79. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  80. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  81. package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
  82. package/dist/storage/WriteCheckpointAPI.js +33 -0
  83. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  84. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
  85. package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
  86. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  87. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  88. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  89. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  90. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  91. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
  92. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  93. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  94. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  95. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  96. package/dist/storage/storage-index.d.ts +10 -1
  97. package/dist/storage/storage-index.js +10 -1
  98. package/dist/storage/storage-index.js.map +1 -1
  99. package/dist/storage/storage-metrics.js +120 -4
  100. package/dist/storage/storage-metrics.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +12 -2
  102. package/dist/sync/BucketChecksumState.js +69 -21
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.d.ts +7 -0
  105. package/dist/sync/RequestTracker.js +12 -0
  106. package/dist/sync/RequestTracker.js.map +1 -1
  107. package/dist/sync/sync.js +184 -43
  108. package/dist/sync/sync.js.map +1 -1
  109. package/dist/sync/util.d.ts +5 -0
  110. package/dist/sync/util.js +19 -1
  111. package/dist/sync/util.js.map +1 -1
  112. package/dist/system/ServiceContext.d.ts +2 -0
  113. package/dist/system/ServiceContext.js +2 -0
  114. package/dist/system/ServiceContext.js.map +1 -1
  115. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  116. package/dist/tracing/PerformanceTracer.js +11 -0
  117. package/dist/tracing/PerformanceTracer.js.map +1 -1
  118. package/dist/util/checkpointing.d.ts +4 -8
  119. package/dist/util/checkpointing.js +1 -16
  120. package/dist/util/checkpointing.js.map +1 -1
  121. package/dist/util/config/compound-config-collector.js +23 -2
  122. package/dist/util/config/compound-config-collector.js.map +1 -1
  123. package/dist/util/config/defaults.d.ts +2 -0
  124. package/dist/util/config/defaults.js +2 -0
  125. package/dist/util/config/defaults.js.map +1 -1
  126. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  127. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  128. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  130. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  131. package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
  132. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  133. package/dist/util/config/types.d.ts +4 -1
  134. package/dist/util/lsn.d.ts +4 -0
  135. package/dist/util/lsn.js +10 -0
  136. package/dist/util/lsn.js.map +1 -1
  137. package/dist/util/protocol-types.d.ts +4 -4
  138. package/dist/util/protocol-types.js +3 -2
  139. package/dist/util/protocol-types.js.map +1 -1
  140. package/dist/util/util-index.d.ts +1 -0
  141. package/dist/util/util-index.js +1 -0
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +3 -2
  144. package/dist/util/utils.js +17 -2
  145. package/dist/util/utils.js.map +1 -1
  146. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  147. package/dist/util/write-checkpoint-batcher.js +100 -0
  148. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  149. package/package.json +7 -7
  150. package/src/api/RouteAPI.ts +43 -14
  151. package/src/api/diagnostics.ts +45 -26
  152. package/src/entry/commands/compact-action.ts +37 -23
  153. package/src/metrics/metrics-interfaces.ts +6 -1
  154. package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
  155. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
  156. package/src/metrics/open-telemetry/util.ts +7 -3
  157. package/src/modules/AbstractModule.ts +6 -2
  158. package/src/replication/AbstractReplicator.ts +212 -78
  159. package/src/replication/ReplicationModule.ts +2 -3
  160. package/src/routes/RouterEngine.ts +7 -1
  161. package/src/routes/configure-fastify.ts +25 -2
  162. package/src/routes/endpoints/admin.ts +75 -51
  163. package/src/routes/endpoints/checkpointing.ts +79 -8
  164. package/src/routes/endpoints/socket-route.ts +3 -2
  165. package/src/routes/endpoints/sync-rules.ts +26 -14
  166. package/src/routes/endpoints/sync-stream.ts +1 -1
  167. package/src/runner/teardown.ts +8 -8
  168. package/src/storage/BucketStorageBatch.ts +65 -18
  169. package/src/storage/BucketStorageFactory.ts +73 -46
  170. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  171. package/src/storage/ChecksumCache.ts +27 -14
  172. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  173. package/src/storage/PersistedReplicationStream.ts +55 -0
  174. package/src/storage/PersistedSyncConfigContent.ts +207 -0
  175. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  176. package/src/storage/ReplicationEventPayload.ts +6 -1
  177. package/src/storage/SourceEntity.ts +12 -2
  178. package/src/storage/SourceTable.ts +96 -8
  179. package/src/storage/SourceTableReconciler.ts +188 -0
  180. package/src/storage/StorageVersionConfig.ts +11 -1
  181. package/src/storage/SyncRulesBucketStorage.ts +189 -27
  182. package/src/storage/WriteCheckpointAPI.ts +140 -6
  183. package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
  184. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  185. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
  186. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  187. package/src/storage/storage-index.ts +11 -1
  188. package/src/storage/storage-metrics.ts +154 -8
  189. package/src/sync/BucketChecksumState.ts +105 -21
  190. package/src/sync/RequestTracker.ts +20 -0
  191. package/src/sync/sync.ts +214 -50
  192. package/src/sync/util.ts +33 -8
  193. package/src/system/ServiceContext.ts +7 -0
  194. package/src/tracing/PerformanceTracer.ts +41 -1
  195. package/src/util/checkpointing.ts +4 -22
  196. package/src/util/config/compound-config-collector.ts +31 -1
  197. package/src/util/config/defaults.ts +2 -0
  198. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  199. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  200. package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
  201. package/src/util/config/types.ts +4 -1
  202. package/src/util/lsn.ts +9 -0
  203. package/src/util/protocol-types.ts +3 -2
  204. package/src/util/util-index.ts +1 -0
  205. package/src/util/utils.ts +16 -2
  206. package/src/util/write-checkpoint-batcher.ts +142 -0
  207. package/test/src/AbstractReplicator.test.ts +177 -0
  208. package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
  209. package/test/src/PersistedSyncConfigContent.test.ts +152 -0
  210. package/test/src/checksum_cache.test.ts +22 -0
  211. package/test/src/config.test.ts +142 -0
  212. package/test/src/diagnostics.test.ts +87 -34
  213. package/test/src/module-loader.test.ts +33 -1
  214. package/test/src/routes/admin.test.ts +188 -18
  215. package/test/src/routes/checkpointing.test.ts +54 -0
  216. package/test/src/routes/mocks.ts +6 -2
  217. package/test/src/routes/stream.test.ts +123 -2
  218. package/test/src/source-table-reconciler.test.ts +244 -0
  219. package/test/src/storage/SourceTable.test.ts +92 -0
  220. package/test/src/storage-metrics.test.ts +148 -0
  221. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  222. package/test/src/sync/util.test.ts +28 -1
  223. package/test/src/util/checkpointing.test.ts +249 -0
  224. package/test/src/util/utils.test.ts +23 -0
  225. package/tsconfig.json +1 -4
  226. package/tsconfig.tsbuildinfo +1 -1
  227. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  228. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  229. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  230. package/src/storage/PersistedSyncRulesContent.ts +0 -174
package/src/sync/sync.ts CHANGED
@@ -1,18 +1,25 @@
1
1
  import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
2
2
  import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
3
3
 
4
- import { AbortError } from 'ix/aborterror.js';
5
-
6
4
  import * as auth from '../auth/auth-index.js';
7
5
  import * as storage from '../storage/storage-index.js';
8
6
  import * as util from '../util/util-index.js';
9
7
 
10
8
  import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
9
+ import { isBatchEnd } from '../storage/storage-index.js';
11
10
  import { mergeAsyncIterables } from '../streams/streams-index.js';
12
- import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
11
+ import { PerformanceTracer, type Span } from '../tracing/PerformanceTracer.js';
12
+ import { BucketChecksumState, CheckpointLine, type SyncCheckpointTraceCategory } from './BucketChecksumState.js';
13
13
  import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
14
14
  import { SyncContext } from './SyncContext.js';
15
- import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
15
+ import { TokenStreamOptions, acquireSemaphoreAbortable, isAbortError, settledPromise, tokenStream } from './util.js';
16
+
17
+ type CheckpointTiming = Record<string, number>;
18
+
19
+ interface ActiveCheckpointTrace {
20
+ tracer: PerformanceTracer<SyncCheckpointTraceCategory>;
21
+ span: Span;
22
+ }
16
23
 
17
24
  export interface SyncStreamParameters {
18
25
  syncContext: SyncContext;
@@ -79,7 +86,7 @@ export async function* streamResponse(
79
86
  try {
80
87
  yield* merged;
81
88
  } catch (e) {
82
- if (e instanceof AbortError) {
89
+ if (isAbortError(e)) {
83
90
  return;
84
91
  } else {
85
92
  throw e;
@@ -112,15 +119,21 @@ async function* streamResponseInner(
112
119
  syncRequest: params,
113
120
  logger: logger
114
121
  });
122
+ // The checkpoint iterator can have a pending next() while a data batch fails.
123
+ // Aborting this on cleanup makes that pending next() settle, so the watcher subscription
124
+ // is released instead of staying open until another checkpoint arrives. See the cleanup
125
+ // in the finally block below.
126
+ const checkpointWatchController = new AbortController();
115
127
  const stream = bucketStorage.watchCheckpointChanges({
116
128
  user_id: checkpointUserId,
117
- signal
129
+ signal: AbortSignal.any([signal, checkpointWatchController.signal])
118
130
  });
119
131
  const newCheckpoints = stream[Symbol.asyncIterator]();
120
132
 
121
133
  type CheckpointAndLine = {
122
- checkpoint: bigint;
134
+ checkpoint: storage.ReplicationCheckpoint;
123
135
  line: CheckpointLine | null;
136
+ trace: ActiveCheckpointTrace | null;
124
137
  };
125
138
 
126
139
  async function waitForNewCheckpointLine(): Promise<IteratorResult<CheckpointAndLine>> {
@@ -128,11 +141,50 @@ async function* streamResponseInner(
128
141
  if (next.done) {
129
142
  return { done: true, value: undefined };
130
143
  }
144
+ const cp = next.value.base;
145
+
146
+ const tracer = new PerformanceTracer<SyncCheckpointTraceCategory>(`sync checkpoint ${cp.checkpoint}`);
147
+ const trace = {
148
+ tracer,
149
+ span: tracer.span('checkpoint')
150
+ };
151
+
152
+ try {
153
+ const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer);
154
+ return { done: false, value: { checkpoint: cp, line, trace: line == null ? null : trace } };
155
+ } catch (e) {
156
+ if (e instanceof storage.CheckpointChecksumInvalidatedError) {
157
+ // The checksum was not usable, so buildNextCheckpointLine has not advanced
158
+ // the connection state. Drop this candidate and wait for a checkpoint that
159
+ // is not split by a compaction-produced bucket-data document.
160
+ // This is different from other checkpoint_invalidated cases in that we hit
161
+ // this during checksum calculation, instead of on data read.
162
+ trace.span.end();
163
+ checksumState.invalidateChecksumBaseline();
164
+ logger.info(`checkpoint_invalidated: ${cp.checkpoint}`, {
165
+ reason: 'compacted_before_checkpoint_line',
166
+ bucket: e.bucket,
167
+ checkpoint: cp.checkpoint,
168
+ user_id: tokenPayload.userIdJson
169
+ });
170
+ return { done: false, value: { checkpoint: cp, line: null, trace: null } };
171
+ }
172
+ // Only end the span if we error. If we return normally, we pass ownership on to the caller.
173
+ trace.span.end();
174
+ throw e;
175
+ }
176
+ }
131
177
 
132
- const line = await checksumState.buildNextCheckpointLine(next.value);
133
- return { done: false, value: { checkpoint: next.value.base.checkpoint, line } };
178
+ function checkpointLogDetails(cp: storage.ReplicationCheckpoint) {
179
+ return {
180
+ checkpoint: cp.checkpoint,
181
+ user_id: tokenPayload.userIdJson,
182
+ ...tracker.getIncrementalCheckpointStats()
183
+ };
134
184
  }
135
185
 
186
+ let activeCheckpoint: { checkpoint: storage.ReplicationCheckpoint; trace: ActiveCheckpointTrace } | null = null;
187
+
136
188
  try {
137
189
  let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
138
190
 
@@ -155,12 +207,19 @@ async function* streamResponseInner(
155
207
  // No update to sync
156
208
  continue;
157
209
  }
210
+ const trace = next.value.value.trace!;
211
+ const checkpoint = next.value.value.checkpoint;
212
+ const tracer = trace.tracer;
213
+ activeCheckpoint = { checkpoint, trace };
158
214
 
159
- const { checkpointLine, bucketsToFetch } = line;
215
+ const { checkpointLine, bucketsToFetch, bucketDataRequestHint } = line;
160
216
 
161
217
  // Since yielding can block, we update the state just before yielding the line.
162
218
  line.advance();
163
- yield checkpointLine;
219
+ {
220
+ using _ = tracer.span('sending', 'checkpoint');
221
+ yield checkpointLine;
222
+ }
164
223
 
165
224
  // Start syncing data for buckets up to the checkpoint. As soon as we have completed at least one priority and
166
225
  // at least 1000 operations, we also start listening for new checkpoints concurrently. When a new checkpoint comes
@@ -169,6 +228,8 @@ async function* streamResponseInner(
169
228
  const abortCheckpointController = new AbortController();
170
229
  let syncedOperations = 0;
171
230
 
231
+ let checkpointResult: CheckpointResult | null = null;
232
+
172
233
  const abortCheckpointSignal = AbortSignal.any([abortCheckpointController.signal, signal]);
173
234
 
174
235
  const bucketsByPriority = [...Map.groupBy(bucketsToFetch, (bucket) => bucket.priority).entries()];
@@ -190,6 +251,11 @@ async function* streamResponseInner(
190
251
  while (true) {
191
252
  const next = await settledPromise(waitForNewCheckpointLine());
192
253
  if (next.status == 'rejected') {
254
+ if (isAbortError(next.reason)) {
255
+ checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_cancelled' };
256
+ } else {
257
+ checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_error' };
258
+ }
193
259
  abortCheckpointController.abort();
194
260
  } else if (!next.value.done) {
195
261
  if (next.value.value.line == null) {
@@ -200,6 +266,7 @@ async function* streamResponseInner(
200
266
 
201
267
  // A new sync line can be emitted. Stop running the bucketDataInBatches() iterations, making the
202
268
  // main flow reach the new checkpoint.
269
+ checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_superseded' };
203
270
  abortCheckpointController.abort();
204
271
  }
205
272
 
@@ -223,11 +290,12 @@ async function* streamResponseInner(
223
290
  break;
224
291
  }
225
292
 
226
- yield* bucketDataInBatches({
293
+ const batchResult = yield* bucketDataInBatches({
227
294
  syncContext: syncContext,
228
295
  bucketStorage: bucketStorage,
229
- checkpoint: next.value.value.checkpoint,
296
+ checkpoint,
230
297
  bucketsToFetch: buckets,
298
+ requestHint: bucketDataRequestHint,
231
299
  checkpointLine: line,
232
300
  legacyDataLines: !isEncodingAsBson && params.raw_data != true,
233
301
  onRowsSent: markOperationsSent,
@@ -237,8 +305,39 @@ async function* streamResponseInner(
237
305
  // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
238
306
  // sync complete message instead.
239
307
  forPriority: !isLast ? priority : null,
240
- logger
308
+ logger,
309
+ tracer
310
+ });
311
+ if (batchResult?.result == 'partial_checkpoint_complete') {
312
+ // We don't specifically include timing details here, since that would end the span.
313
+ // Timings are included in the checkpoint_complete following this.
314
+ logger.info(`partial_checkpoint_complete: ${checkpoint.checkpoint}`, {
315
+ priority,
316
+ ...checkpointLogDetails(checkpoint)
317
+ });
318
+ } else if (batchResult?.result != null) {
319
+ checkpointResult = batchResult;
320
+ break;
321
+ }
322
+ }
323
+
324
+ if (checkpointResult?.result == 'checkpoint_complete') {
325
+ logger.info(`checkpoint_complete: ${checkpoint.checkpoint}`, {
326
+ // Incremental stats since the last checkpoint_complete, partial_checkpoint_complete or checkpoint_invalidated
327
+ ...checkpointLogDetails(checkpoint),
328
+ // Timings since the last checkpoint or checkpoint_diff
329
+ ms: getCheckpointTraceTimings(trace.span)
241
330
  });
331
+ } else if (checkpointResult?.result == 'invalidated') {
332
+ // A new checkpoint and checkpoint_complete should follow soon, but we log the stats already
333
+ logger.info(`checkpoint_invalidated: ${checkpoint.checkpoint}`, {
334
+ reason: checkpointResult.invalidationReason,
335
+ ...checkpointLogDetails(checkpoint),
336
+ ms: getCheckpointTraceTimings(trace.span)
337
+ });
338
+ }
339
+ if (checkpointResult != null) {
340
+ activeCheckpoint = null;
242
341
  }
243
342
 
244
343
  if (!abortCheckpointSignal.aborted) {
@@ -246,14 +345,33 @@ async function* streamResponseInner(
246
345
  }
247
346
  } while (!signal.aborted);
248
347
  } finally {
249
- await newCheckpoints.return?.();
348
+ if (activeCheckpoint != null) {
349
+ const { checkpoint, trace } = activeCheckpoint;
350
+ logger.info(`checkpoint_interrupted: ${checkpoint.checkpoint}`, {
351
+ ...checkpointLogDetails(checkpoint),
352
+ ms: getCheckpointTraceTimings(trace.span)
353
+ });
354
+ }
355
+
356
+ // The abort makes a pending next() settle as soon as the storage watcher observes it,
357
+ // releasing the subscription. return() is still needed for the case where there is no
358
+ // pending next(): the watcher is then suspended at a yield and never observes the abort.
359
+ // Neither is awaited: async generators queue return() behind a pending next(), so
360
+ // awaiting it here would keep the connection open until the next checkpoint arrives.
361
+ checkpointWatchController.abort();
362
+ newCheckpoints.return?.().catch((e) => {
363
+ if (!isAbortError(e)) {
364
+ logger.warn('Error while closing the checkpoint watcher', e);
365
+ }
366
+ });
250
367
  }
251
368
  }
252
369
 
253
370
  interface BucketDataRequest {
254
371
  syncContext: SyncContext;
255
372
  bucketStorage: storage.SyncRulesBucketStorage;
256
- checkpoint: util.InternalOpId;
373
+ checkpoint: storage.ReplicationCheckpoint;
374
+ requestHint: storage.BucketRequestHint;
257
375
  /** Contains current bucket state. Modified by the request as data is sent. */
258
376
  checkpointLine: CheckpointLine;
259
377
  /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
@@ -272,9 +390,20 @@ interface BucketDataRequest {
272
390
  forPriority: BucketPriority | null;
273
391
  onRowsSent: (stats: OperationsSentStats) => void;
274
392
  logger: Logger;
393
+ tracer: PerformanceTracer<SyncCheckpointTraceCategory>;
275
394
  }
276
395
 
277
- async function* bucketDataInBatches(request: BucketDataRequest) {
396
+ type CheckpointResult =
397
+ | { result: 'partial_checkpoint_complete' }
398
+ | { result: 'checkpoint_complete' }
399
+ | {
400
+ result: 'invalidated';
401
+ invalidationReason: 'compacted' | 'checkpoint_error' | 'checkpoint_superseded' | 'checkpoint_cancelled';
402
+ };
403
+
404
+ async function* bucketDataInBatches(
405
+ request: BucketDataRequest
406
+ ): AsyncGenerator<util.StreamingSyncLine | string | null, CheckpointResult | null> {
278
407
  let isDone = false;
279
408
  while (!request.abort_batch.aborted && !isDone) {
280
409
  // The code below is functionally the same as this for-await loop below.
@@ -291,9 +420,13 @@ async function* bucketDataInBatches(request: BucketDataRequest) {
291
420
  while (true) {
292
421
  const { value, done: iterDone } = await iter.next();
293
422
  if (iterDone) {
423
+ if (value != null) {
424
+ return value;
425
+ }
294
426
  break;
295
427
  } else {
296
428
  const { done, data } = value;
429
+ using _ = request.tracer.span('sending', 'data');
297
430
  yield data;
298
431
  if (done) {
299
432
  isDone = true;
@@ -301,9 +434,10 @@ async function* bucketDataInBatches(request: BucketDataRequest) {
301
434
  }
302
435
  }
303
436
  } finally {
304
- await iter.return();
437
+ await iter.return(null);
305
438
  }
306
439
  }
440
+ return null;
307
441
  }
308
442
 
309
443
  interface BucketDataBatchResult {
@@ -314,11 +448,14 @@ interface BucketDataBatchResult {
314
448
  /**
315
449
  * Extracted as a separate internal function just to avoid memory leaks.
316
450
  */
317
- async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
451
+ async function* bucketDataBatch(
452
+ request: BucketDataRequest
453
+ ): AsyncGenerator<BucketDataBatchResult, CheckpointResult | null> {
318
454
  const {
319
455
  syncContext,
320
456
  bucketStorage: storage,
321
457
  checkpoint,
458
+ requestHint,
322
459
  bucketsToFetch,
323
460
  checkpointLine,
324
461
  legacyDataLines,
@@ -328,45 +465,57 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
328
465
  logger
329
466
  } = request;
330
467
 
468
+ const tracer = request.tracer;
331
469
  let checkpointInvalidated = false;
332
470
 
333
- if (syncContext.syncSemaphore.isLocked()) {
334
- logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.userIdForLogs });
335
- }
336
- const acquired = await acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
471
+ const acquired = await tracer.span('acquiring_lock').with(async () => {
472
+ return acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
473
+ });
337
474
  if (acquired === 'aborted') {
338
- return;
475
+ return null;
339
476
  }
340
477
 
341
- const [value, release] = acquired;
478
+ const [, release] = acquired;
342
479
  try {
343
- if (value <= 3) {
344
- // This can be noisy, so we only log when we get close to the
345
- // concurrency limit.
346
- logger.info(`Got sync lock. Slots available: ${value - 1}`, {
347
- user_id: request.userIdForLogs,
348
- sync_data_slots: value - 1
349
- });
350
- }
480
+ using _ = tracer.span('bucket_data');
481
+ let has_more = false;
351
482
  // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
352
483
  // For the first batch, this will be all buckets.
353
484
  const filteredBuckets = checkpointLine.getFilteredBucketPositions(bucketsToFetch);
354
- const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets);
355
-
356
- let has_more = false;
357
-
358
- for await (let { chunkData: r, targetOp } of dataBatches) {
485
+ const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, {
486
+ requestHint,
487
+ tracer,
488
+ // Checkpoint supersession is a cooperative batch handoff. Only abort
489
+ // in-flight storage work when the connection itself is closed.
490
+ signal: abort_connection
491
+ });
492
+ for await (let chunk of dataBatches) {
359
493
  // Abort in current batch if the connection is closed
360
494
  if (abort_connection.aborted) {
361
- return;
495
+ return null;
362
496
  }
497
+ if (isBatchEnd(chunk)) {
498
+ // This replaces any other has_more value, since the batch end is the last chunk.
499
+ has_more = chunk.hasMore;
500
+ break;
501
+ }
502
+ const { chunkData: r, targetOp } = chunk;
363
503
  if (r.has_more) {
364
504
  has_more = true;
365
505
  }
366
- if (targetOp != null && targetOp > checkpoint) {
506
+ if (targetOp != null && targetOp > checkpoint.checkpoint) {
367
507
  checkpointInvalidated = true;
368
508
  }
369
509
  if (r.data.length == 0) {
510
+ // An empty chunk is not sent to the client, but may still report progress:
511
+ // storage can emit an empty chunk for a bucket that has no operations in the
512
+ // requested range, and its position must advance so the bucket is not
513
+ // re-requested indefinitely.
514
+ checkpointLine.updateBucketPosition({
515
+ bucket: r.bucket,
516
+ nextAfter: BigInt(r.next_after),
517
+ hasMore: r.has_more
518
+ });
370
519
  continue;
371
520
  }
372
521
  logger.debug(`Sending data for ${r.bucket}`);
@@ -393,7 +542,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
393
542
  // Check if syncing bucket data is supposed to stop before fetching more data
394
543
  // from storage.
395
544
  if (abort_batch.aborted) {
396
- return;
545
+ return null;
397
546
  }
398
547
  }
399
548
 
@@ -403,35 +552,50 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
403
552
  // Don't send the checkpoint_complete line in this case.
404
553
  // More data should be available immediately for a new checkpoint.
405
554
  yield { data: null, done: true };
555
+ return { result: 'invalidated', invalidationReason: 'compacted' };
406
556
  } else {
407
557
  if (request.forPriority != null) {
408
558
  const line: util.StreamingSyncCheckpointPartiallyComplete = {
409
559
  partial_checkpoint_complete: {
410
- last_op_id: util.internalToExternalOpId(checkpoint),
560
+ last_op_id: util.internalToExternalOpId(checkpoint.checkpoint),
411
561
  priority: request.forPriority
412
562
  }
413
563
  };
414
564
  yield { data: line, done: true };
565
+ return { result: 'partial_checkpoint_complete' };
415
566
  } else {
416
567
  const line: util.StreamingSyncCheckpointComplete = {
417
568
  checkpoint_complete: {
418
- last_op_id: util.internalToExternalOpId(checkpoint)
569
+ last_op_id: util.internalToExternalOpId(checkpoint.checkpoint)
419
570
  }
420
571
  };
421
572
  yield { data: line, done: true };
573
+ return { result: 'checkpoint_complete' };
422
574
  }
423
575
  }
424
576
  }
425
577
  } finally {
426
- if (value <= 3) {
427
- // This can be noisy, so we only log when we get close to the
428
- // concurrency limit.
429
- logger.info(`Releasing sync lock`, {
430
- user_id: request.userIdForLogs
431
- });
432
- }
433
578
  release();
434
579
  }
580
+ return null;
581
+ }
582
+
583
+ function getCheckpointTraceTimings(span: Span): CheckpointTiming {
584
+ const timings = span.end();
585
+ const result: CheckpointTiming = {};
586
+ for (let key in timings) {
587
+ addTiming(result, key, timings[key]);
588
+ }
589
+ addTiming(result, 'other', span.selfDuration);
590
+ addTiming(result, 'total', span.endAt - span.startAt);
591
+ return result;
592
+ }
593
+
594
+ function addTiming(timings: CheckpointTiming, key: string, durationMicros: number) {
595
+ const ms = Math.round(durationMicros / 1000);
596
+ if (ms > 0) {
597
+ timings[key] = (timings[key] ?? 0) + ms;
598
+ }
435
599
  }
436
600
 
437
601
  function transformLegacyResponse(bucketData: util.SyncBucketData): util.SyncBucketData<util.ProtocolOplogData> {
package/src/sync/util.ts CHANGED
@@ -2,6 +2,7 @@ import * as timers from 'timers/promises';
2
2
 
3
3
  import { SemaphoreInterface } from 'async-mutex';
4
4
  import { serialize } from 'bson';
5
+ import { AbortError } from 'ix/aborterror.js';
5
6
  import * as util from '../util/util-index.js';
6
7
  import { RequestTracker } from './RequestTracker.js';
7
8
 
@@ -23,6 +24,17 @@ const DEFAULT_TOKEN_STREAM_OPTIONS: TokenStreamOptions = {
23
24
  expire_warning_period: 20_000
24
25
  };
25
26
 
27
+ /**
28
+ * Recognize both ix's AbortError and native abort errors, which Node represents
29
+ * as DOMExceptions with the name "AbortError".
30
+ */
31
+ export function isAbortError(error: unknown): boolean {
32
+ return (
33
+ error instanceof AbortError ||
34
+ (typeof error == 'object' && error != null && 'name' in error && error.name === 'AbortError')
35
+ );
36
+ }
37
+
26
38
  /**
27
39
  * An iterator that periodically yields token and optionally keepalive events, and returns once the
28
40
  * provided token expiry is reached.
@@ -135,6 +147,13 @@ export function acquireSemaphoreAbortable(
135
147
  abort: AbortSignal
136
148
  ): Promise<[number, SemaphoreInterface.Releaser] | 'aborted'> {
137
149
  return new Promise((resolve, reject) => {
150
+ // An already-aborted signal never fires its listener, and we would wait for the semaphore
151
+ // indefinitely.
152
+ if (abort.aborted) {
153
+ resolve('aborted');
154
+ return;
155
+ }
156
+
138
157
  let aborted = false;
139
158
  let hasSemaphore = false;
140
159
 
@@ -147,16 +166,22 @@ export function acquireSemaphoreAbortable(
147
166
  };
148
167
  abort.addEventListener('abort', listener);
149
168
 
150
- semaphone.acquire().then((acquired) => {
151
- hasSemaphore = true;
152
- if (aborted) {
153
- // Release semaphore, already aborted
154
- acquired[1]();
155
- } else {
169
+ semaphone.acquire().then(
170
+ (acquired) => {
171
+ hasSemaphore = true;
172
+ if (aborted) {
173
+ // Release semaphore, already aborted
174
+ acquired[1]();
175
+ } else {
176
+ abort.removeEventListener('abort', listener);
177
+ resolve(acquired);
178
+ }
179
+ },
180
+ (error) => {
156
181
  abort.removeEventListener('abort', listener);
157
- resolve(acquired);
182
+ reject(error);
158
183
  }
159
- }, reject);
184
+ );
160
185
  });
161
186
  }
162
187
 
@@ -19,6 +19,7 @@ export interface ServiceContext {
19
19
  storageEngine: storage.StorageEngine;
20
20
  migrations: PowerSyncMigrationManager;
21
21
  syncContext: SyncContext;
22
+ writeCheckpointBatcher: utils.WriteCheckpointBatcher;
22
23
  serviceMode: ServiceContextMode;
23
24
  eventsEngine: EventsEngine;
24
25
  }
@@ -50,6 +51,7 @@ export class ServiceContextContainer implements ServiceContext {
50
51
  eventsEngine: EventsEngine;
51
52
  syncContext: SyncContext;
52
53
  routerEngine: routes.RouterEngine;
54
+ writeCheckpointBatcher: utils.WriteCheckpointBatcher;
53
55
  serviceMode: ServiceContextMode;
54
56
 
55
57
  constructor(options: ServiceContextOptions) {
@@ -84,6 +86,11 @@ export class ServiceContextContainer implements ServiceContext {
84
86
  stop: (routerEngine) => routerEngine.shutDown()
85
87
  });
86
88
 
89
+ this.writeCheckpointBatcher = new utils.WriteCheckpointBatcher(
90
+ () => this.routerEngine.getAPI(),
91
+ () => this.storageEngine.activeBucketStorage
92
+ );
93
+
87
94
  this.syncContext = new SyncContext({
88
95
  maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
89
96
  maxBuckets: configuration.api_parameters.max_buckets_per_connection,
@@ -2,21 +2,48 @@ import { traceWriter } from './TraceWriter.js';
2
2
 
3
3
  export interface Span extends Disposable {
4
4
  name: string;
5
+ /**
6
+ * Start time in microseconds since an arbitrary epoch.
7
+ */
5
8
  startAt: number;
9
+ /**
10
+ * End time in microseconds since an arbitrary epoch.
11
+ *
12
+ * 0 if the span hasn't ended yet.
13
+ */
6
14
  endAt: number;
15
+
16
+ /**
17
+ * Time spent not in nested spans, in microseconds.
18
+ */
7
19
  selfDuration: number;
20
+
8
21
  nestedSince: number | undefined;
9
22
  subtrackFromSelf: number;
23
+
24
+ /**
25
+ * Durations spent in nested spans, in microseconds.
26
+ */
10
27
  nestedDurations: Record<string, number>;
11
28
 
29
+ /**
30
+ * Total duration of this span in milliseconds, rounded up. Only valid after the span has ended.
31
+ */
32
+ durationMillis: number;
33
+
12
34
  /**
13
35
  * End the span - same as [Symbol.dispose]().
14
36
  *
15
37
  * Safe to call multiple times. Any nested spans will automatically end as well.
16
38
  *
17
- * Returns an aggregate record of category -> "selfDuration".
39
+ * Returns an aggregate record of category -> "selfDuration", in microseconds.
18
40
  */
19
41
  end(): Record<string, number>;
42
+
43
+ /**
44
+ * Call a nested function, then end the span, returning the function's result.
45
+ */
46
+ with<T>(cb: () => Promise<T>): Promise<T>;
20
47
  }
21
48
 
22
49
  function now() {
@@ -115,8 +142,21 @@ export class PerformanceTracer<K extends string> {
115
142
  }
116
143
  return this.nestedDurations;
117
144
  },
145
+
146
+ get durationMillis() {
147
+ return Math.ceil((this.endAt - this.startAt) / 1000);
148
+ },
149
+
118
150
  [Symbol.dispose]() {
119
151
  this.end();
152
+ },
153
+
154
+ async with<T>(cb: () => Promise<T>): Promise<T> {
155
+ try {
156
+ return await cb();
157
+ } finally {
158
+ this.end();
159
+ }
120
160
  }
121
161
  };
122
162
  this.stack.push(s);
@@ -1,33 +1,15 @@
1
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
- import { RouteAPI } from '../api/RouteAPI.js';
3
- import { BucketStorageFactory } from '../storage/storage-index.js';
1
+ import { WriteCheckpointBatcher } from './write-checkpoint-batcher.js';
4
2
 
5
3
  export interface CreateWriteCheckpointOptions {
6
4
  userId: string | undefined;
7
5
  clientId: string | undefined;
8
- api: RouteAPI;
9
- storage: BucketStorageFactory;
6
+ checkpointRequestId?: bigint;
7
+ batcher: WriteCheckpointBatcher;
10
8
  }
11
9
  export async function createWriteCheckpoint(options: CreateWriteCheckpointOptions) {
12
10
  const full_user_id = checkpointUserId(options.userId, options.clientId);
13
11
 
14
- const syncBucketStorage = await options.storage.getActiveStorage();
15
- if (!syncBucketStorage) {
16
- throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
17
- }
18
-
19
- const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
20
- const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
21
- user_id: full_user_id,
22
- heads: { '1': currentCheckpoint }
23
- });
24
- return { writeCheckpoint, currentCheckpoint };
25
- });
26
-
27
- return {
28
- writeCheckpoint: String(writeCheckpoint),
29
- replicationHead: currentCheckpoint
30
- };
12
+ return options.batcher.enqueue(full_user_id, options.checkpointRequestId);
31
13
  }
32
14
 
33
15
  export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {