@powersync/service-core 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/CHANGELOG.md +102 -5
  2. package/dist/api/RouteAPI.d.ts +17 -3
  3. package/dist/api/api-index.d.ts +1 -1
  4. package/dist/api/api-index.js +1 -1
  5. package/dist/api/api-index.js.map +1 -1
  6. package/dist/api/api-metrics.js.map +1 -1
  7. package/dist/api/diagnostics.d.ts +1 -1
  8. package/dist/api/diagnostics.js +32 -14
  9. package/dist/api/diagnostics.js.map +1 -1
  10. package/dist/auth/CachedKeyCollector.js +1 -1
  11. package/dist/auth/CachedKeyCollector.js.map +1 -1
  12. package/dist/auth/CompoundKeyCollector.js.map +1 -1
  13. package/dist/auth/KeyStore.js.map +1 -1
  14. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  15. package/dist/auth/StaticKeyCollector.d.ts +1 -1
  16. package/dist/auth/StaticKeyCollector.js.map +1 -1
  17. package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
  18. package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
  19. package/dist/auth/utils.d.ts +2 -1
  20. package/dist/auth/utils.js +1 -1
  21. package/dist/auth/utils.js.map +1 -1
  22. package/dist/entry/commands/compact-action.js +26 -5
  23. package/dist/entry/commands/compact-action.js.map +1 -1
  24. package/dist/entry/commands/teardown-action.js +2 -2
  25. package/dist/entry/commands/teardown-action.js.map +1 -1
  26. package/dist/entry/entry-index.d.ts +1 -1
  27. package/dist/entry/entry-index.js +1 -1
  28. package/dist/entry/entry-index.js.map +1 -1
  29. package/dist/events/EventsEngine.js +1 -1
  30. package/dist/events/EventsEngine.js.map +1 -1
  31. package/dist/index.d.ts +1 -0
  32. package/dist/index.js +1 -0
  33. package/dist/index.js.map +1 -1
  34. package/dist/metrics/MetricsEngine.d.ts +1 -1
  35. package/dist/metrics/RollingBucketMax.d.ts +28 -0
  36. package/dist/metrics/RollingBucketMax.js +80 -0
  37. package/dist/metrics/RollingBucketMax.js.map +1 -0
  38. package/dist/metrics/metrics-index.d.ts +3 -2
  39. package/dist/metrics/metrics-index.js +3 -2
  40. package/dist/metrics/metrics-index.js.map +1 -1
  41. package/dist/metrics/open-telemetry/util.js +1 -1
  42. package/dist/metrics/open-telemetry/util.js.map +1 -1
  43. package/dist/metrics/register-metrics.js +2 -2
  44. package/dist/metrics/register-metrics.js.map +1 -1
  45. package/dist/modules/AbstractModule.d.ts +2 -2
  46. package/dist/modules/AbstractModule.js.map +1 -1
  47. package/dist/modules/modules-index.d.ts +1 -1
  48. package/dist/modules/modules-index.js +1 -1
  49. package/dist/modules/modules-index.js.map +1 -1
  50. package/dist/replication/AbstractReplicationJob.d.ts +2 -2
  51. package/dist/replication/AbstractReplicationJob.js +1 -1
  52. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  53. package/dist/replication/AbstractReplicator.d.ts +7 -7
  54. package/dist/replication/AbstractReplicator.js +31 -28
  55. package/dist/replication/AbstractReplicator.js.map +1 -1
  56. package/dist/replication/ReplicationLagTracker.d.ts +50 -0
  57. package/dist/replication/ReplicationLagTracker.js +78 -0
  58. package/dist/replication/ReplicationLagTracker.js.map +1 -0
  59. package/dist/replication/replication-index.d.ts +3 -2
  60. package/dist/replication/replication-index.js +3 -2
  61. package/dist/replication/replication-index.js.map +1 -1
  62. package/dist/replication/replication-metrics.js.map +1 -1
  63. package/dist/routes/configure-fastify.d.ts +59 -32
  64. package/dist/routes/endpoints/admin.d.ts +108 -54
  65. package/dist/routes/endpoints/admin.js +8 -3
  66. package/dist/routes/endpoints/admin.js.map +1 -1
  67. package/dist/routes/endpoints/checkpointing.js +1 -1
  68. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  69. package/dist/routes/endpoints/socket-route.js +1 -1
  70. package/dist/routes/endpoints/socket-route.js.map +1 -1
  71. package/dist/routes/endpoints/sync-rules.js +10 -10
  72. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  73. package/dist/routes/endpoints/sync-stream.d.ts +10 -10
  74. package/dist/routes/endpoints/sync-stream.js +8 -3
  75. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  76. package/dist/routes/hooks.js +1 -1
  77. package/dist/routes/hooks.js.map +1 -1
  78. package/dist/routes/route-register.js.map +1 -1
  79. package/dist/runner/teardown.js +4 -4
  80. package/dist/runner/teardown.js.map +1 -1
  81. package/dist/storage/BucketStorage.d.ts +9 -9
  82. package/dist/storage/BucketStorage.js +9 -9
  83. package/dist/storage/BucketStorageBatch.d.ts +30 -9
  84. package/dist/storage/BucketStorageBatch.js.map +1 -1
  85. package/dist/storage/BucketStorageFactory.d.ts +32 -20
  86. package/dist/storage/BucketStorageFactory.js +19 -16
  87. package/dist/storage/BucketStorageFactory.js.map +1 -1
  88. package/dist/storage/ChecksumCache.d.ts +5 -2
  89. package/dist/storage/ChecksumCache.js +8 -4
  90. package/dist/storage/ChecksumCache.js.map +1 -1
  91. package/dist/storage/PersistedSyncRulesContent.d.ts +9 -3
  92. package/dist/storage/PersistedSyncRulesContent.js +25 -5
  93. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  94. package/dist/storage/ReplicationEventPayload.d.ts +1 -1
  95. package/dist/storage/SourceTable.d.ts +10 -5
  96. package/dist/storage/SourceTable.js +3 -3
  97. package/dist/storage/SourceTable.js.map +1 -1
  98. package/dist/storage/StorageVersionConfig.d.ts +34 -1
  99. package/dist/storage/StorageVersionConfig.js +40 -7
  100. package/dist/storage/StorageVersionConfig.js.map +1 -1
  101. package/dist/storage/SyncRulesBucketStorage.d.ts +63 -11
  102. package/dist/storage/SyncRulesBucketStorage.js +14 -0
  103. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  104. package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
  105. package/dist/storage/WriteCheckpointAPI.js +1 -1
  106. package/dist/storage/bson.d.ts +0 -1
  107. package/dist/storage/bson.js +0 -4
  108. package/dist/storage/bson.js.map +1 -1
  109. package/dist/storage/storage-index.d.ts +8 -8
  110. package/dist/storage/storage-index.js +8 -8
  111. package/dist/storage/storage-index.js.map +1 -1
  112. package/dist/storage/storage-metrics.js.map +1 -1
  113. package/dist/streams/streams-index.d.ts +2 -2
  114. package/dist/streams/streams-index.js +2 -2
  115. package/dist/streams/streams-index.js.map +1 -1
  116. package/dist/sync/BucketChecksumState.d.ts +5 -8
  117. package/dist/sync/BucketChecksumState.js +118 -116
  118. package/dist/sync/BucketChecksumState.js.map +1 -1
  119. package/dist/sync/RequestTracker.js +1 -1
  120. package/dist/sync/RequestTracker.js.map +1 -1
  121. package/dist/sync/sync-index.d.ts +2 -2
  122. package/dist/sync/sync-index.js +2 -2
  123. package/dist/sync/sync-index.js.map +1 -1
  124. package/dist/sync/sync.js.map +1 -1
  125. package/dist/sync/util.d.ts +1 -0
  126. package/dist/sync/util.js +10 -0
  127. package/dist/sync/util.js.map +1 -1
  128. package/dist/system/ServiceContext.d.ts +1 -1
  129. package/dist/system/ServiceContext.js +1 -1
  130. package/dist/system/ServiceContext.js.map +1 -1
  131. package/dist/tracing/PerformanceTracer.d.ts +44 -0
  132. package/dist/tracing/PerformanceTracer.js +102 -0
  133. package/dist/tracing/PerformanceTracer.js.map +1 -0
  134. package/dist/tracing/TraceWriter.d.ts +22 -0
  135. package/dist/tracing/TraceWriter.js +63 -0
  136. package/dist/tracing/TraceWriter.js.map +1 -0
  137. package/dist/util/checkpointing.js +1 -1
  138. package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
  139. package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
  140. package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
  141. package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
  142. package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
  143. package/dist/util/config/compound-config-collector.d.ts +1 -1
  144. package/dist/util/config/compound-config-collector.js +2 -2
  145. package/dist/util/config/compound-config-collector.js.map +1 -1
  146. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
  147. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  148. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  149. package/dist/util/config.js +1 -1
  150. package/dist/util/config.js.map +1 -1
  151. package/dist/util/env.js +1 -1
  152. package/dist/util/errors.d.ts +3 -0
  153. package/dist/util/errors.js +15 -0
  154. package/dist/util/errors.js.map +1 -0
  155. package/dist/util/protocol-types.d.ts +3 -3
  156. package/dist/util/protocol-types.js +1 -1
  157. package/dist/util/util-index.d.ts +1 -1
  158. package/dist/util/util-index.js +1 -1
  159. package/dist/util/util-index.js.map +1 -1
  160. package/dist/util/utils.d.ts +1 -1
  161. package/package.json +11 -11
  162. package/src/api/RouteAPI.ts +20 -3
  163. package/src/api/api-index.ts +1 -1
  164. package/src/api/api-metrics.ts +1 -1
  165. package/src/api/diagnostics.ts +42 -20
  166. package/src/auth/CachedKeyCollector.ts +2 -3
  167. package/src/auth/CompoundKeyCollector.ts +2 -3
  168. package/src/auth/KeyStore.ts +1 -1
  169. package/src/auth/RemoteJWKSCollector.ts +0 -1
  170. package/src/auth/StaticKeyCollector.ts +1 -1
  171. package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
  172. package/src/auth/utils.ts +5 -2
  173. package/src/entry/commands/compact-action.ts +29 -5
  174. package/src/entry/commands/teardown-action.ts +2 -2
  175. package/src/entry/entry-index.ts +1 -1
  176. package/src/events/EventsEngine.ts +1 -1
  177. package/src/index.ts +2 -0
  178. package/src/metrics/MetricsEngine.ts +1 -1
  179. package/src/metrics/RollingBucketMax.ts +109 -0
  180. package/src/metrics/metrics-index.ts +3 -2
  181. package/src/metrics/open-telemetry/util.ts +1 -1
  182. package/src/metrics/register-metrics.ts +3 -3
  183. package/src/modules/AbstractModule.ts +2 -2
  184. package/src/modules/modules-index.ts +1 -1
  185. package/src/replication/AbstractReplicationJob.ts +3 -3
  186. package/src/replication/AbstractReplicator.ts +32 -30
  187. package/src/replication/ReplicationLagTracker.ts +86 -0
  188. package/src/replication/replication-index.ts +3 -2
  189. package/src/replication/replication-metrics.ts +1 -1
  190. package/src/routes/endpoints/admin.ts +8 -3
  191. package/src/routes/endpoints/checkpointing.ts +1 -1
  192. package/src/routes/endpoints/socket-route.ts +1 -1
  193. package/src/routes/endpoints/sync-rules.ts +10 -12
  194. package/src/routes/endpoints/sync-stream.ts +8 -3
  195. package/src/routes/hooks.ts +2 -2
  196. package/src/routes/route-register.ts +2 -10
  197. package/src/runner/teardown.ts +4 -4
  198. package/src/storage/BucketStorage.ts +9 -9
  199. package/src/storage/BucketStorageBatch.ts +34 -10
  200. package/src/storage/BucketStorageFactory.ts +51 -34
  201. package/src/storage/ChecksumCache.ts +14 -6
  202. package/src/storage/PersistedSyncRulesContent.ts +36 -7
  203. package/src/storage/ReplicationEventPayload.ts +1 -1
  204. package/src/storage/SourceTable.ts +10 -4
  205. package/src/storage/StorageVersionConfig.ts +55 -7
  206. package/src/storage/SyncRulesBucketStorage.ts +77 -11
  207. package/src/storage/WriteCheckpointAPI.ts +6 -6
  208. package/src/storage/bson.ts +0 -5
  209. package/src/storage/storage-index.ts +8 -8
  210. package/src/storage/storage-metrics.ts +2 -2
  211. package/src/streams/streams-index.ts +2 -2
  212. package/src/sync/BucketChecksumState.ts +142 -139
  213. package/src/sync/RequestTracker.ts +1 -1
  214. package/src/sync/sync-index.ts +2 -2
  215. package/src/sync/sync.ts +4 -4
  216. package/src/sync/util.ts +11 -1
  217. package/src/system/ServiceContext.ts +1 -1
  218. package/src/tracing/PerformanceTracer.ts +126 -0
  219. package/src/tracing/TraceWriter.ts +67 -0
  220. package/src/util/checkpointing.ts +1 -1
  221. package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
  222. package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
  223. package/src/util/config/compound-config-collector.ts +3 -3
  224. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
  225. package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
  226. package/src/util/config.ts +1 -1
  227. package/src/util/env.ts +1 -1
  228. package/src/util/errors.ts +21 -0
  229. package/src/util/protocol-types.ts +1 -1
  230. package/src/util/util-index.ts +1 -1
  231. package/src/util/utils.ts +1 -1
  232. package/test/src/ReplicationLagTracker.test.ts +53 -0
  233. package/test/src/RollingBucketMax.test.ts +106 -0
  234. package/test/src/auth.test.ts +26 -7
  235. package/test/src/checksum_cache.test.ts +102 -57
  236. package/test/src/config.test.ts +1 -0
  237. package/test/src/diagnostics.test.ts +151 -0
  238. package/test/src/module-loader.test.ts +1 -1
  239. package/test/src/routes/mocks.ts +1 -1
  240. package/test/src/routes/stream.test.ts +1 -2
  241. package/test/src/sync/BucketChecksumState.test.ts +209 -80
  242. package/test/src/util/protocol_types.test.ts +1 -1
  243. package/test/src/utils.ts +9 -0
  244. package/test/tsconfig.json +0 -1
  245. package/tsconfig.tsbuildinfo +1 -1
  246. package/vitest.config.ts +1 -1
@@ -1,9 +1,9 @@
1
1
  import {
2
- BucketDescription,
3
2
  BucketParameterQuerier,
4
3
  BucketPriority,
5
4
  BucketSource,
6
5
  HydratedSyncRules,
6
+ mergeBuckets,
7
7
  QuerierError,
8
8
  RequestedStream,
9
9
  RequestParameters,
@@ -21,6 +21,7 @@ import {
21
21
  ServiceError
22
22
  } from '@powersync/lib-services-framework';
23
23
  import { JwtPayload } from '../auth/JwtPayload.js';
24
+ import { ParameterSetLimitExceededError } from '../storage/storage-index.js';
24
25
  import { SyncContext } from './SyncContext.js';
25
26
  import { getIntersection, hasIntersection } from './util.js';
26
27
 
@@ -118,16 +119,38 @@ export class BucketChecksumState {
118
119
  const storage = this.bucketStorage;
119
120
 
120
121
  const update = await this.parameterState.getCheckpointUpdate(next);
121
- const { buckets: allBuckets, updatedBuckets, parameterQueryResultsByDefinition } = update;
122
+ const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
122
123
 
123
124
  /** Set of all buckets in this checkpoint. */
124
125
  const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
125
126
 
126
127
  if (bucketDescriptionMap.size > this.context.maxBuckets) {
127
- throw new ServiceError(
128
+ const error = new ServiceError(
128
129
  ErrorCode.PSYNC_S2305,
129
130
  `Too many buckets: ${bucketDescriptionMap.size} (limit of ${this.context.maxBuckets})`
130
131
  );
132
+
133
+ let errorMessage = error.message;
134
+ const logData: any = {
135
+ checkpoint: next.base.checkpoint,
136
+ user_id: this.parameterState.syncParams.userId,
137
+ buckets: allBuckets.length
138
+ };
139
+
140
+ // Count buckets per definition.
141
+ const bucketsByDefinition = new Map<string, number>();
142
+ for (const bucket of bucketDescriptionMap.values()) {
143
+ const definition = bucket.definition;
144
+ const count = bucketsByDefinition.get(definition) ?? 0;
145
+ bucketsByDefinition.set(definition, count + 1);
146
+ }
147
+
148
+ const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition);
149
+ errorMessage += breakdown.message;
150
+ logData.buckets_by_definition = breakdown.countsByDefinition;
151
+
152
+ this.logger.error(errorMessage, logData);
153
+ throw error;
131
154
  }
132
155
 
133
156
  let checksumMap: util.ChecksumMap;
@@ -137,20 +160,20 @@ export class BucketChecksumState {
137
160
  }
138
161
 
139
162
  // Re-check updated buckets only
140
- let checksumLookups: string[] = [];
163
+ let checksumLookups: storage.BucketChecksumRequest[] = [];
141
164
 
142
165
  let newChecksums = new Map<string, util.BucketChecksum>();
143
- for (let bucket of bucketDescriptionMap.keys()) {
144
- if (!updatedBuckets.has(bucket)) {
145
- const existing = this.lastChecksums.get(bucket);
166
+ for (let desc of bucketDescriptionMap.values()) {
167
+ if (!updatedBuckets.has(desc.bucket)) {
168
+ const existing = this.lastChecksums.get(desc.bucket);
146
169
  if (existing == null) {
147
170
  // If this happens, it means updatedBuckets did not correctly include all new buckets
148
- throw new ServiceAssertionError(`Existing checksum not found for bucket ${bucket}`);
171
+ throw new ServiceAssertionError(`Existing checksum not found for bucket ${desc.bucket}`);
149
172
  }
150
173
  // Bucket is not specifically updated, and we have a previous checksum
151
- newChecksums.set(bucket, existing);
174
+ newChecksums.set(desc.bucket, existing);
152
175
  } else {
153
- checksumLookups.push(bucket);
176
+ checksumLookups.push({ bucket: desc.bucket, source: desc.source });
154
177
  }
155
178
  }
156
179
 
@@ -163,12 +186,12 @@ export class BucketChecksumState {
163
186
  checksumMap = newChecksums;
164
187
  } else {
165
188
  // Re-check all buckets
166
- const bucketList = [...bucketDescriptionMap.keys()];
189
+ const bucketList = [...bucketDescriptionMap.values()].map((b) => ({ bucket: b.bucket, source: b.source }));
167
190
  checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
168
191
  }
169
192
 
170
193
  // Subset of buckets for which there may be new data in this batch.
171
- let bucketsToFetch: BucketDescription[];
194
+ let bucketsToFetch: ResolvedBucket[];
172
195
 
173
196
  let checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
174
197
 
@@ -207,20 +230,14 @@ export class BucketChecksumState {
207
230
  ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
208
231
  }));
209
232
  bucketsToFetch = [...generateBucketsToFetch].map((b) => {
210
- return {
211
- priority: bucketDescriptionMap.get(b)!.priority,
212
- bucket: b
213
- };
233
+ return bucketDescriptionMap.get(b)!;
214
234
  });
215
235
 
216
236
  deferredLog = () => {
217
- const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
218
237
  let message = `Updated checkpoint: ${base.checkpoint} | `;
219
238
  message += `write: ${writeCheckpoint} | `;
220
239
  message += `buckets: ${allBuckets.length} | `;
221
- if (totalParamResults !== undefined) {
222
- message += `param_results: ${totalParamResults} | `;
223
- }
240
+ message += `param_results: ${usedParameterResults} | `;
224
241
  message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
225
242
  message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
226
243
  logCheckpoint(
@@ -233,7 +250,7 @@ export class BucketChecksumState {
233
250
  updated: diff.updatedBuckets.length,
234
251
  removed: diff.removedBuckets.length
235
252
  },
236
- totalParamResults
253
+ usedParameterResults
237
254
  );
238
255
  };
239
256
 
@@ -247,12 +264,9 @@ export class BucketChecksumState {
247
264
  } satisfies util.StreamingSyncCheckpointDiff;
248
265
  } else {
249
266
  deferredLog = () => {
250
- const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
251
267
  let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
252
268
  message += `buckets: ${allBuckets.length}`;
253
- if (totalParamResults !== undefined) {
254
- message += ` | param_results: ${totalParamResults}`;
255
- }
269
+ message += ` | param_results: ${usedParameterResults}`;
256
270
  message += ` ${limitedBuckets(allBuckets, 20)}`;
257
271
  logCheckpoint(
258
272
  this.logger,
@@ -262,10 +276,10 @@ export class BucketChecksumState {
262
276
  user_id: userIdForLogs,
263
277
  buckets: allBuckets.length
264
278
  },
265
- totalParamResults
279
+ usedParameterResults
266
280
  );
267
281
  };
268
- bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
282
+ bucketsToFetch = allBuckets;
269
283
 
270
284
  const subscriptions: util.StreamDescription[] = [];
271
285
  const streamNameToIndex = new Map<string, number>();
@@ -342,17 +356,17 @@ export class BucketChecksumState {
342
356
  deferredLog();
343
357
  },
344
358
 
345
- getFilteredBucketPositions: (buckets?: BucketDescription[]): Map<string, util.InternalOpId> => {
359
+ getFilteredBucketPositions: (buckets?: ResolvedBucket[]): storage.BucketDataRequest[] => {
346
360
  if (!hasAdvanced) {
347
361
  throw new ServiceAssertionError('Call line.advance() before getFilteredBucketPositions()');
348
362
  }
349
363
  buckets ??= bucketsToFetch;
350
- const filtered = new Map<string, util.InternalOpId>();
364
+ const filtered: storage.BucketDataRequest[] = [];
351
365
 
352
366
  for (let bucket of buckets) {
353
367
  const state = this.bucketDataPositions.get(bucket.bucket);
354
368
  if (state) {
355
- filtered.set(bucket.bucket, state.start_op_id);
369
+ filtered.push({ bucket: bucket.bucket, start: state.start_op_id, source: bucket.source });
356
370
  }
357
371
  }
358
372
  return filtered;
@@ -395,11 +409,8 @@ export interface CheckpointUpdate {
395
409
  */
396
410
  updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
397
411
 
398
- /**
399
- * Number of parameter query results per sync stream definition (before deduplication).
400
- * Map from definition name to count.
401
- */
402
- parameterQueryResultsByDefinition?: Map<string, number>;
412
+ /** The amount of rows fetched from parameters indexes. */
413
+ usedParameterResults: number;
403
414
  }
404
415
 
405
416
  export class BucketParameterState {
@@ -447,6 +458,7 @@ export class BucketParameterState {
447
458
  const subscription = explicitStreamSubscriptions[i];
448
459
 
449
460
  const syncRuleStream: RequestedStream = {
461
+ priorityOverride: subscription.override_priority as BucketPriority | null,
450
462
  parameters: subscription.parameters ?? {},
451
463
  opaque_id: i
452
464
  };
@@ -482,25 +494,9 @@ export class BucketParameterState {
482
494
  * reference default buckets by their stream index instead of duplicating the name on wire.
483
495
  */
484
496
  translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map<string, number>): util.ClientBucketDescription {
485
- // If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that
486
- // priority.
487
- let priorityOverride: BucketPriority | null = null;
488
- for (const reason of description.inclusion_reasons) {
489
- if (reason != 'default') {
490
- const requestedPriority = this.explicitStreamSubscriptions[reason.subscription]?.override_priority;
491
- if (requestedPriority != null) {
492
- if (priorityOverride == null) {
493
- priorityOverride = requestedPriority as BucketPriority;
494
- } else {
495
- priorityOverride = Math.min(requestedPriority, priorityOverride) as BucketPriority;
496
- }
497
- }
498
- }
499
- }
500
-
501
497
  return {
502
498
  bucket: description.bucket,
503
- priority: priorityOverride ?? description.priority,
499
+ priority: description.priority,
504
500
  subscriptions: description.inclusion_reasons.map((reason) => {
505
501
  if (reason == 'default') {
506
502
  const stream = description.definition;
@@ -520,36 +516,41 @@ export class BucketParameterState {
520
516
  const querier = this.querier;
521
517
  let update: CheckpointUpdate;
522
518
  if (querier.hasDynamicBuckets) {
523
- update = await this.getCheckpointUpdateDynamic(checkpoint);
524
- } else {
525
- update = await this.getCheckpointUpdateStatic(checkpoint);
526
- }
527
-
528
- if (update.buckets.length > this.context.maxParameterQueryResults) {
529
- // TODO: Limit number of results even before we get to this point
530
- // This limit applies _before_ we get the unique set
531
- const error = new ServiceError(
532
- ErrorCode.PSYNC_S2305,
533
- `Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`
534
- );
519
+ try {
520
+ update = await this.getCheckpointUpdateDynamic(checkpoint);
521
+ } catch (e: unknown) {
522
+ if (e instanceof ParameterSetLimitExceededError) {
523
+ // Too many parameter results, create a breakdown of which streams are responsible for the most queries and
524
+ // then abort.
525
+ const error = new ServiceError(
526
+ ErrorCode.PSYNC_S2305,
527
+ `Too many parameter query results (limit of ${this.context.maxParameterQueryResults})`
528
+ );
529
+
530
+ let errorMessage = error.message;
531
+ const logData: any = {
532
+ checkpoint: checkpoint.base.checkpoint,
533
+ user_id: this.syncParams.userId
534
+ };
535
+
536
+ if (e.breakdown) {
537
+ const breakdown = formatParameterQueryBreakdown(e.breakdown);
538
+ if (breakdown) {
539
+ errorMessage += breakdown.message;
540
+ logData.parameterResults = breakdown.largestResults;
541
+ }
542
+ }
535
543
 
536
- let errorMessage = error.message;
537
- const logData: any = {
538
- checkpoint: checkpoint,
539
- user_id: this.syncParams.userId,
540
- parameter_query_results: update.buckets.length
541
- };
544
+ this.logger.error(errorMessage, logData);
545
+ throw error;
546
+ }
542
547
 
543
- if (update.parameterQueryResultsByDefinition && update.parameterQueryResultsByDefinition.size > 0) {
544
- const breakdown = formatParameterQueryBreakdown(update.parameterQueryResultsByDefinition);
545
- errorMessage += breakdown.message;
546
- logData.parameter_query_results_by_definition = breakdown.countsByDefinition;
548
+ throw e;
547
549
  }
548
-
549
- this.logger.error(errorMessage, logData);
550
-
551
- throw error;
550
+ } else {
551
+ update = await this.getCheckpointUpdateStatic(checkpoint);
552
552
  }
553
+
553
554
  return update;
554
555
  }
555
556
 
@@ -563,14 +564,16 @@ export class BucketParameterState {
563
564
  if (update.invalidateDataBuckets) {
564
565
  return {
565
566
  buckets: staticBuckets,
566
- updatedBuckets: INVALIDATE_ALL_BUCKETS
567
+ updatedBuckets: INVALIDATE_ALL_BUCKETS,
568
+ usedParameterResults: 0
567
569
  };
568
570
  }
569
571
 
570
572
  const updatedBuckets = new Set<string>(getIntersection(this.staticBuckets, update.updatedDataBuckets));
571
573
  return {
572
574
  buckets: staticBuckets,
573
- updatedBuckets
575
+ updatedBuckets,
576
+ usedParameterResults: 0
574
577
  };
575
578
  }
576
579
 
@@ -602,27 +605,41 @@ export class BucketParameterState {
602
605
  }
603
606
 
604
607
  let dynamicBuckets: ResolvedBucket[];
605
- let parameterQueryResultsByDefinition: Map<string, number> | undefined;
608
+ let usedParameterResults = 0;
606
609
  if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
607
610
  const recordedLookups = new Set<string>();
611
+ const parameterLimit = this.context.maxParameterQueryResults;
612
+ let remainingBudget = parameterLimit;
613
+
614
+ // Log of lookups to provide a breakdown if we exceed the dynamic lookup limit.
615
+ // FIXME: This is horrible, Sync Streams will invoke the callback concurrently and we can't properly deal with
616
+ // that. We should replace queriers for Sync Streams with an explicit graph structure based on sync plans instead
617
+ // of adding these checks to the imperative querier interface.
618
+ const lookupLog: storage.ParameterQueryInvocationLog[] = [];
608
619
 
609
620
  dynamicBuckets = await querier.queryDynamicBucketDescriptions({
610
- getParameterSets(lookups) {
621
+ async getParameterSets(lookups, definition) {
611
622
  for (const lookup of lookups) {
612
623
  recordedLookups.add(lookup.serializedRepresentation);
613
624
  }
614
625
 
615
- return checkpoint.base.getParameterSets(lookups);
626
+ try {
627
+ const rows = await checkpoint.base.getParameterSets(lookups, remainingBudget);
628
+ lookupLog.push({ definition, resultsOrLimit: rows.length, didExceedLimit: false });
629
+ remainingBudget -= rows.length;
630
+ usedParameterResults += rows.length;
631
+ return rows;
632
+ } catch (e: unknown) {
633
+ if (e instanceof ParameterSetLimitExceededError) {
634
+ lookupLog.push({ definition, resultsOrLimit: remainingBudget, didExceedLimit: true });
635
+ throw new ParameterSetLimitExceededError(parameterLimit, lookupLog);
636
+ }
637
+
638
+ throw e;
639
+ }
616
640
  }
617
641
  });
618
642
 
619
- // Count parameter query results per definition (before deduplication)
620
- parameterQueryResultsByDefinition = new Map<string, number>();
621
- for (const bucket of dynamicBuckets) {
622
- const count = parameterQueryResultsByDefinition.get(bucket.definition) ?? 0;
623
- parameterQueryResultsByDefinition.set(bucket.definition, count + 1);
624
- }
625
-
626
643
  this.cachedDynamicBuckets = dynamicBuckets;
627
644
  this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
628
645
  this.lookupsFromPreviousCheckpoint = recordedLookups;
@@ -646,13 +663,13 @@ export class BucketParameterState {
646
663
  buckets: allBuckets,
647
664
  // We cannot track individual bucket updates for dynamic lookups yet
648
665
  updatedBuckets: INVALIDATE_ALL_BUCKETS,
649
- parameterQueryResultsByDefinition
666
+ usedParameterResults
650
667
  };
651
668
  } else {
652
669
  return {
653
670
  buckets: allBuckets,
654
671
  updatedBuckets: updatedBuckets,
655
- parameterQueryResultsByDefinition
672
+ usedParameterResults
656
673
  };
657
674
  }
658
675
  }
@@ -660,7 +677,7 @@ export class BucketParameterState {
660
677
 
661
678
  export interface CheckpointLine {
662
679
  checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
663
- bucketsToFetch: BucketDescription[];
680
+ bucketsToFetch: ResolvedBucket[];
664
681
 
665
682
  /**
666
683
  * Call when a checkpoint line is being sent to a client, to update the internal state.
@@ -672,7 +689,7 @@ export interface CheckpointLine {
672
689
  *
673
690
  * @param bucketsToFetch List of buckets to fetch - either this.bucketsToFetch, or a subset of it. Defaults to this.bucketsToFetch.
674
691
  */
675
- getFilteredBucketPositions(bucketsToFetch?: BucketDescription[]): Map<string, util.InternalOpId>;
692
+ getFilteredBucketPositions(bucketsToFetch?: ResolvedBucket[]): storage.BucketDataRequest[];
676
693
 
677
694
  /**
678
695
  * Update the position of bucket data the client has, after it was sent to the client.
@@ -686,18 +703,6 @@ export interface CheckpointLine {
686
703
  // Use a more specific type to simplify testing
687
704
  export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
688
705
 
689
- /**
690
- * Compute the total number of parameter query results across all definitions.
691
- */
692
- function computeTotalParamResults(
693
- parameterQueryResultsByDefinition: Map<string, number> | undefined
694
- ): number | undefined {
695
- if (!parameterQueryResultsByDefinition) {
696
- return undefined;
697
- }
698
- return Array.from(parameterQueryResultsByDefinition.values()).reduce((sum, count) => sum + count, 0);
699
- }
700
-
701
706
  /**
702
707
  * Log a checkpoint message, enriching it with parameter query result counts if available.
703
708
  *
@@ -719,20 +724,20 @@ function logCheckpoint(
719
724
  }
720
725
 
721
726
  /**
722
- * Format a breakdown of parameter query results by sync rule definition.
727
+ * Format a breakdown of dynamic bucket by sync stream definition.
723
728
  *
724
729
  * Sorts definitions by count (descending), includes the top 10, and returns both the
725
730
  * formatted message string and the counts record suitable for structured log data.
726
731
  */
727
- function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<string, number>): {
732
+ function formatBucketDefinitionBreakdown(bucketsByDefinition: Map<string, number>): {
728
733
  message: string;
729
734
  countsByDefinition: Record<string, number>;
730
735
  } {
731
736
  // Sort definitions by count (descending) and take top 10
732
- const allSorted = Array.from(parameterQueryResultsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
737
+ const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
733
738
  const sortedDefinitions = allSorted.slice(0, 10);
734
739
 
735
- let message = '\nParameter query results by definition:';
740
+ let message = '\Buckets by definition:';
736
741
  const countsByDefinition: Record<string, number> = {};
737
742
  for (const [definition, count] of sortedDefinitions) {
738
743
  message += `\n ${definition}: ${count}`;
@@ -748,6 +753,33 @@ function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<st
748
753
  return { message, countsByDefinition };
749
754
  }
750
755
 
756
+ function formatParameterQueryBreakdown(log: storage.ParameterQueryInvocationLog[]) {
757
+ if (log.length == 0) {
758
+ return;
759
+ }
760
+
761
+ // When an exception about too many parameter results is thrown, the last entry is the one that finally exceeded the
762
+ // limit.
763
+ const results = Array.from(log);
764
+ const failure = results.pop()!;
765
+
766
+ let message = '\nInvoked parameter queries by definition:';
767
+ results.sort((a, b) => b.resultsOrLimit - a.resultsOrLimit);
768
+ const largestResults = results.splice(0, 9);
769
+
770
+ for (const entry of largestResults) {
771
+ message += `\n ${entry.definition}: ${entry.resultsOrLimit} results.`;
772
+ }
773
+
774
+ if (largestResults.length < log.length) {
775
+ message += `\n ... and ${log.length - largestResults.length} more invocations`;
776
+ }
777
+
778
+ message += `\n ${failure.definition} exceeded the remaining limit of ${failure.resultsOrLimit} available results.`;
779
+ largestResults.push(failure);
780
+ return { message, largestResults };
781
+ }
782
+
751
783
  function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
752
784
  buckets = buckets.map((b) => {
753
785
  if (typeof b != 'string') {
@@ -762,32 +794,3 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number)
762
794
  const limited = buckets.slice(0, limit);
763
795
  return `${JSON.stringify(limited)}...`;
764
796
  }
765
-
766
- /**
767
- * Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate.
768
- *
769
- * It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g.
770
- *
771
- * ```
772
- * sync_streams:
773
- * assets_by_category:
774
- * query: select * from assets where category in (request.parameters() -> 'categories')
775
- * ```
776
- *
777
- * Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each
778
- * subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]`
779
- * bucket.
780
- */
781
- function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] {
782
- const byBucketId: Record<string, ResolvedBucket> = {};
783
-
784
- for (const bucket of buckets) {
785
- if (Object.hasOwn(byBucketId, bucket.bucket)) {
786
- byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons);
787
- } else {
788
- byBucketId[bucket.bucket] = structuredClone(bucket);
789
- }
790
- }
791
-
792
- return Object.values(byBucketId);
793
- }
@@ -1,8 +1,8 @@
1
1
  import { MetricsEngine } from '../metrics/MetricsEngine.js';
2
2
 
3
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
3
4
  import { APIMetric } from '@powersync/service-types';
4
5
  import { SyncBucketData } from '../util/protocol-types.js';
5
- import { ServiceAssertionError } from '@powersync/lib-services-framework';
6
6
 
7
7
  /**
8
8
  * Record sync stats per request stream.
@@ -1,6 +1,6 @@
1
+ export * from './BucketChecksumState.js';
1
2
  export * from './RequestTracker.js';
2
3
  export * from './safeRace.js';
3
4
  export * from './sync.js';
4
- export * from './util.js';
5
- export * from './BucketChecksumState.js';
6
5
  export * from './SyncContext.js';
6
+ export * from './util.js';
package/src/sync/sync.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
2
- import { BucketDescription, BucketPriority, HydratedSyncRules, SqliteJsonValue } from '@powersync/service-sync-rules';
2
+ import { BucketPriority, HydratedSyncRules, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
3
3
 
4
4
  import { AbortError } from 'ix/aborterror.js';
5
5
 
@@ -179,7 +179,7 @@ async function* streamResponseInner(
179
179
  // receive a sync complete message after the synchronization is done (which happens in the last
180
180
  // bucketDataInBatches iteration). Without any batch, the line is missing and clients might not complete their
181
181
  // sync properly.
182
- const priorityBatches: [BucketPriority | null, BucketDescription[]][] = bucketsByPriority;
182
+ const priorityBatches: [BucketPriority | null, ResolvedBucket[]][] = bucketsByPriority;
183
183
  if (priorityBatches.length == 0) {
184
184
  priorityBatches.push([null, []]);
185
185
  }
@@ -257,7 +257,7 @@ interface BucketDataRequest {
257
257
  /** Contains current bucket state. Modified by the request as data is sent. */
258
258
  checkpointLine: CheckpointLine;
259
259
  /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
260
- bucketsToFetch: BucketDescription[];
260
+ bucketsToFetch: ResolvedBucket[];
261
261
  /** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
262
262
  legacyDataLines: boolean;
263
263
  /** Signals that the connection was aborted and that streaming should stop ASAP. */
@@ -267,7 +267,7 @@ interface BucketDataRequest {
267
267
  * This signal also fires when abort_connection fires.
268
268
  */
269
269
  abort_batch: AbortSignal;
270
- /** User id for debug purposes, not for sync rules. */
270
+ /** User id for debug purposes, not for sync config. */
271
271
  userIdForLogs?: SqliteJsonValue;
272
272
  forPriority: BucketPriority | null;
273
273
  onRowsSent: (stats: OperationsSentStats) => void;
package/src/sync/util.ts CHANGED
@@ -1,9 +1,9 @@
1
1
  import * as timers from 'timers/promises';
2
2
 
3
3
  import { SemaphoreInterface } from 'async-mutex';
4
+ import { serialize } from 'bson';
4
5
  import * as util from '../util/util-index.js';
5
6
  import { RequestTracker } from './RequestTracker.js';
6
- import { serialize } from 'bson';
7
7
 
8
8
  export type TokenStreamOptions = {
9
9
  /**
@@ -183,6 +183,16 @@ export function settledPromise<T>(promise: Promise<T>): Promise<PromiseSettledRe
183
183
  );
184
184
  }
185
185
 
186
+ export function unsettledPromise<T>(settled: Promise<PromiseSettledResult<T>>): Promise<T> {
187
+ return settled.then((result) => {
188
+ if (result.status === 'fulfilled') {
189
+ return Promise.resolve(result.value);
190
+ } else {
191
+ return Promise.reject(result.reason);
192
+ }
193
+ });
194
+ }
195
+
186
196
  export type MapOrSet<T> = Map<T, any> | Set<T>;
187
197
 
188
198
  /**
@@ -1,5 +1,6 @@
1
1
  import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
2
2
 
3
+ import { EventsEngine } from '../events/EventsEngine.js';
3
4
  import { framework } from '../index.js';
4
5
  import * as metrics from '../metrics/MetricsEngine.js';
5
6
  import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
@@ -8,7 +9,6 @@ import * as routes from '../routes/routes-index.js';
8
9
  import * as storage from '../storage/storage-index.js';
9
10
  import { SyncContext } from '../sync/SyncContext.js';
10
11
  import * as utils from '../util/util-index.js';
11
- import { EventsEngine } from '../events/EventsEngine.js';
12
12
 
13
13
  export interface ServiceContext {
14
14
  configuration: utils.ResolvedPowerSyncConfig;