@powersync/service-core 1.20.5 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/CHANGELOG.md +66 -0
  2. package/dist/api/RouteAPI.d.ts +3 -3
  3. package/dist/api/diagnostics.d.ts +1 -1
  4. package/dist/api/diagnostics.js +19 -3
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/auth/RemoteJWKSCollector.js +3 -2
  7. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  8. package/dist/entry/commands/teardown-action.js +1 -1
  9. package/dist/entry/commands/teardown-action.js.map +1 -1
  10. package/dist/index.d.ts +1 -0
  11. package/dist/index.js +1 -0
  12. package/dist/index.js.map +1 -1
  13. package/dist/modules/AbstractModule.d.ts +1 -1
  14. package/dist/replication/AbstractReplicationJob.js +1 -1
  15. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  16. package/dist/replication/AbstractReplicator.d.ts +6 -6
  17. package/dist/replication/AbstractReplicator.js +21 -21
  18. package/dist/replication/AbstractReplicator.js.map +1 -1
  19. package/dist/replication/RelationCache.d.ts +9 -2
  20. package/dist/replication/RelationCache.js +21 -2
  21. package/dist/replication/RelationCache.js.map +1 -1
  22. package/dist/routes/configure-fastify.js +3 -1
  23. package/dist/routes/configure-fastify.js.map +1 -1
  24. package/dist/routes/endpoints/admin.js +16 -8
  25. package/dist/routes/endpoints/admin.js.map +1 -1
  26. package/dist/routes/endpoints/checkpointing.js +1 -1
  27. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  28. package/dist/routes/endpoints/socket-route.js +1 -1
  29. package/dist/routes/endpoints/socket-route.js.map +1 -1
  30. package/dist/routes/endpoints/sync-rules.js +8 -8
  31. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  32. package/dist/routes/endpoints/sync-stream.js +2 -2
  33. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  34. package/dist/routes/route-register.d.ts +2 -0
  35. package/dist/routes/route-register.js +65 -3
  36. package/dist/routes/route-register.js.map +1 -1
  37. package/dist/runner/teardown.js +4 -4
  38. package/dist/runner/teardown.js.map +1 -1
  39. package/dist/storage/BucketStorage.d.ts +9 -9
  40. package/dist/storage/BucketStorage.js +9 -9
  41. package/dist/storage/BucketStorageBatch.d.ts +29 -0
  42. package/dist/storage/BucketStorageBatch.js.map +1 -1
  43. package/dist/storage/BucketStorageFactory.d.ts +27 -18
  44. package/dist/storage/BucketStorageFactory.js +13 -12
  45. package/dist/storage/BucketStorageFactory.js.map +1 -1
  46. package/dist/storage/PersistedSyncRulesContent.d.ts +6 -4
  47. package/dist/storage/PersistedSyncRulesContent.js +15 -8
  48. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  49. package/dist/storage/SourceEntity.d.ts +8 -1
  50. package/dist/storage/SourceTable.d.ts +32 -11
  51. package/dist/storage/SourceTable.js +41 -15
  52. package/dist/storage/SourceTable.js.map +1 -1
  53. package/dist/storage/StorageVersionConfig.d.ts +1 -1
  54. package/dist/storage/StorageVersionConfig.js +1 -1
  55. package/dist/storage/SyncRulesBucketStorage.d.ts +63 -18
  56. package/dist/storage/SyncRulesBucketStorage.js +14 -0
  57. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  58. package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
  59. package/dist/storage/WriteCheckpointAPI.js +1 -1
  60. package/dist/storage/bson.d.ts +0 -1
  61. package/dist/storage/bson.js +0 -4
  62. package/dist/storage/bson.js.map +1 -1
  63. package/dist/sync/BucketChecksumState.d.ts +6 -9
  64. package/dist/sync/BucketChecksumState.js +117 -58
  65. package/dist/sync/BucketChecksumState.js.map +1 -1
  66. package/dist/sync/sync.d.ts +2 -2
  67. package/dist/sync/sync.js.map +1 -1
  68. package/dist/tracing/PerformanceTracer.d.ts +60 -0
  69. package/dist/tracing/PerformanceTracer.js +105 -0
  70. package/dist/tracing/PerformanceTracer.js.map +1 -0
  71. package/dist/tracing/TraceWriter.d.ts +22 -0
  72. package/dist/tracing/TraceWriter.js +63 -0
  73. package/dist/tracing/TraceWriter.js.map +1 -0
  74. package/dist/util/checkpointing.js +1 -1
  75. package/dist/util/config/compound-config-collector.d.ts +1 -1
  76. package/dist/util/config/compound-config-collector.js +2 -2
  77. package/dist/util/config/compound-config-collector.js.map +1 -1
  78. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
  79. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  80. package/dist/util/env.js +1 -1
  81. package/dist/util/protocol-types.d.ts +1 -1
  82. package/dist/util/protocol-types.js +1 -1
  83. package/dist/util/util-index.d.ts +1 -0
  84. package/dist/util/util-index.js +1 -0
  85. package/dist/util/util-index.js.map +1 -1
  86. package/dist/util/utils.d.ts +5 -0
  87. package/dist/util/utils.js +7 -0
  88. package/dist/util/utils.js.map +1 -1
  89. package/package.json +11 -11
  90. package/src/api/RouteAPI.ts +3 -3
  91. package/src/api/diagnostics.ts +29 -6
  92. package/src/auth/RemoteJWKSCollector.ts +3 -1
  93. package/src/entry/commands/teardown-action.ts +1 -1
  94. package/src/index.ts +2 -0
  95. package/src/modules/AbstractModule.ts +1 -1
  96. package/src/replication/AbstractReplicationJob.ts +1 -1
  97. package/src/replication/AbstractReplicator.ts +23 -23
  98. package/src/replication/RelationCache.ts +23 -4
  99. package/src/routes/configure-fastify.ts +8 -1
  100. package/src/routes/endpoints/admin.ts +17 -8
  101. package/src/routes/endpoints/checkpointing.ts +1 -1
  102. package/src/routes/endpoints/socket-route.ts +1 -1
  103. package/src/routes/endpoints/sync-rules.ts +8 -8
  104. package/src/routes/endpoints/sync-stream.ts +2 -2
  105. package/src/routes/route-register.ts +73 -4
  106. package/src/runner/teardown.ts +4 -4
  107. package/src/storage/BucketStorage.ts +9 -9
  108. package/src/storage/BucketStorageBatch.ts +32 -0
  109. package/src/storage/BucketStorageFactory.ts +35 -23
  110. package/src/storage/PersistedSyncRulesContent.ts +20 -12
  111. package/src/storage/SourceEntity.ts +9 -1
  112. package/src/storage/SourceTable.ts +56 -22
  113. package/src/storage/StorageVersionConfig.ts +1 -1
  114. package/src/storage/SyncRulesBucketStorage.ts +74 -22
  115. package/src/storage/WriteCheckpointAPI.ts +6 -6
  116. package/src/storage/bson.ts +0 -5
  117. package/src/sync/BucketChecksumState.ts +142 -78
  118. package/src/sync/sync.ts +4 -4
  119. package/src/tracing/PerformanceTracer.ts +149 -0
  120. package/src/tracing/TraceWriter.ts +67 -0
  121. package/src/util/checkpointing.ts +1 -1
  122. package/src/util/config/compound-config-collector.ts +3 -3
  123. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
  124. package/src/util/env.ts +1 -1
  125. package/src/util/protocol-types.ts +1 -1
  126. package/src/util/util-index.ts +1 -0
  127. package/src/util/utils.ts +8 -0
  128. package/test/src/auth.test.ts +120 -1
  129. package/test/src/diagnostics.test.ts +155 -0
  130. package/test/src/routes/error-handler.integration.test.ts +275 -0
  131. package/test/src/routes/stream.test.ts +15 -4
  132. package/test/src/storage/SourceTable.test.ts +89 -0
  133. package/test/src/sync/BucketChecksumState.test.ts +244 -80
  134. package/test/tsconfig.json +0 -1
  135. package/tsconfig.tsbuildinfo +1 -1
@@ -2,7 +2,7 @@ import {
2
2
  BucketParameterQuerier,
3
3
  BucketPriority,
4
4
  BucketSource,
5
- HydratedSyncRules,
5
+ HydratedSyncConfig,
6
6
  mergeBuckets,
7
7
  QuerierError,
8
8
  RequestedStream,
@@ -21,13 +21,14 @@ import {
21
21
  ServiceError
22
22
  } from '@powersync/lib-services-framework';
23
23
  import { JwtPayload } from '../auth/JwtPayload.js';
24
+ import { ParameterSetLimitExceededError } from '../storage/storage-index.js';
24
25
  import { SyncContext } from './SyncContext.js';
25
26
  import { getIntersection, hasIntersection } from './util.js';
26
27
 
27
28
  export interface BucketChecksumStateOptions {
28
29
  syncContext: SyncContext;
29
30
  bucketStorage: BucketChecksumStateStorage;
30
- syncRules: HydratedSyncRules;
31
+ syncRules: HydratedSyncConfig;
31
32
  tokenPayload: JwtPayload;
32
33
  syncRequest: util.StreamingSyncRequest;
33
34
  logger?: Logger;
@@ -118,16 +119,38 @@ export class BucketChecksumState {
118
119
  const storage = this.bucketStorage;
119
120
 
120
121
  const update = await this.parameterState.getCheckpointUpdate(next);
121
- const { buckets: allBuckets, updatedBuckets, parameterQueryResultsByDefinition } = update;
122
+ const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
122
123
 
123
124
  /** Set of all buckets in this checkpoint. */
124
125
  const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
125
126
 
126
127
  if (bucketDescriptionMap.size > this.context.maxBuckets) {
127
- throw new ServiceError(
128
+ const error = new ServiceError(
128
129
  ErrorCode.PSYNC_S2305,
129
130
  `Too many buckets: ${bucketDescriptionMap.size} (limit of ${this.context.maxBuckets})`
130
131
  );
132
+
133
+ let errorMessage = error.message;
134
+ const logData: any = {
135
+ checkpoint: next.base.checkpoint,
136
+ user_id: this.parameterState.syncParams.userId,
137
+ buckets: allBuckets.length
138
+ };
139
+
140
+ // Count buckets per definition.
141
+ const bucketsByDefinition = new Map<string, number>();
142
+ for (const bucket of bucketDescriptionMap.values()) {
143
+ const definition = bucket.definition;
144
+ const count = bucketsByDefinition.get(definition) ?? 0;
145
+ bucketsByDefinition.set(definition, count + 1);
146
+ }
147
+
148
+ const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition);
149
+ errorMessage += breakdown.message;
150
+ logData.buckets_by_definition = breakdown.countsByDefinition;
151
+
152
+ this.logger.error(errorMessage, logData);
153
+ throw error;
131
154
  }
132
155
 
133
156
  let checksumMap: util.ChecksumMap;
@@ -211,13 +234,10 @@ export class BucketChecksumState {
211
234
  });
212
235
 
213
236
  deferredLog = () => {
214
- const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
215
237
  let message = `Updated checkpoint: ${base.checkpoint} | `;
216
238
  message += `write: ${writeCheckpoint} | `;
217
239
  message += `buckets: ${allBuckets.length} | `;
218
- if (totalParamResults !== undefined) {
219
- message += `param_results: ${totalParamResults} | `;
220
- }
240
+ message += `param_results: ${usedParameterResults} | `;
221
241
  message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
222
242
  message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
223
243
  logCheckpoint(
@@ -230,7 +250,7 @@ export class BucketChecksumState {
230
250
  updated: diff.updatedBuckets.length,
231
251
  removed: diff.removedBuckets.length
232
252
  },
233
- totalParamResults
253
+ usedParameterResults
234
254
  );
235
255
  };
236
256
 
@@ -244,12 +264,9 @@ export class BucketChecksumState {
244
264
  } satisfies util.StreamingSyncCheckpointDiff;
245
265
  } else {
246
266
  deferredLog = () => {
247
- const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
248
267
  let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
249
268
  message += `buckets: ${allBuckets.length}`;
250
- if (totalParamResults !== undefined) {
251
- message += ` | param_results: ${totalParamResults}`;
252
- }
269
+ message += ` | param_results: ${usedParameterResults}`;
253
270
  message += ` ${limitedBuckets(allBuckets, 20)}`;
254
271
  logCheckpoint(
255
272
  this.logger,
@@ -259,7 +276,7 @@ export class BucketChecksumState {
259
276
  user_id: userIdForLogs,
260
277
  buckets: allBuckets.length
261
278
  },
262
- totalParamResults
279
+ usedParameterResults
263
280
  );
264
281
  };
265
282
  bucketsToFetch = allBuckets;
@@ -268,7 +285,7 @@ export class BucketChecksumState {
268
285
  const streamNameToIndex = new Map<string, number>();
269
286
  this.streamNameToIndex = streamNameToIndex;
270
287
 
271
- for (const source of this.parameterState.syncRules.definition.bucketSources) {
288
+ for (const source of this.parameterState.syncRules.bucketSourceDefinitions) {
272
289
  if (this.parameterState.isSubscribedToStream(source)) {
273
290
  streamNameToIndex.set(source.name, subscriptions.length);
274
291
 
@@ -392,17 +409,14 @@ export interface CheckpointUpdate {
392
409
  */
393
410
  updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
394
411
 
395
- /**
396
- * Number of parameter query results per sync stream definition (before deduplication).
397
- * Map from definition name to count.
398
- */
399
- parameterQueryResultsByDefinition?: Map<string, number>;
412
+ /** The amount of rows fetched from parameters indexes. */
413
+ usedParameterResults: number;
400
414
  }
401
415
 
402
416
  export class BucketParameterState {
403
417
  private readonly context: SyncContext;
404
418
  public readonly bucketStorage: BucketChecksumStateStorage;
405
- public readonly syncRules: HydratedSyncRules;
419
+ public readonly syncRules: HydratedSyncConfig;
406
420
  public readonly syncParams: RequestParameters;
407
421
  private readonly querier: BucketParameterQuerier;
408
422
  /**
@@ -425,7 +439,7 @@ export class BucketParameterState {
425
439
  constructor(
426
440
  context: SyncContext,
427
441
  bucketStorage: BucketChecksumStateStorage,
428
- syncRules: HydratedSyncRules,
442
+ syncRules: HydratedSyncConfig,
429
443
  tokenPayload: JwtPayload,
430
444
  request: util.StreamingSyncRequest,
431
445
  logger: Logger
@@ -502,36 +516,41 @@ export class BucketParameterState {
502
516
  const querier = this.querier;
503
517
  let update: CheckpointUpdate;
504
518
  if (querier.hasDynamicBuckets) {
505
- update = await this.getCheckpointUpdateDynamic(checkpoint);
506
- } else {
507
- update = await this.getCheckpointUpdateStatic(checkpoint);
508
- }
509
-
510
- if (update.buckets.length > this.context.maxParameterQueryResults) {
511
- // TODO: Limit number of results even before we get to this point
512
- // This limit applies _before_ we get the unique set
513
- const error = new ServiceError(
514
- ErrorCode.PSYNC_S2305,
515
- `Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`
516
- );
519
+ try {
520
+ update = await this.getCheckpointUpdateDynamic(checkpoint);
521
+ } catch (e: unknown) {
522
+ if (e instanceof ParameterSetLimitExceededError) {
523
+ // Too many parameter results, create a breakdown of which streams are responsible for the most queries and
524
+ // then abort.
525
+ const error = new ServiceError(
526
+ ErrorCode.PSYNC_S2305,
527
+ `Too many parameter query results (limit of ${this.context.maxParameterQueryResults})`
528
+ );
529
+
530
+ let errorMessage = error.message;
531
+ const logData: any = {
532
+ checkpoint: checkpoint.base.checkpoint,
533
+ user_id: this.syncParams.userId
534
+ };
535
+
536
+ if (e.breakdown) {
537
+ const breakdown = formatParameterQueryBreakdown(e.breakdown);
538
+ if (breakdown) {
539
+ errorMessage += breakdown.message;
540
+ logData.parameterResults = breakdown.largestResults;
541
+ }
542
+ }
517
543
 
518
- let errorMessage = error.message;
519
- const logData: any = {
520
- checkpoint: checkpoint.base.checkpoint,
521
- user_id: this.syncParams.userId,
522
- parameter_query_results: update.buckets.length
523
- };
544
+ this.logger.error(errorMessage, logData);
545
+ throw error;
546
+ }
524
547
 
525
- if (update.parameterQueryResultsByDefinition && update.parameterQueryResultsByDefinition.size > 0) {
526
- const breakdown = formatParameterQueryBreakdown(update.parameterQueryResultsByDefinition);
527
- errorMessage += breakdown.message;
528
- logData.parameter_query_results_by_definition = breakdown.countsByDefinition;
548
+ throw e;
529
549
  }
530
-
531
- this.logger.error(errorMessage, logData);
532
-
533
- throw error;
550
+ } else {
551
+ update = await this.getCheckpointUpdateStatic(checkpoint);
534
552
  }
553
+
535
554
  return update;
536
555
  }
537
556
 
@@ -545,14 +564,16 @@ export class BucketParameterState {
545
564
  if (update.invalidateDataBuckets) {
546
565
  return {
547
566
  buckets: staticBuckets,
548
- updatedBuckets: INVALIDATE_ALL_BUCKETS
567
+ updatedBuckets: INVALIDATE_ALL_BUCKETS,
568
+ usedParameterResults: 0
549
569
  };
550
570
  }
551
571
 
552
572
  const updatedBuckets = new Set<string>(getIntersection(this.staticBuckets, update.updatedDataBuckets));
553
573
  return {
554
574
  buckets: staticBuckets,
555
- updatedBuckets
575
+ updatedBuckets,
576
+ usedParameterResults: 0
556
577
  };
557
578
  }
558
579
 
@@ -584,27 +605,55 @@ export class BucketParameterState {
584
605
  }
585
606
 
586
607
  let dynamicBuckets: ResolvedBucket[];
587
- let parameterQueryResultsByDefinition: Map<string, number> | undefined;
608
+ let usedParameterResults = 0;
588
609
  if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
589
610
  const recordedLookups = new Set<string>();
611
+ const parameterLimit = this.context.maxParameterQueryResults;
612
+ let remainingBudget = parameterLimit;
613
+
614
+ // Log of lookups to provide a breakdown if we exceed the dynamic lookup limit.
615
+ // FIXME: This is horrible, Sync Streams will invoke the callback concurrently and we can't properly deal with
616
+ // that. We should replace queriers for Sync Streams with an explicit graph structure based on sync plans instead
617
+ // of adding these checks to the imperative querier interface.
618
+ const lookupLog: storage.ParameterQueryInvocationLog[] = [];
590
619
 
591
620
  dynamicBuckets = await querier.queryDynamicBucketDescriptions({
592
- getParameterSets(lookups) {
621
+ getParameterSets: async (lookups, definition) => {
622
+ if (lookups.length > parameterLimit) {
623
+ // Sync Streams can chain parameter lookups, so a large output from an earlier call may become the input
624
+ // here. We reuse the output limit as a generous upper bound; legitimate queries are much smaller.
625
+ const msg = `Attempted to fetch ${lookups.length} lookups at once, a maximum of ${parameterLimit} lookups are allowed.`;
626
+ this.logger.error(msg, {
627
+ user_id: this.syncParams.userId,
628
+ checkpoint: checkpoint.base.checkpoint,
629
+ cause: definition
630
+ });
631
+ throw new ServiceError(ErrorCode.PSYNC_S2305, msg);
632
+ }
633
+
593
634
  for (const lookup of lookups) {
594
635
  recordedLookups.add(lookup.serializedRepresentation);
595
636
  }
596
637
 
597
- return checkpoint.base.getParameterSets(lookups);
638
+ try {
639
+ const results = await checkpoint.base.getParameterSets(lookups, remainingBudget);
640
+ const numRows = results.reduce((a, b) => a + b.rows.length, 0);
641
+
642
+ lookupLog.push({ definition, resultsOrLimit: numRows, didExceedLimit: false });
643
+ remainingBudget -= numRows;
644
+ usedParameterResults += numRows;
645
+ return results;
646
+ } catch (e: unknown) {
647
+ if (e instanceof ParameterSetLimitExceededError) {
648
+ lookupLog.push({ definition, resultsOrLimit: remainingBudget, didExceedLimit: true });
649
+ throw new ParameterSetLimitExceededError(parameterLimit, lookupLog);
650
+ }
651
+
652
+ throw e;
653
+ }
598
654
  }
599
655
  });
600
656
 
601
- // Count parameter query results per definition (before deduplication)
602
- parameterQueryResultsByDefinition = new Map<string, number>();
603
- for (const bucket of dynamicBuckets) {
604
- const count = parameterQueryResultsByDefinition.get(bucket.definition) ?? 0;
605
- parameterQueryResultsByDefinition.set(bucket.definition, count + 1);
606
- }
607
-
608
657
  this.cachedDynamicBuckets = dynamicBuckets;
609
658
  this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
610
659
  this.lookupsFromPreviousCheckpoint = recordedLookups;
@@ -628,13 +677,13 @@ export class BucketParameterState {
628
677
  buckets: allBuckets,
629
678
  // We cannot track individual bucket updates for dynamic lookups yet
630
679
  updatedBuckets: INVALIDATE_ALL_BUCKETS,
631
- parameterQueryResultsByDefinition
680
+ usedParameterResults
632
681
  };
633
682
  } else {
634
683
  return {
635
684
  buckets: allBuckets,
636
685
  updatedBuckets: updatedBuckets,
637
- parameterQueryResultsByDefinition
686
+ usedParameterResults
638
687
  };
639
688
  }
640
689
  }
@@ -668,18 +717,6 @@ export interface CheckpointLine {
668
717
  // Use a more specific type to simplify testing
669
718
  export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
670
719
 
671
- /**
672
- * Compute the total number of parameter query results across all definitions.
673
- */
674
- function computeTotalParamResults(
675
- parameterQueryResultsByDefinition: Map<string, number> | undefined
676
- ): number | undefined {
677
- if (!parameterQueryResultsByDefinition) {
678
- return undefined;
679
- }
680
- return Array.from(parameterQueryResultsByDefinition.values()).reduce((sum, count) => sum + count, 0);
681
- }
682
-
683
720
  /**
684
721
  * Log a checkpoint message, enriching it with parameter query result counts if available.
685
722
  *
@@ -701,20 +738,20 @@ function logCheckpoint(
701
738
  }
702
739
 
703
740
  /**
704
- * Format a breakdown of parameter query results by sync rule definition.
741
+ * Format a breakdown of dynamic bucket by sync stream definition.
705
742
  *
706
743
  * Sorts definitions by count (descending), includes the top 10, and returns both the
707
744
  * formatted message string and the counts record suitable for structured log data.
708
745
  */
709
- function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<string, number>): {
746
+ function formatBucketDefinitionBreakdown(bucketsByDefinition: Map<string, number>): {
710
747
  message: string;
711
748
  countsByDefinition: Record<string, number>;
712
749
  } {
713
750
  // Sort definitions by count (descending) and take top 10
714
- const allSorted = Array.from(parameterQueryResultsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
751
+ const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
715
752
  const sortedDefinitions = allSorted.slice(0, 10);
716
753
 
717
- let message = '\nParameter query results by definition:';
754
+ let message = '\Buckets by definition:';
718
755
  const countsByDefinition: Record<string, number> = {};
719
756
  for (const [definition, count] of sortedDefinitions) {
720
757
  message += `\n ${definition}: ${count}`;
@@ -730,6 +767,33 @@ function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<st
730
767
  return { message, countsByDefinition };
731
768
  }
732
769
 
770
+ function formatParameterQueryBreakdown(log: storage.ParameterQueryInvocationLog[]) {
771
+ if (log.length == 0) {
772
+ return;
773
+ }
774
+
775
+ // When an exception about too many parameter results is thrown, the last entry is the one that finally exceeded the
776
+ // limit.
777
+ const results = Array.from(log);
778
+ const failure = results.pop()!;
779
+
780
+ let message = '\nInvoked parameter queries by definition:';
781
+ results.sort((a, b) => b.resultsOrLimit - a.resultsOrLimit);
782
+ const largestResults = results.splice(0, 9);
783
+
784
+ for (const entry of largestResults) {
785
+ message += `\n ${entry.definition}: ${entry.resultsOrLimit} results.`;
786
+ }
787
+
788
+ if (largestResults.length < log.length) {
789
+ message += `\n ... and ${log.length - largestResults.length} more invocations`;
790
+ }
791
+
792
+ message += `\n ${failure.definition} exceeded the remaining limit of ${failure.resultsOrLimit} available results.`;
793
+ largestResults.push(failure);
794
+ return { message, largestResults };
795
+ }
796
+
733
797
  function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
734
798
  buckets = buckets.map((b) => {
735
799
  if (typeof b != 'string') {
package/src/sync/sync.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
2
- import { BucketPriority, HydratedSyncRules, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
2
+ import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
3
3
 
4
4
  import { AbortError } from 'ix/aborterror.js';
5
5
 
@@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr
17
17
  export interface SyncStreamParameters {
18
18
  syncContext: SyncContext;
19
19
  bucketStorage: storage.SyncRulesBucketStorage;
20
- syncRules: HydratedSyncRules;
20
+ syncRules: HydratedSyncConfig;
21
21
  params: util.StreamingSyncRequest;
22
22
  token: auth.JwtPayload;
23
23
  logger?: Logger;
@@ -94,7 +94,7 @@ export async function* streamResponse(
94
94
  async function* streamResponseInner(
95
95
  syncContext: SyncContext,
96
96
  bucketStorage: storage.SyncRulesBucketStorage,
97
- syncRules: HydratedSyncRules,
97
+ syncRules: HydratedSyncConfig,
98
98
  params: util.StreamingSyncRequest,
99
99
  tokenPayload: auth.JwtPayload,
100
100
  tracker: RequestTracker,
@@ -267,7 +267,7 @@ interface BucketDataRequest {
267
267
  * This signal also fires when abort_connection fires.
268
268
  */
269
269
  abort_batch: AbortSignal;
270
- /** User id for debug purposes, not for sync rules. */
270
+ /** User id for debug purposes, not for sync config. */
271
271
  userIdForLogs?: SqliteJsonValue;
272
272
  forPriority: BucketPriority | null;
273
273
  onRowsSent: (stats: OperationsSentStats) => void;
@@ -0,0 +1,149 @@
1
+ import { traceWriter } from './TraceWriter.js';
2
+
3
+ export interface Span extends Disposable {
4
+ name: string;
5
+ /**
6
+ * Start time in microseconds since an arbitrary epoch.
7
+ */
8
+ startAt: number;
9
+ /**
10
+ * End time in microseconds since an arbitrary epoch.
11
+ */
12
+ endAt: number;
13
+ /**
14
+ * Time spent not in nested spans.
15
+ */
16
+ selfDuration: number;
17
+
18
+ nestedSince: number | undefined;
19
+ subtrackFromSelf: number;
20
+
21
+ /**
22
+ * Durations spent in nested spans, in microseconds.
23
+ */
24
+ nestedDurations: Record<string, number>;
25
+
26
+ /**
27
+ * Total duration of this span in milliseconds, rounded up. Only valid after the span has ended.
28
+ */
29
+ durationMillis: number;
30
+
31
+ /**
32
+ * End the span - same as [Symbol.dispose]().
33
+ *
34
+ * Safe to call multiple times. Any nested spans will automatically end as well.
35
+ *
36
+ * Returns an aggregate record of category -> "selfDuration", in microseconds.
37
+ */
38
+ end(): Record<string, number>;
39
+ }
40
+
41
+ function now() {
42
+ return Number(process.hrtime.bigint() / 1000n);
43
+ }
44
+
45
+ let nextThreadId = 1;
46
+
47
+ /**
48
+ * Lightweight tracing helper, with two main goals:
49
+ * 1. Generate aggregate timing info with low overhead.
50
+ * 2. Optional support for generating trace files during development.
51
+ *
52
+ * This is only intended for a single "thread" - concurrent operations on the same instance have undefined behavior.
53
+ * To trace concurrent operations, use separate instances of PerformanceTracer.
54
+ *
55
+ * Spans cannot be overlapping: If a parent span is ended, all nested spans are automatically ended.
56
+ */
57
+ export class PerformanceTracer<K extends string> {
58
+ stack: Span[] = [];
59
+ threadId: number;
60
+
61
+ constructor(traceName: string) {
62
+ this.threadId = nextThreadId;
63
+ nextThreadId += 1;
64
+ traceWriter?.write({
65
+ ph: 'M',
66
+ cat: '__metadata',
67
+ name: 'thread_name',
68
+ pid: process.pid,
69
+ tid: this.threadId,
70
+ args: { name: `PowerSync ${traceName}` }
71
+ });
72
+ }
73
+
74
+ /**
75
+ * Recommended usage:
76
+ *
77
+ * using _ = tracer.span('cat', 'details');
78
+ *
79
+ * The above automatically ends the span when it goes out of scope. Alternatively, call
80
+ * .end() on the span to end it earlier.
81
+ *
82
+ * @param category one of the defined categories
83
+ * @param subcat optional subcategory. Not used for calculating "self" durations in the aggregate API.
84
+ */
85
+ span(category: K, subcat?: string): Span {
86
+ const stack = this.stack;
87
+ const index = this.stack.length;
88
+ const parent = this.stack[this.stack.length - 1];
89
+ const threadId = this.threadId;
90
+ const startAt = now();
91
+ if (parent != null) {
92
+ parent.nestedSince ??= startAt;
93
+ }
94
+ let name: string = category;
95
+ if (subcat) {
96
+ name += ':' + subcat;
97
+ }
98
+ const s: Span = {
99
+ name,
100
+ startAt: now(),
101
+ selfDuration: 0,
102
+ endAt: 0,
103
+ nestedSince: undefined,
104
+ subtrackFromSelf: 0,
105
+ nestedDurations: {},
106
+ end() {
107
+ if (this.endAt != 0) {
108
+ return this.nestedDurations;
109
+ }
110
+ while (stack.length - 1 > index) {
111
+ stack[stack.length - 1].end();
112
+ }
113
+ const endAt = now();
114
+ this.endAt = endAt;
115
+ const endTime = this.nestedSince ?? endAt;
116
+ this.selfDuration = endTime - startAt - this.subtrackFromSelf;
117
+ traceWriter?.write({
118
+ name,
119
+ cat: 'powersync',
120
+ ph: 'X',
121
+ ts: this.startAt,
122
+ dur: endAt - startAt,
123
+ pid: process.pid,
124
+ tid: threadId
125
+ });
126
+ stack.pop();
127
+ if (parent != null) {
128
+ parent.subtrackFromSelf += endAt - parent.nestedSince!;
129
+ for (let key in this.nestedDurations) {
130
+ parent.nestedDurations[key] = (parent.nestedDurations[key] ?? 0) + this.nestedDurations[key];
131
+ }
132
+ parent.nestedDurations[category] = (parent.nestedDurations[category] ?? 0) + this.selfDuration;
133
+ parent.nestedSince = undefined;
134
+ }
135
+ return this.nestedDurations;
136
+ },
137
+
138
+ get durationMillis() {
139
+ return Math.ceil((this.endAt - this.startAt) / 1000);
140
+ },
141
+ [Symbol.dispose]() {
142
+ this.end();
143
+ }
144
+ };
145
+ this.stack.push(s);
146
+
147
+ return s;
148
+ }
149
+ }
@@ -0,0 +1,67 @@
1
+ import { Mutex } from 'async-mutex';
2
+ import * as fs from 'node:fs/promises';
3
+
4
+ /**
5
+ * Write traces in the Chrome JSON Trace Format.
6
+ *
7
+ * View at https://ui.perfetto.dev/
8
+ */
9
+ class TraceWriter {
10
+ handle: fs.FileHandle | null = null;
11
+ length = 0;
12
+ queue: any[] = [];
13
+ private mutex = new Mutex();
14
+
15
+ constructor(public readonly path: string) {
16
+ this.open().catch((e) => {
17
+ console.error(`Failed to open trace file at ${path}`, e);
18
+ });
19
+ }
20
+
21
+ async open() {
22
+ await this.mutex.runExclusive(async () => {
23
+ this.handle = await fs.open(this.path, 'w+');
24
+ this.handle.truncate(0);
25
+ await this.handle.write('[]');
26
+ this.length = 2;
27
+ });
28
+ }
29
+
30
+ write(...traceEvents: any[]) {
31
+ this.writeAsync(...traceEvents).catch((e) => {
32
+ console.error(`Failed to write trace file`, e);
33
+ });
34
+ }
35
+
36
+ async writeAsync(...traceEvents: any[]) {
37
+ this.queue.push(...traceEvents);
38
+ await this.mutex.runExclusive(async () => {
39
+ if (this.queue.length > 0) {
40
+ // Write queued events.
41
+ // After each write, we end the file as a valid JSON array.
42
+ // On the next write, we overwrite the last character to extend the array.
43
+ const buffer = Buffer.from(JSON.stringify(this.queue));
44
+ await this.handle?.write(buffer, 1, buffer.length - 1, this.length - 1);
45
+ this.queue = [];
46
+ this.length += buffer.length - 2;
47
+ }
48
+ });
49
+ }
50
+ }
51
+
52
+ const traceFile = process.env.POWERSYNC_TRACE_FILE;
53
+ /**
54
+ * traceWriter, only present if POWERSYNC_TRACE_FILE env var is configured.
55
+ */
56
+ export const traceWriter = traceFile ? new TraceWriter(traceFile) : null;
57
+
58
+ if (traceWriter) {
59
+ traceWriter.write({
60
+ ph: 'M',
61
+ cat: '__metadata',
62
+ name: 'process_name',
63
+ pid: process.pid,
64
+ tid: 1000,
65
+ args: { name: 'powersync' }
66
+ });
67
+ }
@@ -13,7 +13,7 @@ export async function createWriteCheckpoint(options: CreateWriteCheckpointOption
13
13
 
14
14
  const syncBucketStorage = await options.storage.getActiveStorage();
15
15
  if (!syncBucketStorage) {
16
- throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync rules are active.`);
16
+ throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
17
17
  }
18
18
 
19
19
  const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
@@ -26,7 +26,7 @@ export type CompoundConfigCollectorOptions = {
26
26
  */
27
27
  configCollectors: ConfigCollector[];
28
28
  /**
29
- * Collectors for PowerSync sync rules content.
29
+ * Collectors for PowerSync sync config content.
30
30
  * The configuration from first collector to provide a configuration
31
31
  * is used. The order of the collectors specifies precedence
32
32
  */
@@ -236,11 +236,11 @@ export class CompoundConfigCollector {
236
236
  return config;
237
237
  }
238
238
  logger.debug(
239
- `Could not collect sync rules with ${collector.name} method. Moving on to next method if available.`
239
+ `Could not collect sync config with ${collector.name} method. Moving on to next method if available.`
240
240
  );
241
241
  } catch (ex) {
242
242
  // An error in a collector is a hard stop
243
- throw new Error(`Could not collect sync rules using ${collector.name} method. Caught exception: ${ex}`);
243
+ throw new Error(`Could not collect sync config using ${collector.name} method. Caught exception: ${ex}`);
244
244
  }
245
245
  }
246
246
  return {
@@ -16,7 +16,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
16
16
 
17
17
  const { config_path } = runnerConfig;
18
18
 
19
- // Depending on the container, the sync rules may not actually be present.
19
+ // Depending on the container, the sync config may not actually be present.
20
20
  // Only persist the path here, and load on demand using `loadSyncRules()`.
21
21
  return {
22
22
  present: true,