@powersync/service-core 1.20.5 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/dist/api/RouteAPI.d.ts +3 -3
- package/dist/api/diagnostics.d.ts +1 -1
- package/dist/api/diagnostics.js +19 -3
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js +3 -2
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +1 -1
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +1 -1
- package/dist/replication/AbstractReplicationJob.js +1 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +6 -6
- package/dist/replication/AbstractReplicator.js +21 -21
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/RelationCache.d.ts +9 -2
- package/dist/replication/RelationCache.js +21 -2
- package/dist/replication/RelationCache.js.map +1 -1
- package/dist/routes/configure-fastify.js +3 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.js +16 -8
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +8 -8
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +2 -2
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +2 -0
- package/dist/routes/route-register.js +65 -3
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +4 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +9 -9
- package/dist/storage/BucketStorage.js +9 -9
- package/dist/storage/BucketStorageBatch.d.ts +29 -0
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +27 -18
- package/dist/storage/BucketStorageFactory.js +13 -12
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +6 -4
- package/dist/storage/PersistedSyncRulesContent.js +15 -8
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/SourceEntity.d.ts +8 -1
- package/dist/storage/SourceTable.d.ts +32 -11
- package/dist/storage/SourceTable.js +41 -15
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +1 -1
- package/dist/storage/StorageVersionConfig.js +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +63 -18
- package/dist/storage/SyncRulesBucketStorage.js +14 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
- package/dist/storage/WriteCheckpointAPI.js +1 -1
- package/dist/storage/bson.d.ts +0 -1
- package/dist/storage/bson.js +0 -4
- package/dist/storage/bson.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +6 -9
- package/dist/sync/BucketChecksumState.js +117 -58
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -2
- package/dist/sync/sync.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +60 -0
- package/dist/tracing/PerformanceTracer.js +105 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -0
- package/dist/tracing/TraceWriter.d.ts +22 -0
- package/dist/tracing/TraceWriter.js +63 -0
- package/dist/tracing/TraceWriter.js.map +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/config/compound-config-collector.d.ts +1 -1
- package/dist/util/config/compound-config-collector.js +2 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/protocol-types.d.ts +1 -1
- package/dist/util/protocol-types.js +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +5 -0
- package/dist/util/utils.js +7 -0
- package/dist/util/utils.js.map +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +3 -3
- package/src/api/diagnostics.ts +29 -6
- package/src/auth/RemoteJWKSCollector.ts +3 -1
- package/src/entry/commands/teardown-action.ts +1 -1
- package/src/index.ts +2 -0
- package/src/modules/AbstractModule.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +1 -1
- package/src/replication/AbstractReplicator.ts +23 -23
- package/src/replication/RelationCache.ts +23 -4
- package/src/routes/configure-fastify.ts +8 -1
- package/src/routes/endpoints/admin.ts +17 -8
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +8 -8
- package/src/routes/endpoints/sync-stream.ts +2 -2
- package/src/routes/route-register.ts +73 -4
- package/src/runner/teardown.ts +4 -4
- package/src/storage/BucketStorage.ts +9 -9
- package/src/storage/BucketStorageBatch.ts +32 -0
- package/src/storage/BucketStorageFactory.ts +35 -23
- package/src/storage/PersistedSyncRulesContent.ts +20 -12
- package/src/storage/SourceEntity.ts +9 -1
- package/src/storage/SourceTable.ts +56 -22
- package/src/storage/StorageVersionConfig.ts +1 -1
- package/src/storage/SyncRulesBucketStorage.ts +74 -22
- package/src/storage/WriteCheckpointAPI.ts +6 -6
- package/src/storage/bson.ts +0 -5
- package/src/sync/BucketChecksumState.ts +142 -78
- package/src/sync/sync.ts +4 -4
- package/src/tracing/PerformanceTracer.ts +149 -0
- package/src/tracing/TraceWriter.ts +67 -0
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/compound-config-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
- package/src/util/env.ts +1 -1
- package/src/util/protocol-types.ts +1 -1
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +8 -0
- package/test/src/auth.test.ts +120 -1
- package/test/src/diagnostics.test.ts +155 -0
- package/test/src/routes/error-handler.integration.test.ts +275 -0
- package/test/src/routes/stream.test.ts +15 -4
- package/test/src/storage/SourceTable.test.ts +89 -0
- package/test/src/sync/BucketChecksumState.test.ts +244 -80
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
BucketParameterQuerier,
|
|
3
3
|
BucketPriority,
|
|
4
4
|
BucketSource,
|
|
5
|
-
|
|
5
|
+
HydratedSyncConfig,
|
|
6
6
|
mergeBuckets,
|
|
7
7
|
QuerierError,
|
|
8
8
|
RequestedStream,
|
|
@@ -21,13 +21,14 @@ import {
|
|
|
21
21
|
ServiceError
|
|
22
22
|
} from '@powersync/lib-services-framework';
|
|
23
23
|
import { JwtPayload } from '../auth/JwtPayload.js';
|
|
24
|
+
import { ParameterSetLimitExceededError } from '../storage/storage-index.js';
|
|
24
25
|
import { SyncContext } from './SyncContext.js';
|
|
25
26
|
import { getIntersection, hasIntersection } from './util.js';
|
|
26
27
|
|
|
27
28
|
export interface BucketChecksumStateOptions {
|
|
28
29
|
syncContext: SyncContext;
|
|
29
30
|
bucketStorage: BucketChecksumStateStorage;
|
|
30
|
-
syncRules:
|
|
31
|
+
syncRules: HydratedSyncConfig;
|
|
31
32
|
tokenPayload: JwtPayload;
|
|
32
33
|
syncRequest: util.StreamingSyncRequest;
|
|
33
34
|
logger?: Logger;
|
|
@@ -118,16 +119,38 @@ export class BucketChecksumState {
|
|
|
118
119
|
const storage = this.bucketStorage;
|
|
119
120
|
|
|
120
121
|
const update = await this.parameterState.getCheckpointUpdate(next);
|
|
121
|
-
const { buckets: allBuckets, updatedBuckets,
|
|
122
|
+
const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
|
|
122
123
|
|
|
123
124
|
/** Set of all buckets in this checkpoint. */
|
|
124
125
|
const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
|
|
125
126
|
|
|
126
127
|
if (bucketDescriptionMap.size > this.context.maxBuckets) {
|
|
127
|
-
|
|
128
|
+
const error = new ServiceError(
|
|
128
129
|
ErrorCode.PSYNC_S2305,
|
|
129
130
|
`Too many buckets: ${bucketDescriptionMap.size} (limit of ${this.context.maxBuckets})`
|
|
130
131
|
);
|
|
132
|
+
|
|
133
|
+
let errorMessage = error.message;
|
|
134
|
+
const logData: any = {
|
|
135
|
+
checkpoint: next.base.checkpoint,
|
|
136
|
+
user_id: this.parameterState.syncParams.userId,
|
|
137
|
+
buckets: allBuckets.length
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
// Count buckets per definition.
|
|
141
|
+
const bucketsByDefinition = new Map<string, number>();
|
|
142
|
+
for (const bucket of bucketDescriptionMap.values()) {
|
|
143
|
+
const definition = bucket.definition;
|
|
144
|
+
const count = bucketsByDefinition.get(definition) ?? 0;
|
|
145
|
+
bucketsByDefinition.set(definition, count + 1);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition);
|
|
149
|
+
errorMessage += breakdown.message;
|
|
150
|
+
logData.buckets_by_definition = breakdown.countsByDefinition;
|
|
151
|
+
|
|
152
|
+
this.logger.error(errorMessage, logData);
|
|
153
|
+
throw error;
|
|
131
154
|
}
|
|
132
155
|
|
|
133
156
|
let checksumMap: util.ChecksumMap;
|
|
@@ -211,13 +234,10 @@ export class BucketChecksumState {
|
|
|
211
234
|
});
|
|
212
235
|
|
|
213
236
|
deferredLog = () => {
|
|
214
|
-
const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
|
|
215
237
|
let message = `Updated checkpoint: ${base.checkpoint} | `;
|
|
216
238
|
message += `write: ${writeCheckpoint} | `;
|
|
217
239
|
message += `buckets: ${allBuckets.length} | `;
|
|
218
|
-
|
|
219
|
-
message += `param_results: ${totalParamResults} | `;
|
|
220
|
-
}
|
|
240
|
+
message += `param_results: ${usedParameterResults} | `;
|
|
221
241
|
message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
|
|
222
242
|
message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
|
|
223
243
|
logCheckpoint(
|
|
@@ -230,7 +250,7 @@ export class BucketChecksumState {
|
|
|
230
250
|
updated: diff.updatedBuckets.length,
|
|
231
251
|
removed: diff.removedBuckets.length
|
|
232
252
|
},
|
|
233
|
-
|
|
253
|
+
usedParameterResults
|
|
234
254
|
);
|
|
235
255
|
};
|
|
236
256
|
|
|
@@ -244,12 +264,9 @@ export class BucketChecksumState {
|
|
|
244
264
|
} satisfies util.StreamingSyncCheckpointDiff;
|
|
245
265
|
} else {
|
|
246
266
|
deferredLog = () => {
|
|
247
|
-
const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
|
|
248
267
|
let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
|
|
249
268
|
message += `buckets: ${allBuckets.length}`;
|
|
250
|
-
|
|
251
|
-
message += ` | param_results: ${totalParamResults}`;
|
|
252
|
-
}
|
|
269
|
+
message += ` | param_results: ${usedParameterResults}`;
|
|
253
270
|
message += ` ${limitedBuckets(allBuckets, 20)}`;
|
|
254
271
|
logCheckpoint(
|
|
255
272
|
this.logger,
|
|
@@ -259,7 +276,7 @@ export class BucketChecksumState {
|
|
|
259
276
|
user_id: userIdForLogs,
|
|
260
277
|
buckets: allBuckets.length
|
|
261
278
|
},
|
|
262
|
-
|
|
279
|
+
usedParameterResults
|
|
263
280
|
);
|
|
264
281
|
};
|
|
265
282
|
bucketsToFetch = allBuckets;
|
|
@@ -268,7 +285,7 @@ export class BucketChecksumState {
|
|
|
268
285
|
const streamNameToIndex = new Map<string, number>();
|
|
269
286
|
this.streamNameToIndex = streamNameToIndex;
|
|
270
287
|
|
|
271
|
-
for (const source of this.parameterState.syncRules.
|
|
288
|
+
for (const source of this.parameterState.syncRules.bucketSourceDefinitions) {
|
|
272
289
|
if (this.parameterState.isSubscribedToStream(source)) {
|
|
273
290
|
streamNameToIndex.set(source.name, subscriptions.length);
|
|
274
291
|
|
|
@@ -392,17 +409,14 @@ export interface CheckpointUpdate {
|
|
|
392
409
|
*/
|
|
393
410
|
updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
|
|
394
411
|
|
|
395
|
-
/**
|
|
396
|
-
|
|
397
|
-
* Map from definition name to count.
|
|
398
|
-
*/
|
|
399
|
-
parameterQueryResultsByDefinition?: Map<string, number>;
|
|
412
|
+
/** The amount of rows fetched from parameters indexes. */
|
|
413
|
+
usedParameterResults: number;
|
|
400
414
|
}
|
|
401
415
|
|
|
402
416
|
export class BucketParameterState {
|
|
403
417
|
private readonly context: SyncContext;
|
|
404
418
|
public readonly bucketStorage: BucketChecksumStateStorage;
|
|
405
|
-
public readonly syncRules:
|
|
419
|
+
public readonly syncRules: HydratedSyncConfig;
|
|
406
420
|
public readonly syncParams: RequestParameters;
|
|
407
421
|
private readonly querier: BucketParameterQuerier;
|
|
408
422
|
/**
|
|
@@ -425,7 +439,7 @@ export class BucketParameterState {
|
|
|
425
439
|
constructor(
|
|
426
440
|
context: SyncContext,
|
|
427
441
|
bucketStorage: BucketChecksumStateStorage,
|
|
428
|
-
syncRules:
|
|
442
|
+
syncRules: HydratedSyncConfig,
|
|
429
443
|
tokenPayload: JwtPayload,
|
|
430
444
|
request: util.StreamingSyncRequest,
|
|
431
445
|
logger: Logger
|
|
@@ -502,36 +516,41 @@ export class BucketParameterState {
|
|
|
502
516
|
const querier = this.querier;
|
|
503
517
|
let update: CheckpointUpdate;
|
|
504
518
|
if (querier.hasDynamicBuckets) {
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
519
|
+
try {
|
|
520
|
+
update = await this.getCheckpointUpdateDynamic(checkpoint);
|
|
521
|
+
} catch (e: unknown) {
|
|
522
|
+
if (e instanceof ParameterSetLimitExceededError) {
|
|
523
|
+
// Too many parameter results, create a breakdown of which streams are responsible for the most queries and
|
|
524
|
+
// then abort.
|
|
525
|
+
const error = new ServiceError(
|
|
526
|
+
ErrorCode.PSYNC_S2305,
|
|
527
|
+
`Too many parameter query results (limit of ${this.context.maxParameterQueryResults})`
|
|
528
|
+
);
|
|
529
|
+
|
|
530
|
+
let errorMessage = error.message;
|
|
531
|
+
const logData: any = {
|
|
532
|
+
checkpoint: checkpoint.base.checkpoint,
|
|
533
|
+
user_id: this.syncParams.userId
|
|
534
|
+
};
|
|
535
|
+
|
|
536
|
+
if (e.breakdown) {
|
|
537
|
+
const breakdown = formatParameterQueryBreakdown(e.breakdown);
|
|
538
|
+
if (breakdown) {
|
|
539
|
+
errorMessage += breakdown.message;
|
|
540
|
+
logData.parameterResults = breakdown.largestResults;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
517
543
|
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
user_id: this.syncParams.userId,
|
|
522
|
-
parameter_query_results: update.buckets.length
|
|
523
|
-
};
|
|
544
|
+
this.logger.error(errorMessage, logData);
|
|
545
|
+
throw error;
|
|
546
|
+
}
|
|
524
547
|
|
|
525
|
-
|
|
526
|
-
const breakdown = formatParameterQueryBreakdown(update.parameterQueryResultsByDefinition);
|
|
527
|
-
errorMessage += breakdown.message;
|
|
528
|
-
logData.parameter_query_results_by_definition = breakdown.countsByDefinition;
|
|
548
|
+
throw e;
|
|
529
549
|
}
|
|
530
|
-
|
|
531
|
-
this.
|
|
532
|
-
|
|
533
|
-
throw error;
|
|
550
|
+
} else {
|
|
551
|
+
update = await this.getCheckpointUpdateStatic(checkpoint);
|
|
534
552
|
}
|
|
553
|
+
|
|
535
554
|
return update;
|
|
536
555
|
}
|
|
537
556
|
|
|
@@ -545,14 +564,16 @@ export class BucketParameterState {
|
|
|
545
564
|
if (update.invalidateDataBuckets) {
|
|
546
565
|
return {
|
|
547
566
|
buckets: staticBuckets,
|
|
548
|
-
updatedBuckets: INVALIDATE_ALL_BUCKETS
|
|
567
|
+
updatedBuckets: INVALIDATE_ALL_BUCKETS,
|
|
568
|
+
usedParameterResults: 0
|
|
549
569
|
};
|
|
550
570
|
}
|
|
551
571
|
|
|
552
572
|
const updatedBuckets = new Set<string>(getIntersection(this.staticBuckets, update.updatedDataBuckets));
|
|
553
573
|
return {
|
|
554
574
|
buckets: staticBuckets,
|
|
555
|
-
updatedBuckets
|
|
575
|
+
updatedBuckets,
|
|
576
|
+
usedParameterResults: 0
|
|
556
577
|
};
|
|
557
578
|
}
|
|
558
579
|
|
|
@@ -584,27 +605,55 @@ export class BucketParameterState {
|
|
|
584
605
|
}
|
|
585
606
|
|
|
586
607
|
let dynamicBuckets: ResolvedBucket[];
|
|
587
|
-
let
|
|
608
|
+
let usedParameterResults = 0;
|
|
588
609
|
if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
|
|
589
610
|
const recordedLookups = new Set<string>();
|
|
611
|
+
const parameterLimit = this.context.maxParameterQueryResults;
|
|
612
|
+
let remainingBudget = parameterLimit;
|
|
613
|
+
|
|
614
|
+
// Log of lookups to provide a breakdown if we exceed the dynamic lookup limit.
|
|
615
|
+
// FIXME: This is horrible, Sync Streams will invoke the callback concurrently and we can't properly deal with
|
|
616
|
+
// that. We should replace queriers for Sync Streams with an explicit graph structure based on sync plans instead
|
|
617
|
+
// of adding these checks to the imperative querier interface.
|
|
618
|
+
const lookupLog: storage.ParameterQueryInvocationLog[] = [];
|
|
590
619
|
|
|
591
620
|
dynamicBuckets = await querier.queryDynamicBucketDescriptions({
|
|
592
|
-
getParameterSets(lookups) {
|
|
621
|
+
getParameterSets: async (lookups, definition) => {
|
|
622
|
+
if (lookups.length > parameterLimit) {
|
|
623
|
+
// Sync Streams can chain parameter lookups, so a large output from an earlier call may become the input
|
|
624
|
+
// here. We reuse the output limit as a generous upper bound; legitimate queries are much smaller.
|
|
625
|
+
const msg = `Attempted to fetch ${lookups.length} lookups at once, a maximum of ${parameterLimit} lookups are allowed.`;
|
|
626
|
+
this.logger.error(msg, {
|
|
627
|
+
user_id: this.syncParams.userId,
|
|
628
|
+
checkpoint: checkpoint.base.checkpoint,
|
|
629
|
+
cause: definition
|
|
630
|
+
});
|
|
631
|
+
throw new ServiceError(ErrorCode.PSYNC_S2305, msg);
|
|
632
|
+
}
|
|
633
|
+
|
|
593
634
|
for (const lookup of lookups) {
|
|
594
635
|
recordedLookups.add(lookup.serializedRepresentation);
|
|
595
636
|
}
|
|
596
637
|
|
|
597
|
-
|
|
638
|
+
try {
|
|
639
|
+
const results = await checkpoint.base.getParameterSets(lookups, remainingBudget);
|
|
640
|
+
const numRows = results.reduce((a, b) => a + b.rows.length, 0);
|
|
641
|
+
|
|
642
|
+
lookupLog.push({ definition, resultsOrLimit: numRows, didExceedLimit: false });
|
|
643
|
+
remainingBudget -= numRows;
|
|
644
|
+
usedParameterResults += numRows;
|
|
645
|
+
return results;
|
|
646
|
+
} catch (e: unknown) {
|
|
647
|
+
if (e instanceof ParameterSetLimitExceededError) {
|
|
648
|
+
lookupLog.push({ definition, resultsOrLimit: remainingBudget, didExceedLimit: true });
|
|
649
|
+
throw new ParameterSetLimitExceededError(parameterLimit, lookupLog);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
throw e;
|
|
653
|
+
}
|
|
598
654
|
}
|
|
599
655
|
});
|
|
600
656
|
|
|
601
|
-
// Count parameter query results per definition (before deduplication)
|
|
602
|
-
parameterQueryResultsByDefinition = new Map<string, number>();
|
|
603
|
-
for (const bucket of dynamicBuckets) {
|
|
604
|
-
const count = parameterQueryResultsByDefinition.get(bucket.definition) ?? 0;
|
|
605
|
-
parameterQueryResultsByDefinition.set(bucket.definition, count + 1);
|
|
606
|
-
}
|
|
607
|
-
|
|
608
657
|
this.cachedDynamicBuckets = dynamicBuckets;
|
|
609
658
|
this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
|
|
610
659
|
this.lookupsFromPreviousCheckpoint = recordedLookups;
|
|
@@ -628,13 +677,13 @@ export class BucketParameterState {
|
|
|
628
677
|
buckets: allBuckets,
|
|
629
678
|
// We cannot track individual bucket updates for dynamic lookups yet
|
|
630
679
|
updatedBuckets: INVALIDATE_ALL_BUCKETS,
|
|
631
|
-
|
|
680
|
+
usedParameterResults
|
|
632
681
|
};
|
|
633
682
|
} else {
|
|
634
683
|
return {
|
|
635
684
|
buckets: allBuckets,
|
|
636
685
|
updatedBuckets: updatedBuckets,
|
|
637
|
-
|
|
686
|
+
usedParameterResults
|
|
638
687
|
};
|
|
639
688
|
}
|
|
640
689
|
}
|
|
@@ -668,18 +717,6 @@ export interface CheckpointLine {
|
|
|
668
717
|
// Use a more specific type to simplify testing
|
|
669
718
|
export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
|
|
670
719
|
|
|
671
|
-
/**
|
|
672
|
-
* Compute the total number of parameter query results across all definitions.
|
|
673
|
-
*/
|
|
674
|
-
function computeTotalParamResults(
|
|
675
|
-
parameterQueryResultsByDefinition: Map<string, number> | undefined
|
|
676
|
-
): number | undefined {
|
|
677
|
-
if (!parameterQueryResultsByDefinition) {
|
|
678
|
-
return undefined;
|
|
679
|
-
}
|
|
680
|
-
return Array.from(parameterQueryResultsByDefinition.values()).reduce((sum, count) => sum + count, 0);
|
|
681
|
-
}
|
|
682
|
-
|
|
683
720
|
/**
|
|
684
721
|
* Log a checkpoint message, enriching it with parameter query result counts if available.
|
|
685
722
|
*
|
|
@@ -701,20 +738,20 @@ function logCheckpoint(
|
|
|
701
738
|
}
|
|
702
739
|
|
|
703
740
|
/**
|
|
704
|
-
* Format a breakdown of
|
|
741
|
+
* Format a breakdown of dynamic bucket by sync stream definition.
|
|
705
742
|
*
|
|
706
743
|
* Sorts definitions by count (descending), includes the top 10, and returns both the
|
|
707
744
|
* formatted message string and the counts record suitable for structured log data.
|
|
708
745
|
*/
|
|
709
|
-
function
|
|
746
|
+
function formatBucketDefinitionBreakdown(bucketsByDefinition: Map<string, number>): {
|
|
710
747
|
message: string;
|
|
711
748
|
countsByDefinition: Record<string, number>;
|
|
712
749
|
} {
|
|
713
750
|
// Sort definitions by count (descending) and take top 10
|
|
714
|
-
const allSorted = Array.from(
|
|
751
|
+
const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
|
|
715
752
|
const sortedDefinitions = allSorted.slice(0, 10);
|
|
716
753
|
|
|
717
|
-
let message = '\
|
|
754
|
+
let message = '\Buckets by definition:';
|
|
718
755
|
const countsByDefinition: Record<string, number> = {};
|
|
719
756
|
for (const [definition, count] of sortedDefinitions) {
|
|
720
757
|
message += `\n ${definition}: ${count}`;
|
|
@@ -730,6 +767,33 @@ function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<st
|
|
|
730
767
|
return { message, countsByDefinition };
|
|
731
768
|
}
|
|
732
769
|
|
|
770
|
+
function formatParameterQueryBreakdown(log: storage.ParameterQueryInvocationLog[]) {
|
|
771
|
+
if (log.length == 0) {
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// When an exception about too many parameter results is thrown, the last entry is the one that finally exceeded the
|
|
776
|
+
// limit.
|
|
777
|
+
const results = Array.from(log);
|
|
778
|
+
const failure = results.pop()!;
|
|
779
|
+
|
|
780
|
+
let message = '\nInvoked parameter queries by definition:';
|
|
781
|
+
results.sort((a, b) => b.resultsOrLimit - a.resultsOrLimit);
|
|
782
|
+
const largestResults = results.splice(0, 9);
|
|
783
|
+
|
|
784
|
+
for (const entry of largestResults) {
|
|
785
|
+
message += `\n ${entry.definition}: ${entry.resultsOrLimit} results.`;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
if (largestResults.length < log.length) {
|
|
789
|
+
message += `\n ... and ${log.length - largestResults.length} more invocations`;
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
message += `\n ${failure.definition} exceeded the remaining limit of ${failure.resultsOrLimit} available results.`;
|
|
793
|
+
largestResults.push(failure);
|
|
794
|
+
return { message, largestResults };
|
|
795
|
+
}
|
|
796
|
+
|
|
733
797
|
function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
|
|
734
798
|
buckets = buckets.map((b) => {
|
|
735
799
|
if (typeof b != 'string') {
|
package/src/sync/sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
|
-
import { BucketPriority,
|
|
2
|
+
import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
3
|
|
|
4
4
|
import { AbortError } from 'ix/aborterror.js';
|
|
5
5
|
|
|
@@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr
|
|
|
17
17
|
export interface SyncStreamParameters {
|
|
18
18
|
syncContext: SyncContext;
|
|
19
19
|
bucketStorage: storage.SyncRulesBucketStorage;
|
|
20
|
-
syncRules:
|
|
20
|
+
syncRules: HydratedSyncConfig;
|
|
21
21
|
params: util.StreamingSyncRequest;
|
|
22
22
|
token: auth.JwtPayload;
|
|
23
23
|
logger?: Logger;
|
|
@@ -94,7 +94,7 @@ export async function* streamResponse(
|
|
|
94
94
|
async function* streamResponseInner(
|
|
95
95
|
syncContext: SyncContext,
|
|
96
96
|
bucketStorage: storage.SyncRulesBucketStorage,
|
|
97
|
-
syncRules:
|
|
97
|
+
syncRules: HydratedSyncConfig,
|
|
98
98
|
params: util.StreamingSyncRequest,
|
|
99
99
|
tokenPayload: auth.JwtPayload,
|
|
100
100
|
tracker: RequestTracker,
|
|
@@ -267,7 +267,7 @@ interface BucketDataRequest {
|
|
|
267
267
|
* This signal also fires when abort_connection fires.
|
|
268
268
|
*/
|
|
269
269
|
abort_batch: AbortSignal;
|
|
270
|
-
/** User id for debug purposes, not for sync
|
|
270
|
+
/** User id for debug purposes, not for sync config. */
|
|
271
271
|
userIdForLogs?: SqliteJsonValue;
|
|
272
272
|
forPriority: BucketPriority | null;
|
|
273
273
|
onRowsSent: (stats: OperationsSentStats) => void;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { traceWriter } from './TraceWriter.js';
|
|
2
|
+
|
|
3
|
+
export interface Span extends Disposable {
|
|
4
|
+
name: string;
|
|
5
|
+
/**
|
|
6
|
+
* Start time in microseconds since an arbitrary epoch.
|
|
7
|
+
*/
|
|
8
|
+
startAt: number;
|
|
9
|
+
/**
|
|
10
|
+
* End time in microseconds since an arbitrary epoch.
|
|
11
|
+
*/
|
|
12
|
+
endAt: number;
|
|
13
|
+
/**
|
|
14
|
+
* Time spent not in nested spans.
|
|
15
|
+
*/
|
|
16
|
+
selfDuration: number;
|
|
17
|
+
|
|
18
|
+
nestedSince: number | undefined;
|
|
19
|
+
subtrackFromSelf: number;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Durations spent in nested spans, in microseconds.
|
|
23
|
+
*/
|
|
24
|
+
nestedDurations: Record<string, number>;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Total duration of this span in milliseconds, rounded up. Only valid after the span has ended.
|
|
28
|
+
*/
|
|
29
|
+
durationMillis: number;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* End the span - same as [Symbol.dispose]().
|
|
33
|
+
*
|
|
34
|
+
* Safe to call multiple times. Any nested spans will automatically end as well.
|
|
35
|
+
*
|
|
36
|
+
* Returns an aggregate record of category -> "selfDuration", in microseconds.
|
|
37
|
+
*/
|
|
38
|
+
end(): Record<string, number>;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function now() {
|
|
42
|
+
return Number(process.hrtime.bigint() / 1000n);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
let nextThreadId = 1;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Lightweight tracing helper, with two main goals:
|
|
49
|
+
* 1. Generate aggregate timing info with low overhead.
|
|
50
|
+
* 2. Optional support for generating trace files during development.
|
|
51
|
+
*
|
|
52
|
+
* This is only intended for a single "thread" - concurrent operations on the same instance have undefined behavior.
|
|
53
|
+
* To trace concurrent operations, use separate instances of PerformanceTracer.
|
|
54
|
+
*
|
|
55
|
+
* Spans cannot be overlapping: If a parent span is ended, all nested spans are automatically ended.
|
|
56
|
+
*/
|
|
57
|
+
export class PerformanceTracer<K extends string> {
|
|
58
|
+
stack: Span[] = [];
|
|
59
|
+
threadId: number;
|
|
60
|
+
|
|
61
|
+
constructor(traceName: string) {
|
|
62
|
+
this.threadId = nextThreadId;
|
|
63
|
+
nextThreadId += 1;
|
|
64
|
+
traceWriter?.write({
|
|
65
|
+
ph: 'M',
|
|
66
|
+
cat: '__metadata',
|
|
67
|
+
name: 'thread_name',
|
|
68
|
+
pid: process.pid,
|
|
69
|
+
tid: this.threadId,
|
|
70
|
+
args: { name: `PowerSync ${traceName}` }
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Recommended usage:
|
|
76
|
+
*
|
|
77
|
+
* using _ = tracer.span('cat', 'details');
|
|
78
|
+
*
|
|
79
|
+
* The above automatically ends the span when it goes out of scope. Alternatively, call
|
|
80
|
+
* .end() on the span to end it earlier.
|
|
81
|
+
*
|
|
82
|
+
* @param category one of the defined categories
|
|
83
|
+
* @param subcat optional subcategory. Not used for calculating "self" durations in the aggregate API.
|
|
84
|
+
*/
|
|
85
|
+
span(category: K, subcat?: string): Span {
|
|
86
|
+
const stack = this.stack;
|
|
87
|
+
const index = this.stack.length;
|
|
88
|
+
const parent = this.stack[this.stack.length - 1];
|
|
89
|
+
const threadId = this.threadId;
|
|
90
|
+
const startAt = now();
|
|
91
|
+
if (parent != null) {
|
|
92
|
+
parent.nestedSince ??= startAt;
|
|
93
|
+
}
|
|
94
|
+
let name: string = category;
|
|
95
|
+
if (subcat) {
|
|
96
|
+
name += ':' + subcat;
|
|
97
|
+
}
|
|
98
|
+
const s: Span = {
|
|
99
|
+
name,
|
|
100
|
+
startAt: now(),
|
|
101
|
+
selfDuration: 0,
|
|
102
|
+
endAt: 0,
|
|
103
|
+
nestedSince: undefined,
|
|
104
|
+
subtrackFromSelf: 0,
|
|
105
|
+
nestedDurations: {},
|
|
106
|
+
end() {
|
|
107
|
+
if (this.endAt != 0) {
|
|
108
|
+
return this.nestedDurations;
|
|
109
|
+
}
|
|
110
|
+
while (stack.length - 1 > index) {
|
|
111
|
+
stack[stack.length - 1].end();
|
|
112
|
+
}
|
|
113
|
+
const endAt = now();
|
|
114
|
+
this.endAt = endAt;
|
|
115
|
+
const endTime = this.nestedSince ?? endAt;
|
|
116
|
+
this.selfDuration = endTime - startAt - this.subtrackFromSelf;
|
|
117
|
+
traceWriter?.write({
|
|
118
|
+
name,
|
|
119
|
+
cat: 'powersync',
|
|
120
|
+
ph: 'X',
|
|
121
|
+
ts: this.startAt,
|
|
122
|
+
dur: endAt - startAt,
|
|
123
|
+
pid: process.pid,
|
|
124
|
+
tid: threadId
|
|
125
|
+
});
|
|
126
|
+
stack.pop();
|
|
127
|
+
if (parent != null) {
|
|
128
|
+
parent.subtrackFromSelf += endAt - parent.nestedSince!;
|
|
129
|
+
for (let key in this.nestedDurations) {
|
|
130
|
+
parent.nestedDurations[key] = (parent.nestedDurations[key] ?? 0) + this.nestedDurations[key];
|
|
131
|
+
}
|
|
132
|
+
parent.nestedDurations[category] = (parent.nestedDurations[category] ?? 0) + this.selfDuration;
|
|
133
|
+
parent.nestedSince = undefined;
|
|
134
|
+
}
|
|
135
|
+
return this.nestedDurations;
|
|
136
|
+
},
|
|
137
|
+
|
|
138
|
+
get durationMillis() {
|
|
139
|
+
return Math.ceil((this.endAt - this.startAt) / 1000);
|
|
140
|
+
},
|
|
141
|
+
[Symbol.dispose]() {
|
|
142
|
+
this.end();
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
this.stack.push(s);
|
|
146
|
+
|
|
147
|
+
return s;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Mutex } from 'async-mutex';
|
|
2
|
+
import * as fs from 'node:fs/promises';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Write traces in the Chrome JSON Trace Format.
|
|
6
|
+
*
|
|
7
|
+
* View at https://ui.perfetto.dev/
|
|
8
|
+
*/
|
|
9
|
+
class TraceWriter {
|
|
10
|
+
handle: fs.FileHandle | null = null;
|
|
11
|
+
length = 0;
|
|
12
|
+
queue: any[] = [];
|
|
13
|
+
private mutex = new Mutex();
|
|
14
|
+
|
|
15
|
+
constructor(public readonly path: string) {
|
|
16
|
+
this.open().catch((e) => {
|
|
17
|
+
console.error(`Failed to open trace file at ${path}`, e);
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async open() {
|
|
22
|
+
await this.mutex.runExclusive(async () => {
|
|
23
|
+
this.handle = await fs.open(this.path, 'w+');
|
|
24
|
+
this.handle.truncate(0);
|
|
25
|
+
await this.handle.write('[]');
|
|
26
|
+
this.length = 2;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
write(...traceEvents: any[]) {
|
|
31
|
+
this.writeAsync(...traceEvents).catch((e) => {
|
|
32
|
+
console.error(`Failed to write trace file`, e);
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async writeAsync(...traceEvents: any[]) {
|
|
37
|
+
this.queue.push(...traceEvents);
|
|
38
|
+
await this.mutex.runExclusive(async () => {
|
|
39
|
+
if (this.queue.length > 0) {
|
|
40
|
+
// Write queued events.
|
|
41
|
+
// After each write, we end the file as a valid JSON array.
|
|
42
|
+
// On the next write, we overwrite the last character to extend the array.
|
|
43
|
+
const buffer = Buffer.from(JSON.stringify(this.queue));
|
|
44
|
+
await this.handle?.write(buffer, 1, buffer.length - 1, this.length - 1);
|
|
45
|
+
this.queue = [];
|
|
46
|
+
this.length += buffer.length - 2;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const traceFile = process.env.POWERSYNC_TRACE_FILE;
|
|
53
|
+
/**
|
|
54
|
+
* traceWriter, only present if POWERSYNC_TRACE_FILE env var is configured.
|
|
55
|
+
*/
|
|
56
|
+
export const traceWriter = traceFile ? new TraceWriter(traceFile) : null;
|
|
57
|
+
|
|
58
|
+
if (traceWriter) {
|
|
59
|
+
traceWriter.write({
|
|
60
|
+
ph: 'M',
|
|
61
|
+
cat: '__metadata',
|
|
62
|
+
name: 'process_name',
|
|
63
|
+
pid: process.pid,
|
|
64
|
+
tid: 1000,
|
|
65
|
+
args: { name: 'powersync' }
|
|
66
|
+
});
|
|
67
|
+
}
|
|
@@ -13,7 +13,7 @@ export async function createWriteCheckpoint(options: CreateWriteCheckpointOption
|
|
|
13
13
|
|
|
14
14
|
const syncBucketStorage = await options.storage.getActiveStorage();
|
|
15
15
|
if (!syncBucketStorage) {
|
|
16
|
-
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync
|
|
16
|
+
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
|
|
@@ -26,7 +26,7 @@ export type CompoundConfigCollectorOptions = {
|
|
|
26
26
|
*/
|
|
27
27
|
configCollectors: ConfigCollector[];
|
|
28
28
|
/**
|
|
29
|
-
* Collectors for PowerSync sync
|
|
29
|
+
* Collectors for PowerSync sync config content.
|
|
30
30
|
* The configuration from first collector to provide a configuration
|
|
31
31
|
* is used. The order of the collectors specifies precedence
|
|
32
32
|
*/
|
|
@@ -236,11 +236,11 @@ export class CompoundConfigCollector {
|
|
|
236
236
|
return config;
|
|
237
237
|
}
|
|
238
238
|
logger.debug(
|
|
239
|
-
`Could not collect sync
|
|
239
|
+
`Could not collect sync config with ${collector.name} method. Moving on to next method if available.`
|
|
240
240
|
);
|
|
241
241
|
} catch (ex) {
|
|
242
242
|
// An error in a collector is a hard stop
|
|
243
|
-
throw new Error(`Could not collect sync
|
|
243
|
+
throw new Error(`Could not collect sync config using ${collector.name} method. Caught exception: ${ex}`);
|
|
244
244
|
}
|
|
245
245
|
}
|
|
246
246
|
return {
|
|
@@ -16,7 +16,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
|
|
|
16
16
|
|
|
17
17
|
const { config_path } = runnerConfig;
|
|
18
18
|
|
|
19
|
-
// Depending on the container, the sync
|
|
19
|
+
// Depending on the container, the sync config may not actually be present.
|
|
20
20
|
// Only persist the path here, and load on demand using `loadSyncRules()`.
|
|
21
21
|
return {
|
|
22
22
|
present: true,
|