@powersync/service-core 0.0.0-dev-20260313100403 → 0.0.0-dev-20260515144844
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -7
- package/dist/api/RouteAPI.d.ts +17 -3
- package/dist/api/api-index.d.ts +1 -1
- package/dist/api/api-index.js +1 -1
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +1 -1
- package/dist/api/diagnostics.js +32 -14
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/CachedKeyCollector.js +1 -1
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.d.ts +1 -1
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/entry/commands/compact-action.js +26 -5
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -1
- package/dist/entry/entry-index.js +1 -1
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/events/EventsEngine.js +1 -1
- package/dist/events/EventsEngine.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/metrics/MetricsEngine.d.ts +1 -1
- package/dist/metrics/RollingBucketMax.d.ts +28 -0
- package/dist/metrics/RollingBucketMax.js +80 -0
- package/dist/metrics/RollingBucketMax.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +3 -2
- package/dist/metrics/metrics-index.js +3 -2
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +1 -1
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/metrics/register-metrics.js +2 -2
- package/dist/metrics/register-metrics.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +2 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/modules-index.d.ts +1 -1
- package/dist/modules/modules-index.js +1 -1
- package/dist/modules/modules-index.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -2
- package/dist/replication/AbstractReplicationJob.js +1 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +7 -7
- package/dist/replication/AbstractReplicator.js +31 -28
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationLagTracker.d.ts +50 -0
- package/dist/replication/ReplicationLagTracker.js +78 -0
- package/dist/replication/ReplicationLagTracker.js.map +1 -0
- package/dist/replication/replication-index.d.ts +3 -2
- package/dist/replication/replication-index.js +3 -2
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/replication/replication-metrics.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +59 -32
- package/dist/routes/endpoints/admin.d.ts +108 -54
- package/dist/routes/endpoints/admin.js +7 -3
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +10 -10
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -10
- package/dist/routes/endpoints/sync-stream.js +2 -2
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.js +1 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +4 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +9 -9
- package/dist/storage/BucketStorage.js +9 -9
- package/dist/storage/BucketStorageBatch.d.ts +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +27 -20
- package/dist/storage/BucketStorageFactory.js +19 -16
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
- package/dist/storage/PersistedSyncRulesContent.js +24 -5
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/ReportStorage.d.ts +3 -3
- package/dist/storage/SourceTable.d.ts +4 -4
- package/dist/storage/SourceTable.js +3 -3
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +1 -1
- package/dist/storage/StorageVersionConfig.js +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
- package/dist/storage/SyncRulesBucketStorage.js +14 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
- package/dist/storage/WriteCheckpointAPI.js +1 -1
- package/dist/storage/bson.d.ts +0 -1
- package/dist/storage/bson.js +0 -4
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +8 -8
- package/dist/storage/storage-index.js +8 -8
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/streams/streams-index.d.ts +2 -2
- package/dist/streams/streams-index.js +2 -2
- package/dist/streams/streams-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +2 -5
- package/dist/sync/BucketChecksumState.js +119 -75
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.js +1 -1
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +2 -2
- package/dist/sync/sync-index.js +2 -2
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +1 -1
- package/dist/system/ServiceContext.js +1 -1
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +44 -0
- package/dist/tracing/PerformanceTracer.js +102 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -0
- package/dist/tracing/TraceWriter.d.ts +22 -0
- package/dist/tracing/TraceWriter.js +63 -0
- package/dist/tracing/TraceWriter.js.map +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +1 -1
- package/dist/util/config/compound-config-collector.js +2 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config.js +1 -1
- package/dist/util/config.js.map +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/errors.d.ts +3 -0
- package/dist/util/errors.js +15 -0
- package/dist/util/errors.js.map +1 -0
- package/dist/util/protocol-types.d.ts +3 -3
- package/dist/util/protocol-types.js +1 -1
- package/dist/util/util-index.d.ts +1 -1
- package/dist/util/util-index.js +1 -1
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +20 -3
- package/src/api/api-index.ts +1 -1
- package/src/api/api-metrics.ts +1 -1
- package/src/api/diagnostics.ts +42 -20
- package/src/auth/CachedKeyCollector.ts +2 -3
- package/src/auth/CompoundKeyCollector.ts +2 -3
- package/src/auth/KeyStore.ts +1 -1
- package/src/auth/RemoteJWKSCollector.ts +0 -1
- package/src/auth/StaticKeyCollector.ts +1 -1
- package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
- package/src/entry/commands/compact-action.ts +29 -5
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -1
- package/src/events/EventsEngine.ts +1 -1
- package/src/index.ts +2 -0
- package/src/metrics/MetricsEngine.ts +1 -1
- package/src/metrics/RollingBucketMax.ts +109 -0
- package/src/metrics/metrics-index.ts +3 -2
- package/src/metrics/open-telemetry/util.ts +1 -1
- package/src/metrics/register-metrics.ts +3 -3
- package/src/modules/AbstractModule.ts +2 -2
- package/src/modules/modules-index.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +3 -3
- package/src/replication/AbstractReplicator.ts +32 -30
- package/src/replication/ReplicationLagTracker.ts +86 -0
- package/src/replication/replication-index.ts +3 -2
- package/src/replication/replication-metrics.ts +1 -1
- package/src/routes/endpoints/admin.ts +7 -3
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +10 -12
- package/src/routes/endpoints/sync-stream.ts +2 -2
- package/src/routes/hooks.ts +2 -2
- package/src/routes/route-register.ts +2 -10
- package/src/runner/teardown.ts +4 -4
- package/src/storage/BucketStorage.ts +9 -9
- package/src/storage/BucketStorageBatch.ts +1 -1
- package/src/storage/BucketStorageFactory.ts +45 -34
- package/src/storage/ChecksumCache.ts +1 -1
- package/src/storage/PersistedSyncRulesContent.ts +30 -6
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/ReportStorage.ts +3 -3
- package/src/storage/SourceTable.ts +4 -4
- package/src/storage/StorageVersionConfig.ts +1 -1
- package/src/storage/SyncRulesBucketStorage.ts +46 -7
- package/src/storage/WriteCheckpointAPI.ts +6 -6
- package/src/storage/bson.ts +0 -5
- package/src/storage/storage-index.ts +8 -8
- package/src/storage/storage-metrics.ts +2 -2
- package/src/streams/streams-index.ts +2 -2
- package/src/sync/BucketChecksumState.ts +141 -93
- package/src/sync/RequestTracker.ts +1 -1
- package/src/sync/sync-index.ts +2 -2
- package/src/sync/sync.ts +2 -8
- package/src/sync/util.ts +1 -1
- package/src/system/ServiceContext.ts +1 -1
- package/src/tracing/PerformanceTracer.ts +126 -0
- package/src/tracing/TraceWriter.ts +67 -0
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
- package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
- package/src/util/config/compound-config-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
- package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
- package/src/util/config.ts +1 -1
- package/src/util/env.ts +1 -1
- package/src/util/errors.ts +21 -0
- package/src/util/protocol-types.ts +1 -1
- package/src/util/util-index.ts +1 -1
- package/src/util/utils.ts +1 -1
- package/test/src/ReplicationLagTracker.test.ts +53 -0
- package/test/src/RollingBucketMax.test.ts +106 -0
- package/test/src/auth.test.ts +115 -7
- package/test/src/diagnostics.test.ts +151 -0
- package/test/src/module-loader.test.ts +1 -1
- package/test/src/routes/mocks.ts +1 -1
- package/test/src/routes/stream.test.ts +1 -2
- package/test/src/sync/BucketChecksumState.test.ts +223 -67
- package/test/src/util/protocol_types.test.ts +1 -1
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +1 -1
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
|
-
BucketDescription,
|
|
3
2
|
BucketParameterQuerier,
|
|
4
3
|
BucketPriority,
|
|
5
4
|
BucketSource,
|
|
6
5
|
HydratedSyncRules,
|
|
6
|
+
mergeBuckets,
|
|
7
7
|
QuerierError,
|
|
8
8
|
RequestedStream,
|
|
9
9
|
RequestParameters,
|
|
10
|
-
ResolvedBucket
|
|
11
|
-
mergeBuckets
|
|
10
|
+
ResolvedBucket
|
|
12
11
|
} from '@powersync/service-sync-rules';
|
|
13
12
|
|
|
14
13
|
import * as storage from '../storage/storage-index.js';
|
|
@@ -22,6 +21,7 @@ import {
|
|
|
22
21
|
ServiceError
|
|
23
22
|
} from '@powersync/lib-services-framework';
|
|
24
23
|
import { JwtPayload } from '../auth/JwtPayload.js';
|
|
24
|
+
import { ParameterSetLimitExceededError } from '../storage/storage-index.js';
|
|
25
25
|
import { SyncContext } from './SyncContext.js';
|
|
26
26
|
import { getIntersection, hasIntersection } from './util.js';
|
|
27
27
|
|
|
@@ -119,16 +119,38 @@ export class BucketChecksumState {
|
|
|
119
119
|
const storage = this.bucketStorage;
|
|
120
120
|
|
|
121
121
|
const update = await this.parameterState.getCheckpointUpdate(next);
|
|
122
|
-
const { buckets: allBuckets, updatedBuckets,
|
|
122
|
+
const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
|
|
123
123
|
|
|
124
124
|
/** Set of all buckets in this checkpoint. */
|
|
125
125
|
const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
|
|
126
126
|
|
|
127
127
|
if (bucketDescriptionMap.size > this.context.maxBuckets) {
|
|
128
|
-
|
|
128
|
+
const error = new ServiceError(
|
|
129
129
|
ErrorCode.PSYNC_S2305,
|
|
130
130
|
`Too many buckets: ${bucketDescriptionMap.size} (limit of ${this.context.maxBuckets})`
|
|
131
131
|
);
|
|
132
|
+
|
|
133
|
+
let errorMessage = error.message;
|
|
134
|
+
const logData: any = {
|
|
135
|
+
checkpoint: next.base.checkpoint,
|
|
136
|
+
user_id: this.parameterState.syncParams.userId,
|
|
137
|
+
buckets: allBuckets.length
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
// Count buckets per definition.
|
|
141
|
+
const bucketsByDefinition = new Map<string, number>();
|
|
142
|
+
for (const bucket of bucketDescriptionMap.values()) {
|
|
143
|
+
const definition = bucket.definition;
|
|
144
|
+
const count = bucketsByDefinition.get(definition) ?? 0;
|
|
145
|
+
bucketsByDefinition.set(definition, count + 1);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition);
|
|
149
|
+
errorMessage += breakdown.message;
|
|
150
|
+
logData.buckets_by_definition = breakdown.countsByDefinition;
|
|
151
|
+
|
|
152
|
+
this.logger.error(errorMessage, logData);
|
|
153
|
+
throw error;
|
|
132
154
|
}
|
|
133
155
|
|
|
134
156
|
let checksumMap: util.ChecksumMap;
|
|
@@ -212,13 +234,10 @@ export class BucketChecksumState {
|
|
|
212
234
|
});
|
|
213
235
|
|
|
214
236
|
deferredLog = () => {
|
|
215
|
-
const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
|
|
216
237
|
let message = `Updated checkpoint: ${base.checkpoint} | `;
|
|
217
238
|
message += `write: ${writeCheckpoint} | `;
|
|
218
239
|
message += `buckets: ${allBuckets.length} | `;
|
|
219
|
-
|
|
220
|
-
message += `param_results: ${totalParamResults} | `;
|
|
221
|
-
}
|
|
240
|
+
message += `param_results: ${usedParameterResults} | `;
|
|
222
241
|
message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
|
|
223
242
|
message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
|
|
224
243
|
logCheckpoint(
|
|
@@ -231,7 +250,7 @@ export class BucketChecksumState {
|
|
|
231
250
|
updated: diff.updatedBuckets.length,
|
|
232
251
|
removed: diff.removedBuckets.length
|
|
233
252
|
},
|
|
234
|
-
|
|
253
|
+
usedParameterResults
|
|
235
254
|
);
|
|
236
255
|
};
|
|
237
256
|
|
|
@@ -245,12 +264,9 @@ export class BucketChecksumState {
|
|
|
245
264
|
} satisfies util.StreamingSyncCheckpointDiff;
|
|
246
265
|
} else {
|
|
247
266
|
deferredLog = () => {
|
|
248
|
-
const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
|
|
249
267
|
let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
|
|
250
268
|
message += `buckets: ${allBuckets.length}`;
|
|
251
|
-
|
|
252
|
-
message += ` | param_results: ${totalParamResults}`;
|
|
253
|
-
}
|
|
269
|
+
message += ` | param_results: ${usedParameterResults}`;
|
|
254
270
|
message += ` ${limitedBuckets(allBuckets, 20)}`;
|
|
255
271
|
logCheckpoint(
|
|
256
272
|
this.logger,
|
|
@@ -260,7 +276,7 @@ export class BucketChecksumState {
|
|
|
260
276
|
user_id: userIdForLogs,
|
|
261
277
|
buckets: allBuckets.length
|
|
262
278
|
},
|
|
263
|
-
|
|
279
|
+
usedParameterResults
|
|
264
280
|
);
|
|
265
281
|
};
|
|
266
282
|
bucketsToFetch = allBuckets;
|
|
@@ -393,11 +409,8 @@ export interface CheckpointUpdate {
|
|
|
393
409
|
*/
|
|
394
410
|
updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
|
|
395
411
|
|
|
396
|
-
/**
|
|
397
|
-
|
|
398
|
-
* Map from definition name to count.
|
|
399
|
-
*/
|
|
400
|
-
parameterQueryResultsByDefinition?: Map<string, number>;
|
|
412
|
+
/** The amount of rows fetched from parameters indexes. */
|
|
413
|
+
usedParameterResults: number;
|
|
401
414
|
}
|
|
402
415
|
|
|
403
416
|
export class BucketParameterState {
|
|
@@ -445,6 +458,7 @@ export class BucketParameterState {
|
|
|
445
458
|
const subscription = explicitStreamSubscriptions[i];
|
|
446
459
|
|
|
447
460
|
const syncRuleStream: RequestedStream = {
|
|
461
|
+
priorityOverride: subscription.override_priority as BucketPriority | null,
|
|
448
462
|
parameters: subscription.parameters ?? {},
|
|
449
463
|
opaque_id: i
|
|
450
464
|
};
|
|
@@ -480,25 +494,9 @@ export class BucketParameterState {
|
|
|
480
494
|
* reference default buckets by their stream index instead of duplicating the name on wire.
|
|
481
495
|
*/
|
|
482
496
|
translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map<string, number>): util.ClientBucketDescription {
|
|
483
|
-
// If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that
|
|
484
|
-
// priority.
|
|
485
|
-
let priorityOverride: BucketPriority | null = null;
|
|
486
|
-
for (const reason of description.inclusion_reasons) {
|
|
487
|
-
if (reason != 'default') {
|
|
488
|
-
const requestedPriority = this.explicitStreamSubscriptions[reason.subscription]?.override_priority;
|
|
489
|
-
if (requestedPriority != null) {
|
|
490
|
-
if (priorityOverride == null) {
|
|
491
|
-
priorityOverride = requestedPriority as BucketPriority;
|
|
492
|
-
} else {
|
|
493
|
-
priorityOverride = Math.min(requestedPriority, priorityOverride) as BucketPriority;
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
|
|
499
497
|
return {
|
|
500
498
|
bucket: description.bucket,
|
|
501
|
-
priority:
|
|
499
|
+
priority: description.priority,
|
|
502
500
|
subscriptions: description.inclusion_reasons.map((reason) => {
|
|
503
501
|
if (reason == 'default') {
|
|
504
502
|
const stream = description.definition;
|
|
@@ -518,36 +516,41 @@ export class BucketParameterState {
|
|
|
518
516
|
const querier = this.querier;
|
|
519
517
|
let update: CheckpointUpdate;
|
|
520
518
|
if (querier.hasDynamicBuckets) {
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
519
|
+
try {
|
|
520
|
+
update = await this.getCheckpointUpdateDynamic(checkpoint);
|
|
521
|
+
} catch (e: unknown) {
|
|
522
|
+
if (e instanceof ParameterSetLimitExceededError) {
|
|
523
|
+
// Too many parameter results, create a breakdown of which streams are responsible for the most queries and
|
|
524
|
+
// then abort.
|
|
525
|
+
const error = new ServiceError(
|
|
526
|
+
ErrorCode.PSYNC_S2305,
|
|
527
|
+
`Too many parameter query results (limit of ${this.context.maxParameterQueryResults})`
|
|
528
|
+
);
|
|
529
|
+
|
|
530
|
+
let errorMessage = error.message;
|
|
531
|
+
const logData: any = {
|
|
532
|
+
checkpoint: checkpoint.base.checkpoint,
|
|
533
|
+
user_id: this.syncParams.userId
|
|
534
|
+
};
|
|
535
|
+
|
|
536
|
+
if (e.breakdown) {
|
|
537
|
+
const breakdown = formatParameterQueryBreakdown(e.breakdown);
|
|
538
|
+
if (breakdown) {
|
|
539
|
+
errorMessage += breakdown.message;
|
|
540
|
+
logData.parameterResults = breakdown.largestResults;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
533
543
|
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
user_id: this.syncParams.userId,
|
|
538
|
-
parameter_query_results: update.buckets.length
|
|
539
|
-
};
|
|
544
|
+
this.logger.error(errorMessage, logData);
|
|
545
|
+
throw error;
|
|
546
|
+
}
|
|
540
547
|
|
|
541
|
-
|
|
542
|
-
const breakdown = formatParameterQueryBreakdown(update.parameterQueryResultsByDefinition);
|
|
543
|
-
errorMessage += breakdown.message;
|
|
544
|
-
logData.parameter_query_results_by_definition = breakdown.countsByDefinition;
|
|
548
|
+
throw e;
|
|
545
549
|
}
|
|
546
|
-
|
|
547
|
-
this.
|
|
548
|
-
|
|
549
|
-
throw error;
|
|
550
|
+
} else {
|
|
551
|
+
update = await this.getCheckpointUpdateStatic(checkpoint);
|
|
550
552
|
}
|
|
553
|
+
|
|
551
554
|
return update;
|
|
552
555
|
}
|
|
553
556
|
|
|
@@ -561,14 +564,16 @@ export class BucketParameterState {
|
|
|
561
564
|
if (update.invalidateDataBuckets) {
|
|
562
565
|
return {
|
|
563
566
|
buckets: staticBuckets,
|
|
564
|
-
updatedBuckets: INVALIDATE_ALL_BUCKETS
|
|
567
|
+
updatedBuckets: INVALIDATE_ALL_BUCKETS,
|
|
568
|
+
usedParameterResults: 0
|
|
565
569
|
};
|
|
566
570
|
}
|
|
567
571
|
|
|
568
572
|
const updatedBuckets = new Set<string>(getIntersection(this.staticBuckets, update.updatedDataBuckets));
|
|
569
573
|
return {
|
|
570
574
|
buckets: staticBuckets,
|
|
571
|
-
updatedBuckets
|
|
575
|
+
updatedBuckets,
|
|
576
|
+
usedParameterResults: 0
|
|
572
577
|
};
|
|
573
578
|
}
|
|
574
579
|
|
|
@@ -600,27 +605,55 @@ export class BucketParameterState {
|
|
|
600
605
|
}
|
|
601
606
|
|
|
602
607
|
let dynamicBuckets: ResolvedBucket[];
|
|
603
|
-
let
|
|
608
|
+
let usedParameterResults = 0;
|
|
604
609
|
if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
|
|
605
610
|
const recordedLookups = new Set<string>();
|
|
611
|
+
const parameterLimit = this.context.maxParameterQueryResults;
|
|
612
|
+
let remainingBudget = parameterLimit;
|
|
613
|
+
|
|
614
|
+
// Log of lookups to provide a breakdown if we exceed the dynamic lookup limit.
|
|
615
|
+
// FIXME: This is horrible, Sync Streams will invoke the callback concurrently and we can't properly deal with
|
|
616
|
+
// that. We should replace queriers for Sync Streams with an explicit graph structure based on sync plans instead
|
|
617
|
+
// of adding these checks to the imperative querier interface.
|
|
618
|
+
const lookupLog: storage.ParameterQueryInvocationLog[] = [];
|
|
606
619
|
|
|
607
620
|
dynamicBuckets = await querier.queryDynamicBucketDescriptions({
|
|
608
|
-
getParameterSets(lookups) {
|
|
621
|
+
getParameterSets: async (lookups, definition) => {
|
|
622
|
+
if (lookups.length > parameterLimit) {
|
|
623
|
+
// Sync Streams can chain parameter lookups, so a large output from an earlier call may become the input
|
|
624
|
+
// here. We reuse the output limit as a generous upper bound; legitimate queries are much smaller.
|
|
625
|
+
const msg = `Attempted to fetch ${lookups.length} lookups at once, a maximum of ${parameterLimit} lookups are allowed.`;
|
|
626
|
+
this.logger.error(msg, {
|
|
627
|
+
user_id: this.syncParams.userId,
|
|
628
|
+
checkpoint: checkpoint.base.checkpoint,
|
|
629
|
+
cause: definition
|
|
630
|
+
});
|
|
631
|
+
throw new ServiceError(ErrorCode.PSYNC_S2305, msg);
|
|
632
|
+
}
|
|
633
|
+
|
|
609
634
|
for (const lookup of lookups) {
|
|
610
635
|
recordedLookups.add(lookup.serializedRepresentation);
|
|
611
636
|
}
|
|
612
637
|
|
|
613
|
-
|
|
638
|
+
try {
|
|
639
|
+
const results = await checkpoint.base.getParameterSets(lookups, remainingBudget);
|
|
640
|
+
const numRows = results.reduce((a, b) => a + b.rows.length, 0);
|
|
641
|
+
|
|
642
|
+
lookupLog.push({ definition, resultsOrLimit: numRows, didExceedLimit: false });
|
|
643
|
+
remainingBudget -= numRows;
|
|
644
|
+
usedParameterResults += numRows;
|
|
645
|
+
return results;
|
|
646
|
+
} catch (e: unknown) {
|
|
647
|
+
if (e instanceof ParameterSetLimitExceededError) {
|
|
648
|
+
lookupLog.push({ definition, resultsOrLimit: remainingBudget, didExceedLimit: true });
|
|
649
|
+
throw new ParameterSetLimitExceededError(parameterLimit, lookupLog);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
throw e;
|
|
653
|
+
}
|
|
614
654
|
}
|
|
615
655
|
});
|
|
616
656
|
|
|
617
|
-
// Count parameter query results per definition (before deduplication)
|
|
618
|
-
parameterQueryResultsByDefinition = new Map<string, number>();
|
|
619
|
-
for (const bucket of dynamicBuckets) {
|
|
620
|
-
const count = parameterQueryResultsByDefinition.get(bucket.definition) ?? 0;
|
|
621
|
-
parameterQueryResultsByDefinition.set(bucket.definition, count + 1);
|
|
622
|
-
}
|
|
623
|
-
|
|
624
657
|
this.cachedDynamicBuckets = dynamicBuckets;
|
|
625
658
|
this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
|
|
626
659
|
this.lookupsFromPreviousCheckpoint = recordedLookups;
|
|
@@ -644,13 +677,13 @@ export class BucketParameterState {
|
|
|
644
677
|
buckets: allBuckets,
|
|
645
678
|
// We cannot track individual bucket updates for dynamic lookups yet
|
|
646
679
|
updatedBuckets: INVALIDATE_ALL_BUCKETS,
|
|
647
|
-
|
|
680
|
+
usedParameterResults
|
|
648
681
|
};
|
|
649
682
|
} else {
|
|
650
683
|
return {
|
|
651
684
|
buckets: allBuckets,
|
|
652
685
|
updatedBuckets: updatedBuckets,
|
|
653
|
-
|
|
686
|
+
usedParameterResults
|
|
654
687
|
};
|
|
655
688
|
}
|
|
656
689
|
}
|
|
@@ -684,18 +717,6 @@ export interface CheckpointLine {
|
|
|
684
717
|
// Use a more specific type to simplify testing
|
|
685
718
|
export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
|
|
686
719
|
|
|
687
|
-
/**
|
|
688
|
-
* Compute the total number of parameter query results across all definitions.
|
|
689
|
-
*/
|
|
690
|
-
function computeTotalParamResults(
|
|
691
|
-
parameterQueryResultsByDefinition: Map<string, number> | undefined
|
|
692
|
-
): number | undefined {
|
|
693
|
-
if (!parameterQueryResultsByDefinition) {
|
|
694
|
-
return undefined;
|
|
695
|
-
}
|
|
696
|
-
return Array.from(parameterQueryResultsByDefinition.values()).reduce((sum, count) => sum + count, 0);
|
|
697
|
-
}
|
|
698
|
-
|
|
699
720
|
/**
|
|
700
721
|
* Log a checkpoint message, enriching it with parameter query result counts if available.
|
|
701
722
|
*
|
|
@@ -717,20 +738,20 @@ function logCheckpoint(
|
|
|
717
738
|
}
|
|
718
739
|
|
|
719
740
|
/**
|
|
720
|
-
* Format a breakdown of
|
|
741
|
+
* Format a breakdown of dynamic bucket by sync stream definition.
|
|
721
742
|
*
|
|
722
743
|
* Sorts definitions by count (descending), includes the top 10, and returns both the
|
|
723
744
|
* formatted message string and the counts record suitable for structured log data.
|
|
724
745
|
*/
|
|
725
|
-
function
|
|
746
|
+
function formatBucketDefinitionBreakdown(bucketsByDefinition: Map<string, number>): {
|
|
726
747
|
message: string;
|
|
727
748
|
countsByDefinition: Record<string, number>;
|
|
728
749
|
} {
|
|
729
750
|
// Sort definitions by count (descending) and take top 10
|
|
730
|
-
const allSorted = Array.from(
|
|
751
|
+
const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
|
|
731
752
|
const sortedDefinitions = allSorted.slice(0, 10);
|
|
732
753
|
|
|
733
|
-
let message = '\
|
|
754
|
+
let message = '\Buckets by definition:';
|
|
734
755
|
const countsByDefinition: Record<string, number> = {};
|
|
735
756
|
for (const [definition, count] of sortedDefinitions) {
|
|
736
757
|
message += `\n ${definition}: ${count}`;
|
|
@@ -746,6 +767,33 @@ function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<st
|
|
|
746
767
|
return { message, countsByDefinition };
|
|
747
768
|
}
|
|
748
769
|
|
|
770
|
+
function formatParameterQueryBreakdown(log: storage.ParameterQueryInvocationLog[]) {
|
|
771
|
+
if (log.length == 0) {
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// When an exception about too many parameter results is thrown, the last entry is the one that finally exceeded the
|
|
776
|
+
// limit.
|
|
777
|
+
const results = Array.from(log);
|
|
778
|
+
const failure = results.pop()!;
|
|
779
|
+
|
|
780
|
+
let message = '\nInvoked parameter queries by definition:';
|
|
781
|
+
results.sort((a, b) => b.resultsOrLimit - a.resultsOrLimit);
|
|
782
|
+
const largestResults = results.splice(0, 9);
|
|
783
|
+
|
|
784
|
+
for (const entry of largestResults) {
|
|
785
|
+
message += `\n ${entry.definition}: ${entry.resultsOrLimit} results.`;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
if (largestResults.length < log.length) {
|
|
789
|
+
message += `\n ... and ${log.length - largestResults.length} more invocations`;
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
message += `\n ${failure.definition} exceeded the remaining limit of ${failure.resultsOrLimit} available results.`;
|
|
793
|
+
largestResults.push(failure);
|
|
794
|
+
return { message, largestResults };
|
|
795
|
+
}
|
|
796
|
+
|
|
749
797
|
function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
|
|
750
798
|
buckets = buckets.map((b) => {
|
|
751
799
|
if (typeof b != 'string') {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { MetricsEngine } from '../metrics/MetricsEngine.js';
|
|
2
2
|
|
|
3
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
4
|
import { APIMetric } from '@powersync/service-types';
|
|
4
5
|
import { SyncBucketData } from '../util/protocol-types.js';
|
|
5
|
-
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Record sync stats per request stream.
|
package/src/sync/sync-index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
export * from './BucketChecksumState.js';
|
|
1
2
|
export * from './RequestTracker.js';
|
|
2
3
|
export * from './safeRace.js';
|
|
3
4
|
export * from './sync.js';
|
|
4
|
-
export * from './util.js';
|
|
5
|
-
export * from './BucketChecksumState.js';
|
|
6
5
|
export * from './SyncContext.js';
|
|
6
|
+
export * from './util.js';
|
package/src/sync/sync.ts
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
|
-
import {
|
|
3
|
-
BucketDescription,
|
|
4
|
-
BucketPriority,
|
|
5
|
-
HydratedSyncRules,
|
|
6
|
-
ResolvedBucket,
|
|
7
|
-
SqliteJsonValue
|
|
8
|
-
} from '@powersync/service-sync-rules';
|
|
2
|
+
import { BucketPriority, HydratedSyncRules, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
9
3
|
|
|
10
4
|
import { AbortError } from 'ix/aborterror.js';
|
|
11
5
|
|
|
@@ -273,7 +267,7 @@ interface BucketDataRequest {
|
|
|
273
267
|
* This signal also fires when abort_connection fires.
|
|
274
268
|
*/
|
|
275
269
|
abort_batch: AbortSignal;
|
|
276
|
-
/** User id for debug purposes, not for sync
|
|
270
|
+
/** User id for debug purposes, not for sync config. */
|
|
277
271
|
userIdForLogs?: SqliteJsonValue;
|
|
278
272
|
forPriority: BucketPriority | null;
|
|
279
273
|
onRowsSent: (stats: OperationsSentStats) => void;
|
package/src/sync/util.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as timers from 'timers/promises';
|
|
2
2
|
|
|
3
3
|
import { SemaphoreInterface } from 'async-mutex';
|
|
4
|
+
import { serialize } from 'bson';
|
|
4
5
|
import * as util from '../util/util-index.js';
|
|
5
6
|
import { RequestTracker } from './RequestTracker.js';
|
|
6
|
-
import { serialize } from 'bson';
|
|
7
7
|
|
|
8
8
|
export type TokenStreamOptions = {
|
|
9
9
|
/**
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
|
|
2
2
|
|
|
3
|
+
import { EventsEngine } from '../events/EventsEngine.js';
|
|
3
4
|
import { framework } from '../index.js';
|
|
4
5
|
import * as metrics from '../metrics/MetricsEngine.js';
|
|
5
6
|
import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
|
|
@@ -8,7 +9,6 @@ import * as routes from '../routes/routes-index.js';
|
|
|
8
9
|
import * as storage from '../storage/storage-index.js';
|
|
9
10
|
import { SyncContext } from '../sync/SyncContext.js';
|
|
10
11
|
import * as utils from '../util/util-index.js';
|
|
11
|
-
import { EventsEngine } from '../events/EventsEngine.js';
|
|
12
12
|
|
|
13
13
|
export interface ServiceContext {
|
|
14
14
|
configuration: utils.ResolvedPowerSyncConfig;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { traceWriter } from './TraceWriter.js';
|
|
2
|
+
|
|
3
|
+
export interface Span extends Disposable {
|
|
4
|
+
name: string;
|
|
5
|
+
startAt: number;
|
|
6
|
+
endAt: number;
|
|
7
|
+
selfDuration: number;
|
|
8
|
+
nestedSince: number | undefined;
|
|
9
|
+
subtrackFromSelf: number;
|
|
10
|
+
nestedDurations: Record<string, number>;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* End the span - same as [Symbol.dispose]().
|
|
14
|
+
*
|
|
15
|
+
* Safe to call multiple times. Any nested spans will automatically end as well.
|
|
16
|
+
*
|
|
17
|
+
* Returns an aggregate record of category -> "selfDuration".
|
|
18
|
+
*/
|
|
19
|
+
end(): Record<string, number>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function now() {
|
|
23
|
+
return Number(process.hrtime.bigint() / 1000n);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let nextThreadId = 1;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Lightweight tracing helper, with two main goals:
|
|
30
|
+
* 1. Generate aggregate timing info with low overhead.
|
|
31
|
+
* 2. Optional support for generating trace files during development.
|
|
32
|
+
*
|
|
33
|
+
* This is only intended for a single "thread" - concurrent operations on the same instance have undefined behavior.
|
|
34
|
+
* To trace concurrent operations, use separate instances of PerformanceTracer.
|
|
35
|
+
*
|
|
36
|
+
* Spans cannot be overlapping: If a parent span is ended, all nested spans are automatically ended.
|
|
37
|
+
*/
|
|
38
|
+
export class PerformanceTracer<K extends string> {
|
|
39
|
+
stack: Span[] = [];
|
|
40
|
+
threadId: number;
|
|
41
|
+
|
|
42
|
+
constructor(traceName: string) {
|
|
43
|
+
this.threadId = nextThreadId;
|
|
44
|
+
nextThreadId += 1;
|
|
45
|
+
traceWriter?.write({
|
|
46
|
+
ph: 'M',
|
|
47
|
+
cat: '__metadata',
|
|
48
|
+
name: 'thread_name',
|
|
49
|
+
pid: process.pid,
|
|
50
|
+
tid: this.threadId,
|
|
51
|
+
args: { name: `PowerSync ${traceName}` }
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Recommended usage:
|
|
57
|
+
*
|
|
58
|
+
* using _ = tracer.span('cat', 'details');
|
|
59
|
+
*
|
|
60
|
+
* The above automatically ends the span when it goes out of scope. Alternatively, call
|
|
61
|
+
* .end() on the span to end it earlier.
|
|
62
|
+
*
|
|
63
|
+
* @param category one of the defined categories
|
|
64
|
+
* @param subcat optional subcategory. Not used for calculating "self" durations in the aggregate API.
|
|
65
|
+
*/
|
|
66
|
+
span(category: K, subcat?: string): Span {
|
|
67
|
+
const stack = this.stack;
|
|
68
|
+
const index = this.stack.length;
|
|
69
|
+
const parent = this.stack[this.stack.length - 1];
|
|
70
|
+
const threadId = this.threadId;
|
|
71
|
+
const startAt = now();
|
|
72
|
+
if (parent != null) {
|
|
73
|
+
parent.nestedSince ??= startAt;
|
|
74
|
+
}
|
|
75
|
+
let name: string = category;
|
|
76
|
+
if (subcat) {
|
|
77
|
+
name += ':' + subcat;
|
|
78
|
+
}
|
|
79
|
+
const s: Span = {
|
|
80
|
+
name,
|
|
81
|
+
startAt: now(),
|
|
82
|
+
selfDuration: 0,
|
|
83
|
+
endAt: 0,
|
|
84
|
+
nestedSince: undefined,
|
|
85
|
+
subtrackFromSelf: 0,
|
|
86
|
+
nestedDurations: {},
|
|
87
|
+
end() {
|
|
88
|
+
if (this.endAt != 0) {
|
|
89
|
+
return this.nestedDurations;
|
|
90
|
+
}
|
|
91
|
+
while (stack.length - 1 > index) {
|
|
92
|
+
stack[stack.length - 1].end();
|
|
93
|
+
}
|
|
94
|
+
const endAt = now();
|
|
95
|
+
this.endAt = endAt;
|
|
96
|
+
const endTime = this.nestedSince ?? endAt;
|
|
97
|
+
this.selfDuration = endTime - startAt - this.subtrackFromSelf;
|
|
98
|
+
traceWriter?.write({
|
|
99
|
+
name,
|
|
100
|
+
cat: 'powersync',
|
|
101
|
+
ph: 'X',
|
|
102
|
+
ts: this.startAt,
|
|
103
|
+
dur: endAt - startAt,
|
|
104
|
+
pid: process.pid,
|
|
105
|
+
tid: threadId
|
|
106
|
+
});
|
|
107
|
+
stack.pop();
|
|
108
|
+
if (parent != null) {
|
|
109
|
+
parent.subtrackFromSelf += endAt - parent.nestedSince!;
|
|
110
|
+
for (let key in this.nestedDurations) {
|
|
111
|
+
parent.nestedDurations[key] = (parent.nestedDurations[key] ?? 0) + this.nestedDurations[key];
|
|
112
|
+
}
|
|
113
|
+
parent.nestedDurations[category] = (parent.nestedDurations[category] ?? 0) + this.selfDuration;
|
|
114
|
+
parent.nestedSince = undefined;
|
|
115
|
+
}
|
|
116
|
+
return this.nestedDurations;
|
|
117
|
+
},
|
|
118
|
+
[Symbol.dispose]() {
|
|
119
|
+
this.end();
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
this.stack.push(s);
|
|
123
|
+
|
|
124
|
+
return s;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Mutex } from 'async-mutex';
|
|
2
|
+
import * as fs from 'node:fs/promises';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Write traces in the Chrome JSON Trace Format.
|
|
6
|
+
*
|
|
7
|
+
* View at https://ui.perfetto.dev/
|
|
8
|
+
*/
|
|
9
|
+
class TraceWriter {
|
|
10
|
+
handle: fs.FileHandle | null = null;
|
|
11
|
+
length = 0;
|
|
12
|
+
queue: any[] = [];
|
|
13
|
+
private mutex = new Mutex();
|
|
14
|
+
|
|
15
|
+
constructor(public readonly path: string) {
|
|
16
|
+
this.open().catch((e) => {
|
|
17
|
+
console.error(`Failed to open trace file at ${path}`, e);
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async open() {
|
|
22
|
+
await this.mutex.runExclusive(async () => {
|
|
23
|
+
this.handle = await fs.open(this.path, 'w+');
|
|
24
|
+
this.handle.truncate(0);
|
|
25
|
+
await this.handle.write('[]');
|
|
26
|
+
this.length = 2;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
write(...traceEvents: any[]) {
|
|
31
|
+
this.writeAsync(...traceEvents).catch((e) => {
|
|
32
|
+
console.error(`Failed to write trace file`, e);
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async writeAsync(...traceEvents: any[]) {
|
|
37
|
+
this.queue.push(...traceEvents);
|
|
38
|
+
await this.mutex.runExclusive(async () => {
|
|
39
|
+
if (this.queue.length > 0) {
|
|
40
|
+
// Write queued events.
|
|
41
|
+
// After each write, we end the file as a valid JSON array.
|
|
42
|
+
// On the next write, we overwrite the last character to extend the array.
|
|
43
|
+
const buffer = Buffer.from(JSON.stringify(this.queue));
|
|
44
|
+
await this.handle?.write(buffer, 1, buffer.length - 1, this.length - 1);
|
|
45
|
+
this.queue = [];
|
|
46
|
+
this.length += buffer.length - 2;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const traceFile = process.env.POWERSYNC_TRACE_FILE;
|
|
53
|
+
/**
|
|
54
|
+
* traceWriter, only present if POWERSYNC_TRACE_FILE env var is configured.
|
|
55
|
+
*/
|
|
56
|
+
export const traceWriter = traceFile ? new TraceWriter(traceFile) : null;
|
|
57
|
+
|
|
58
|
+
if (traceWriter) {
|
|
59
|
+
traceWriter.write({
|
|
60
|
+
ph: 'M',
|
|
61
|
+
cat: '__metadata',
|
|
62
|
+
name: 'process_name',
|
|
63
|
+
pid: process.pid,
|
|
64
|
+
tid: 1000,
|
|
65
|
+
args: { name: 'powersync' }
|
|
66
|
+
});
|
|
67
|
+
}
|
|
@@ -13,7 +13,7 @@ export async function createWriteCheckpoint(options: CreateWriteCheckpointOption
|
|
|
13
13
|
|
|
14
14
|
const syncBucketStorage = await options.storage.getActiveStorage();
|
|
15
15
|
if (!syncBucketStorage) {
|
|
16
|
-
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync
|
|
16
|
+
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
|