@powersync/service-core 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -5
- package/dist/api/RouteAPI.d.ts +17 -3
- package/dist/api/api-index.d.ts +1 -1
- package/dist/api/api-index.js +1 -1
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +1 -1
- package/dist/api/diagnostics.js +32 -14
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/CachedKeyCollector.js +1 -1
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.d.ts +1 -1
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/auth/utils.d.ts +2 -1
- package/dist/auth/utils.js +1 -1
- package/dist/auth/utils.js.map +1 -1
- package/dist/entry/commands/compact-action.js +26 -5
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -1
- package/dist/entry/entry-index.js +1 -1
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/events/EventsEngine.js +1 -1
- package/dist/events/EventsEngine.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/metrics/MetricsEngine.d.ts +1 -1
- package/dist/metrics/RollingBucketMax.d.ts +28 -0
- package/dist/metrics/RollingBucketMax.js +80 -0
- package/dist/metrics/RollingBucketMax.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +3 -2
- package/dist/metrics/metrics-index.js +3 -2
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +1 -1
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/metrics/register-metrics.js +2 -2
- package/dist/metrics/register-metrics.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +2 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/modules-index.d.ts +1 -1
- package/dist/modules/modules-index.js +1 -1
- package/dist/modules/modules-index.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -2
- package/dist/replication/AbstractReplicationJob.js +1 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +7 -7
- package/dist/replication/AbstractReplicator.js +31 -28
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationLagTracker.d.ts +50 -0
- package/dist/replication/ReplicationLagTracker.js +78 -0
- package/dist/replication/ReplicationLagTracker.js.map +1 -0
- package/dist/replication/replication-index.d.ts +3 -2
- package/dist/replication/replication-index.js +3 -2
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/replication/replication-metrics.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +59 -32
- package/dist/routes/endpoints/admin.d.ts +108 -54
- package/dist/routes/endpoints/admin.js +8 -3
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +10 -10
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -10
- package/dist/routes/endpoints/sync-stream.js +8 -3
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.js +1 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +4 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +9 -9
- package/dist/storage/BucketStorage.js +9 -9
- package/dist/storage/BucketStorageBatch.d.ts +30 -9
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +32 -20
- package/dist/storage/BucketStorageFactory.js +19 -16
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +5 -2
- package/dist/storage/ChecksumCache.js +8 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +9 -3
- package/dist/storage/PersistedSyncRulesContent.js +25 -5
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/SourceTable.d.ts +10 -5
- package/dist/storage/SourceTable.js +3 -3
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +34 -1
- package/dist/storage/StorageVersionConfig.js +40 -7
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +63 -11
- package/dist/storage/SyncRulesBucketStorage.js +14 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
- package/dist/storage/WriteCheckpointAPI.js +1 -1
- package/dist/storage/bson.d.ts +0 -1
- package/dist/storage/bson.js +0 -4
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +8 -8
- package/dist/storage/storage-index.js +8 -8
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/streams/streams-index.d.ts +2 -2
- package/dist/streams/streams-index.js +2 -2
- package/dist/streams/streams-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +5 -8
- package/dist/sync/BucketChecksumState.js +118 -116
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.js +1 -1
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +2 -2
- package/dist/sync/sync-index.js +2 -2
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -0
- package/dist/sync/util.js +10 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +1 -1
- package/dist/system/ServiceContext.js +1 -1
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +44 -0
- package/dist/tracing/PerformanceTracer.js +102 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -0
- package/dist/tracing/TraceWriter.d.ts +22 -0
- package/dist/tracing/TraceWriter.js +63 -0
- package/dist/tracing/TraceWriter.js.map +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +1 -1
- package/dist/util/config/compound-config-collector.js +2 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config.js +1 -1
- package/dist/util/config.js.map +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/errors.d.ts +3 -0
- package/dist/util/errors.js +15 -0
- package/dist/util/errors.js.map +1 -0
- package/dist/util/protocol-types.d.ts +3 -3
- package/dist/util/protocol-types.js +1 -1
- package/dist/util/util-index.d.ts +1 -1
- package/dist/util/util-index.js +1 -1
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +20 -3
- package/src/api/api-index.ts +1 -1
- package/src/api/api-metrics.ts +1 -1
- package/src/api/diagnostics.ts +42 -20
- package/src/auth/CachedKeyCollector.ts +2 -3
- package/src/auth/CompoundKeyCollector.ts +2 -3
- package/src/auth/KeyStore.ts +1 -1
- package/src/auth/RemoteJWKSCollector.ts +0 -1
- package/src/auth/StaticKeyCollector.ts +1 -1
- package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
- package/src/auth/utils.ts +5 -2
- package/src/entry/commands/compact-action.ts +29 -5
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -1
- package/src/events/EventsEngine.ts +1 -1
- package/src/index.ts +2 -0
- package/src/metrics/MetricsEngine.ts +1 -1
- package/src/metrics/RollingBucketMax.ts +109 -0
- package/src/metrics/metrics-index.ts +3 -2
- package/src/metrics/open-telemetry/util.ts +1 -1
- package/src/metrics/register-metrics.ts +3 -3
- package/src/modules/AbstractModule.ts +2 -2
- package/src/modules/modules-index.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +3 -3
- package/src/replication/AbstractReplicator.ts +32 -30
- package/src/replication/ReplicationLagTracker.ts +86 -0
- package/src/replication/replication-index.ts +3 -2
- package/src/replication/replication-metrics.ts +1 -1
- package/src/routes/endpoints/admin.ts +8 -3
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +10 -12
- package/src/routes/endpoints/sync-stream.ts +8 -3
- package/src/routes/hooks.ts +2 -2
- package/src/routes/route-register.ts +2 -10
- package/src/runner/teardown.ts +4 -4
- package/src/storage/BucketStorage.ts +9 -9
- package/src/storage/BucketStorageBatch.ts +34 -10
- package/src/storage/BucketStorageFactory.ts +51 -34
- package/src/storage/ChecksumCache.ts +14 -6
- package/src/storage/PersistedSyncRulesContent.ts +36 -7
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/SourceTable.ts +10 -4
- package/src/storage/StorageVersionConfig.ts +55 -7
- package/src/storage/SyncRulesBucketStorage.ts +77 -11
- package/src/storage/WriteCheckpointAPI.ts +6 -6
- package/src/storage/bson.ts +0 -5
- package/src/storage/storage-index.ts +8 -8
- package/src/storage/storage-metrics.ts +2 -2
- package/src/streams/streams-index.ts +2 -2
- package/src/sync/BucketChecksumState.ts +142 -139
- package/src/sync/RequestTracker.ts +1 -1
- package/src/sync/sync-index.ts +2 -2
- package/src/sync/sync.ts +4 -4
- package/src/sync/util.ts +11 -1
- package/src/system/ServiceContext.ts +1 -1
- package/src/tracing/PerformanceTracer.ts +126 -0
- package/src/tracing/TraceWriter.ts +67 -0
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
- package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
- package/src/util/config/compound-config-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
- package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
- package/src/util/config.ts +1 -1
- package/src/util/env.ts +1 -1
- package/src/util/errors.ts +21 -0
- package/src/util/protocol-types.ts +1 -1
- package/src/util/util-index.ts +1 -1
- package/src/util/utils.ts +1 -1
- package/test/src/ReplicationLagTracker.test.ts +53 -0
- package/test/src/RollingBucketMax.test.ts +106 -0
- package/test/src/auth.test.ts +26 -7
- package/test/src/checksum_cache.test.ts +102 -57
- package/test/src/config.test.ts +1 -0
- package/test/src/diagnostics.test.ts +151 -0
- package/test/src/module-loader.test.ts +1 -1
- package/test/src/routes/mocks.ts +1 -1
- package/test/src/routes/stream.test.ts +1 -2
- package/test/src/sync/BucketChecksumState.test.ts +209 -80
- package/test/src/util/protocol_types.test.ts +1 -1
- package/test/src/utils.ts +9 -0
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +1 -1
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { traceWriter } from './TraceWriter.js';
|
|
2
|
+
|
|
3
|
+
export interface Span extends Disposable {
|
|
4
|
+
name: string;
|
|
5
|
+
startAt: number;
|
|
6
|
+
endAt: number;
|
|
7
|
+
selfDuration: number;
|
|
8
|
+
nestedSince: number | undefined;
|
|
9
|
+
subtrackFromSelf: number;
|
|
10
|
+
nestedDurations: Record<string, number>;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* End the span - same as [Symbol.dispose]().
|
|
14
|
+
*
|
|
15
|
+
* Safe to call multiple times. Any nested spans will automatically end as well.
|
|
16
|
+
*
|
|
17
|
+
* Returns an aggregate record of category -> "selfDuration".
|
|
18
|
+
*/
|
|
19
|
+
end(): Record<string, number>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function now() {
|
|
23
|
+
return Number(process.hrtime.bigint() / 1000n);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let nextThreadId = 1;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Lightweight tracing helper, with two main goals:
|
|
30
|
+
* 1. Generate aggregate timing info with low overhead.
|
|
31
|
+
* 2. Optional support for generating trace files during development.
|
|
32
|
+
*
|
|
33
|
+
* This is only intended for a single "thread" - concurrent operations on the same instance have undefined behavior.
|
|
34
|
+
* To trace concurrent operations, use separate instances of PerformanceTracer.
|
|
35
|
+
*
|
|
36
|
+
* Spans cannot be overlapping: If a parent span is ended, all nested spans are automatically ended.
|
|
37
|
+
*/
|
|
38
|
+
export class PerformanceTracer<K extends string> {
|
|
39
|
+
stack: Span[] = [];
|
|
40
|
+
threadId: number;
|
|
41
|
+
|
|
42
|
+
constructor(traceName: string) {
|
|
43
|
+
this.threadId = nextThreadId;
|
|
44
|
+
nextThreadId += 1;
|
|
45
|
+
traceWriter?.write({
|
|
46
|
+
ph: 'M',
|
|
47
|
+
cat: '__metadata',
|
|
48
|
+
name: 'thread_name',
|
|
49
|
+
pid: process.pid,
|
|
50
|
+
tid: this.threadId,
|
|
51
|
+
args: { name: `PowerSync ${traceName}` }
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Recommended usage:
|
|
57
|
+
*
|
|
58
|
+
* using _ = tracer.span('cat', 'details');
|
|
59
|
+
*
|
|
60
|
+
* The above automatically ends the span when it goes out of scope. Alternatively, call
|
|
61
|
+
* .end() on the span to end it earlier.
|
|
62
|
+
*
|
|
63
|
+
* @param category one of the defined categories
|
|
64
|
+
* @param subcat optional subcategory. Not used for calculating "self" durations in the aggregate API.
|
|
65
|
+
*/
|
|
66
|
+
span(category: K, subcat?: string): Span {
|
|
67
|
+
const stack = this.stack;
|
|
68
|
+
const index = this.stack.length;
|
|
69
|
+
const parent = this.stack[this.stack.length - 1];
|
|
70
|
+
const threadId = this.threadId;
|
|
71
|
+
const startAt = now();
|
|
72
|
+
if (parent != null) {
|
|
73
|
+
parent.nestedSince ??= startAt;
|
|
74
|
+
}
|
|
75
|
+
let name: string = category;
|
|
76
|
+
if (subcat) {
|
|
77
|
+
name += ':' + subcat;
|
|
78
|
+
}
|
|
79
|
+
const s: Span = {
|
|
80
|
+
name,
|
|
81
|
+
startAt: now(),
|
|
82
|
+
selfDuration: 0,
|
|
83
|
+
endAt: 0,
|
|
84
|
+
nestedSince: undefined,
|
|
85
|
+
subtrackFromSelf: 0,
|
|
86
|
+
nestedDurations: {},
|
|
87
|
+
end() {
|
|
88
|
+
if (this.endAt != 0) {
|
|
89
|
+
return this.nestedDurations;
|
|
90
|
+
}
|
|
91
|
+
while (stack.length - 1 > index) {
|
|
92
|
+
stack[stack.length - 1].end();
|
|
93
|
+
}
|
|
94
|
+
const endAt = now();
|
|
95
|
+
this.endAt = endAt;
|
|
96
|
+
const endTime = this.nestedSince ?? endAt;
|
|
97
|
+
this.selfDuration = endTime - startAt - this.subtrackFromSelf;
|
|
98
|
+
traceWriter?.write({
|
|
99
|
+
name,
|
|
100
|
+
cat: 'powersync',
|
|
101
|
+
ph: 'X',
|
|
102
|
+
ts: this.startAt,
|
|
103
|
+
dur: endAt - startAt,
|
|
104
|
+
pid: process.pid,
|
|
105
|
+
tid: threadId
|
|
106
|
+
});
|
|
107
|
+
stack.pop();
|
|
108
|
+
if (parent != null) {
|
|
109
|
+
parent.subtrackFromSelf += endAt - parent.nestedSince!;
|
|
110
|
+
for (let key in this.nestedDurations) {
|
|
111
|
+
parent.nestedDurations[key] = (parent.nestedDurations[key] ?? 0) + this.nestedDurations[key];
|
|
112
|
+
}
|
|
113
|
+
parent.nestedDurations[category] = (parent.nestedDurations[category] ?? 0) + this.selfDuration;
|
|
114
|
+
parent.nestedSince = undefined;
|
|
115
|
+
}
|
|
116
|
+
return this.nestedDurations;
|
|
117
|
+
},
|
|
118
|
+
[Symbol.dispose]() {
|
|
119
|
+
this.end();
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
this.stack.push(s);
|
|
123
|
+
|
|
124
|
+
return s;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Mutex } from 'async-mutex';
|
|
2
|
+
import * as fs from 'node:fs/promises';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Write traces in the Chrome JSON Trace Format.
|
|
6
|
+
*
|
|
7
|
+
* View at https://ui.perfetto.dev/
|
|
8
|
+
*/
|
|
9
|
+
class TraceWriter {
|
|
10
|
+
handle: fs.FileHandle | null = null;
|
|
11
|
+
length = 0;
|
|
12
|
+
queue: any[] = [];
|
|
13
|
+
private mutex = new Mutex();
|
|
14
|
+
|
|
15
|
+
constructor(public readonly path: string) {
|
|
16
|
+
this.open().catch((e) => {
|
|
17
|
+
console.error(`Failed to open trace file at ${path}`, e);
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async open() {
|
|
22
|
+
await this.mutex.runExclusive(async () => {
|
|
23
|
+
this.handle = await fs.open(this.path, 'w+');
|
|
24
|
+
this.handle.truncate(0);
|
|
25
|
+
await this.handle.write('[]');
|
|
26
|
+
this.length = 2;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
write(...traceEvents: any[]) {
|
|
31
|
+
this.writeAsync(...traceEvents).catch((e) => {
|
|
32
|
+
console.error(`Failed to write trace file`, e);
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async writeAsync(...traceEvents: any[]) {
|
|
37
|
+
this.queue.push(...traceEvents);
|
|
38
|
+
await this.mutex.runExclusive(async () => {
|
|
39
|
+
if (this.queue.length > 0) {
|
|
40
|
+
// Write queued events.
|
|
41
|
+
// After each write, we end the file as a valid JSON array.
|
|
42
|
+
// On the next write, we overwrite the last character to extend the array.
|
|
43
|
+
const buffer = Buffer.from(JSON.stringify(this.queue));
|
|
44
|
+
await this.handle?.write(buffer, 1, buffer.length - 1, this.length - 1);
|
|
45
|
+
this.queue = [];
|
|
46
|
+
this.length += buffer.length - 2;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const traceFile = process.env.POWERSYNC_TRACE_FILE;
|
|
53
|
+
/**
|
|
54
|
+
* traceWriter, only present if POWERSYNC_TRACE_FILE env var is configured.
|
|
55
|
+
*/
|
|
56
|
+
export const traceWriter = traceFile ? new TraceWriter(traceFile) : null;
|
|
57
|
+
|
|
58
|
+
if (traceWriter) {
|
|
59
|
+
traceWriter.write({
|
|
60
|
+
ph: 'M',
|
|
61
|
+
cat: '__metadata',
|
|
62
|
+
name: 'process_name',
|
|
63
|
+
pid: process.pid,
|
|
64
|
+
tid: 1000,
|
|
65
|
+
args: { name: 'powersync' }
|
|
66
|
+
});
|
|
67
|
+
}
|
|
@@ -13,7 +13,7 @@ export async function createWriteCheckpoint(options: CreateWriteCheckpointOption
|
|
|
13
13
|
|
|
14
14
|
const syncBucketStorage = await options.storage.getActiveStorage();
|
|
15
15
|
if (!syncBucketStorage) {
|
|
16
|
-
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync
|
|
16
|
+
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
|
|
4
|
-
import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
|
|
5
|
-
import { RunnerConfig } from '../../types.js';
|
|
6
4
|
import { logger } from '@powersync/lib-services-framework';
|
|
5
|
+
import { RunnerConfig } from '../../types.js';
|
|
6
|
+
import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
|
|
7
7
|
|
|
8
8
|
export class FileSystemConfigCollector extends ConfigCollector {
|
|
9
9
|
get name(): string {
|
|
@@ -26,7 +26,7 @@ export type CompoundConfigCollectorOptions = {
|
|
|
26
26
|
*/
|
|
27
27
|
configCollectors: ConfigCollector[];
|
|
28
28
|
/**
|
|
29
|
-
* Collectors for PowerSync sync
|
|
29
|
+
* Collectors for PowerSync sync config content.
|
|
30
30
|
* The configuration from first collector to provide a configuration
|
|
31
31
|
* is used. The order of the collectors specifies precedence
|
|
32
32
|
*/
|
|
@@ -236,11 +236,11 @@ export class CompoundConfigCollector {
|
|
|
236
236
|
return config;
|
|
237
237
|
}
|
|
238
238
|
logger.debug(
|
|
239
|
-
`Could not collect sync
|
|
239
|
+
`Could not collect sync config with ${collector.name} method. Moving on to next method if available.`
|
|
240
240
|
);
|
|
241
241
|
} catch (ex) {
|
|
242
242
|
// An error in a collector is a hard stop
|
|
243
|
-
throw new Error(`Could not collect sync
|
|
243
|
+
throw new Error(`Could not collect sync config using ${collector.name} method. Caught exception: ${ex}`);
|
|
244
244
|
}
|
|
245
245
|
}
|
|
246
246
|
return {
|
|
@@ -16,7 +16,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
|
|
|
16
16
|
|
|
17
17
|
const { config_path } = runnerConfig;
|
|
18
18
|
|
|
19
|
-
// Depending on the container, the sync
|
|
19
|
+
// Depending on the container, the sync config may not actually be present.
|
|
20
20
|
// Only persist the path here, and load on demand using `loadSyncRules()`.
|
|
21
21
|
return {
|
|
22
22
|
present: true,
|
package/src/util/config.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
2
|
import winston from 'winston';
|
|
3
3
|
|
|
4
|
-
import { container,
|
|
4
|
+
import { container, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, LogFormat, logger } from '@powersync/lib-services-framework';
|
|
5
5
|
import { configFile } from '@powersync/service-types';
|
|
6
6
|
import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
|
|
7
7
|
import { CompoundConfigCollector } from './util-index.js';
|
package/src/util/env.ts
CHANGED
|
@@ -13,7 +13,7 @@ export const env = utils.collectEnvironmentVariables({
|
|
|
13
13
|
POWERSYNC_CONFIG_B64: utils.type.string.optional(),
|
|
14
14
|
/**
|
|
15
15
|
* @deprecated use POWERSYNC_SYNC_CONFIG_B64 instead.
|
|
16
|
-
* Base64 encoded contents of sync
|
|
16
|
+
* Base64 encoded contents of sync config YAML
|
|
17
17
|
*/
|
|
18
18
|
POWERSYNC_SYNC_RULES_B64: utils.type.string.optional(),
|
|
19
19
|
/**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { YamlError } from '@powersync/service-sync-rules';
|
|
2
|
+
import { ReplicationError } from '@powersync/service-types';
|
|
3
|
+
|
|
4
|
+
export function syncConfigYamlErrorToReplicationError(
|
|
5
|
+
{ type, message, location }: YamlError,
|
|
6
|
+
ts?: string
|
|
7
|
+
): ReplicationError {
|
|
8
|
+
const error: ReplicationError = {
|
|
9
|
+
level: type,
|
|
10
|
+
message,
|
|
11
|
+
ts
|
|
12
|
+
};
|
|
13
|
+
if (location != null) {
|
|
14
|
+
error.location = {
|
|
15
|
+
start_offset: location.start,
|
|
16
|
+
end_offset: location.end
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return error;
|
|
21
|
+
}
|
package/src/util/util-index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from './alerting.js';
|
|
2
|
+
export * from './checkpointing.js';
|
|
2
3
|
export * from './env.js';
|
|
3
4
|
export * from './lsn.js';
|
|
4
5
|
export * from './memory-tracking.js';
|
|
@@ -6,7 +7,6 @@ export * from './Mutex.js';
|
|
|
6
7
|
export * from './protocol-types.js';
|
|
7
8
|
export * from './secs.js';
|
|
8
9
|
export * from './utils.js';
|
|
9
|
-
export * from './checkpointing.js';
|
|
10
10
|
export * from './version.js';
|
|
11
11
|
|
|
12
12
|
export * from './config.js';
|
package/src/util/utils.ts
CHANGED
|
@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
|
|
|
2
2
|
import * as bson from 'bson';
|
|
3
3
|
import crypto from 'crypto';
|
|
4
4
|
import * as uuid from 'uuid';
|
|
5
|
-
import { BucketChecksum,
|
|
5
|
+
import { BucketChecksum, OplogEntry, ProtocolOpId } from './protocol-types.js';
|
|
6
6
|
|
|
7
7
|
import * as storage from '../storage/storage-index.js';
|
|
8
8
|
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { ReplicationLagTracker } from '@/replication/ReplicationLagTracker.js';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
|
|
4
|
+
describe('ReplicationLagTracker', () => {
|
|
5
|
+
it('returns undefined before replication has started', () => {
|
|
6
|
+
const tracker = new ReplicationLagTracker();
|
|
7
|
+
|
|
8
|
+
expect(tracker.getLagMillis(0)).toBeUndefined();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it('tracks the oldest in-flight change and returns current lag', () => {
|
|
12
|
+
const tracker = new ReplicationLagTracker();
|
|
13
|
+
|
|
14
|
+
tracker.trackUncommittedChange(new Date(1_000));
|
|
15
|
+
tracker.trackUncommittedChange(new Date(2_000));
|
|
16
|
+
|
|
17
|
+
expect(tracker.oldestUncommittedChange?.getTime()).toBe(1_000);
|
|
18
|
+
expect(tracker.getCurrentLagMillis(4_000)).toBe(3_000);
|
|
19
|
+
expect(tracker.getLagMillis(4_000)).toBe(3_000);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('records commit lag into the rolling window and clears in-flight state', () => {
|
|
23
|
+
const tracker = new ReplicationLagTracker();
|
|
24
|
+
|
|
25
|
+
tracker.trackUncommittedChange(new Date(0));
|
|
26
|
+
tracker.markCommitted(5_000);
|
|
27
|
+
|
|
28
|
+
expect(tracker.oldestUncommittedChange).toBeNull();
|
|
29
|
+
expect(tracker.isStartingReplication).toBe(false);
|
|
30
|
+
expect(tracker.getLagMillis(5_000)).toBe(5_000);
|
|
31
|
+
expect(tracker.getLagMillis(35_000)).toBe(0);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('can clear in-flight state without changing started state', () => {
|
|
35
|
+
const tracker = new ReplicationLagTracker();
|
|
36
|
+
|
|
37
|
+
tracker.trackUncommittedChange(new Date(0));
|
|
38
|
+
tracker.clearUncommittedChange();
|
|
39
|
+
|
|
40
|
+
expect(tracker.oldestUncommittedChange).toBeNull();
|
|
41
|
+
expect(tracker.isStartingReplication).toBe(true);
|
|
42
|
+
expect(tracker.getLagMillis(5_000)).toBeUndefined();
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('can mark replication as started without a committed transaction', () => {
|
|
46
|
+
const tracker = new ReplicationLagTracker();
|
|
47
|
+
|
|
48
|
+
tracker.markStarted();
|
|
49
|
+
|
|
50
|
+
expect(tracker.isStartingReplication).toBe(false);
|
|
51
|
+
expect(tracker.getLagMillis(0)).toBe(0);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { RollingBucketMax } from '@/metrics/RollingBucketMax.js';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
|
|
4
|
+
describe('RollingBucketMax', () => {
|
|
5
|
+
it('returns undefined before any values are reported', () => {
|
|
6
|
+
const tracker = new RollingBucketMax();
|
|
7
|
+
|
|
8
|
+
expect(tracker.getRollingMax(0)).toBeUndefined();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it('tracks the maximum value within a single bucket', () => {
|
|
12
|
+
const tracker = new RollingBucketMax();
|
|
13
|
+
|
|
14
|
+
tracker.report(3, 100);
|
|
15
|
+
tracker.report(9, 1_000);
|
|
16
|
+
tracker.report(5, 4_999);
|
|
17
|
+
|
|
18
|
+
expect(tracker.getRollingMax(4_999)).toBe(9);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('keeps the rolling max across the last six 5s buckets', () => {
|
|
22
|
+
const tracker = new RollingBucketMax();
|
|
23
|
+
|
|
24
|
+
tracker.report(20, 0);
|
|
25
|
+
tracker.report(11, 5_000);
|
|
26
|
+
tracker.report(12, 10_000);
|
|
27
|
+
tracker.report(13, 15_000);
|
|
28
|
+
tracker.report(14, 20_000);
|
|
29
|
+
tracker.report(15, 25_000);
|
|
30
|
+
|
|
31
|
+
expect(tracker.getRollingMax(29_999)).toBe(20);
|
|
32
|
+
expect(tracker.getRollingMax(30_000)).toBe(15);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('slides the rolling max forward as older buckets age out', () => {
|
|
36
|
+
const tracker = new RollingBucketMax();
|
|
37
|
+
|
|
38
|
+
tracker.report(20, 0);
|
|
39
|
+
expect(tracker.getRollingMax(0)).toBe(20);
|
|
40
|
+
|
|
41
|
+
tracker.report(18, 5_000);
|
|
42
|
+
expect(tracker.getRollingMax(5_000)).toBe(20);
|
|
43
|
+
|
|
44
|
+
tracker.report(16, 10_000);
|
|
45
|
+
expect(tracker.getRollingMax(10_000)).toBe(20);
|
|
46
|
+
|
|
47
|
+
tracker.report(14, 15_000);
|
|
48
|
+
expect(tracker.getRollingMax(15_000)).toBe(20);
|
|
49
|
+
|
|
50
|
+
tracker.report(12, 20_000);
|
|
51
|
+
expect(tracker.getRollingMax(20_000)).toBe(20);
|
|
52
|
+
|
|
53
|
+
tracker.report(10, 25_000);
|
|
54
|
+
expect(tracker.getRollingMax(29_999)).toBe(20);
|
|
55
|
+
|
|
56
|
+
tracker.report(8, 30_000);
|
|
57
|
+
expect(tracker.getRollingMax(30_000)).toBe(18);
|
|
58
|
+
|
|
59
|
+
tracker.report(6, 35_000);
|
|
60
|
+
expect(tracker.getRollingMax(35_000)).toBe(16);
|
|
61
|
+
|
|
62
|
+
tracker.report(4, 40_000);
|
|
63
|
+
expect(tracker.getRollingMax(40_000)).toBe(14);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('keeps newer buckets in the rolling window while older peaks fall out', () => {
|
|
67
|
+
const tracker = new RollingBucketMax();
|
|
68
|
+
|
|
69
|
+
tracker.report(50, 0);
|
|
70
|
+
tracker.report(11, 5_000);
|
|
71
|
+
tracker.report(12, 10_000);
|
|
72
|
+
tracker.report(13, 15_000);
|
|
73
|
+
tracker.report(14, 20_000);
|
|
74
|
+
tracker.report(15, 25_000);
|
|
75
|
+
|
|
76
|
+
expect(tracker.getRollingMax(29_999)).toBe(50);
|
|
77
|
+
|
|
78
|
+
tracker.report(40, 30_000);
|
|
79
|
+
expect(tracker.getRollingMax(30_000)).toBe(40);
|
|
80
|
+
expect(tracker.getRollingMax(34_999)).toBe(40);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('expires values after the rolling window passes with no new reports', () => {
|
|
84
|
+
const tracker = new RollingBucketMax();
|
|
85
|
+
|
|
86
|
+
tracker.report(7, 0);
|
|
87
|
+
|
|
88
|
+
expect(tracker.getRollingMax(29_999)).toBe(7);
|
|
89
|
+
expect(tracker.getRollingMax(30_000)).toBeUndefined();
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('supports custom bucket and window sizes', () => {
|
|
93
|
+
const tracker = new RollingBucketMax({
|
|
94
|
+
bucketSizeMs: 1_000,
|
|
95
|
+
windowSizeMs: 3_000
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
tracker.report(4, 0);
|
|
99
|
+
tracker.report(6, 1_000);
|
|
100
|
+
tracker.report(5, 2_000);
|
|
101
|
+
|
|
102
|
+
expect(tracker.getRollingMax(2_999)).toBe(6);
|
|
103
|
+
expect(tracker.getRollingMax(3_000)).toBe(6);
|
|
104
|
+
expect(tracker.getRollingMax(4_000)).toBe(5);
|
|
105
|
+
});
|
|
106
|
+
});
|
package/test/src/auth.test.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
|
|
1
|
+
import { StaticSupabaseKeyCollector } from '@/index.js';
|
|
3
2
|
import * as jose from 'jose';
|
|
4
|
-
import {
|
|
3
|
+
import { describe, expect, test } from 'vitest';
|
|
4
|
+
import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
|
|
5
|
+
import { KeyResult } from '../../src/auth/KeyCollector.js';
|
|
5
6
|
import { KeySpec } from '../../src/auth/KeySpec.js';
|
|
7
|
+
import { KeyStore } from '../../src/auth/KeyStore.js';
|
|
6
8
|
import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
|
|
10
|
-
import { debugKeyNotFound } from '../../src/auth/utils.js';
|
|
9
|
+
import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
|
|
10
|
+
import { getSupabaseJwksUrl } from '../../src/auth/utils.js';
|
|
11
11
|
|
|
12
12
|
const publicKeyRSA: jose.JWK = {
|
|
13
13
|
use: 'sig',
|
|
@@ -496,6 +496,25 @@ describe('JWT Auth', () => {
|
|
|
496
496
|
expect(verified.parsedPayload.claim).toEqual('test-claim-2');
|
|
497
497
|
});
|
|
498
498
|
|
|
499
|
+
test('Supabase connection parsing', () => {
|
|
500
|
+
const details = getSupabaseJwksUrl({
|
|
501
|
+
type: 'postgresql',
|
|
502
|
+
uri: 'postgresql://db.abc123.supabase.co:5432/postgres'
|
|
503
|
+
});
|
|
504
|
+
expect(details).toEqual({
|
|
505
|
+
hostname: 'db.abc123.supabase.co',
|
|
506
|
+
projectId: 'abc123',
|
|
507
|
+
url: 'https://abc123.supabase.co/auth/v1/.well-known/jwks.json'
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
// supabase.com is not a supabase URI
|
|
511
|
+
const other = getSupabaseJwksUrl({
|
|
512
|
+
type: 'postgresql',
|
|
513
|
+
uri: 'postgresql://db.abc123.supabase.com:5432/postgres'
|
|
514
|
+
});
|
|
515
|
+
expect(other).toEqual(null);
|
|
516
|
+
});
|
|
517
|
+
|
|
499
518
|
describe('debugKeyNotFound', () => {
|
|
500
519
|
test('Supabase token with legacy auth not configured', async () => {
|
|
501
520
|
const keys = await StaticSupabaseKeyCollector.importKeys([]);
|