@powersync/service-core 1.20.4 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/api/RouteAPI.d.ts +17 -3
- package/dist/api/api-index.d.ts +1 -1
- package/dist/api/api-index.js +1 -1
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +1 -1
- package/dist/api/diagnostics.js +32 -14
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/CachedKeyCollector.js +1 -1
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.d.ts +1 -1
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -1
- package/dist/entry/entry-index.js +1 -1
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/events/EventsEngine.js +1 -1
- package/dist/events/EventsEngine.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/metrics/MetricsEngine.d.ts +1 -1
- package/dist/metrics/metrics-index.d.ts +3 -3
- package/dist/metrics/metrics-index.js +3 -3
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +1 -1
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/metrics/register-metrics.js +2 -2
- package/dist/metrics/register-metrics.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +2 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/modules-index.d.ts +1 -1
- package/dist/modules/modules-index.js +1 -1
- package/dist/modules/modules-index.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +1 -1
- package/dist/replication/AbstractReplicationJob.js +1 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +6 -6
- package/dist/replication/AbstractReplicator.js +21 -21
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/replication-index.d.ts +3 -3
- package/dist/replication/replication-index.js +3 -3
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/replication/replication-metrics.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +59 -32
- package/dist/routes/endpoints/admin.d.ts +108 -54
- package/dist/routes/endpoints/admin.js +7 -3
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +10 -10
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -10
- package/dist/routes/endpoints/sync-stream.js +2 -2
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.js +1 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +4 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +9 -9
- package/dist/storage/BucketStorage.js +9 -9
- package/dist/storage/BucketStorageBatch.d.ts +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +27 -20
- package/dist/storage/BucketStorageFactory.js +19 -16
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
- package/dist/storage/PersistedSyncRulesContent.js +24 -5
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/SourceTable.d.ts +4 -4
- package/dist/storage/SourceTable.js +3 -3
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +1 -1
- package/dist/storage/StorageVersionConfig.js +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
- package/dist/storage/SyncRulesBucketStorage.js +14 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
- package/dist/storage/WriteCheckpointAPI.js +1 -1
- package/dist/storage/bson.d.ts +0 -1
- package/dist/storage/bson.js +0 -4
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +8 -8
- package/dist/storage/storage-index.js +8 -8
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/streams/streams-index.d.ts +2 -2
- package/dist/streams/streams-index.js +2 -2
- package/dist/streams/streams-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +2 -5
- package/dist/sync/BucketChecksumState.js +119 -75
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.js +1 -1
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +2 -2
- package/dist/sync/sync-index.js +2 -2
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +1 -1
- package/dist/system/ServiceContext.js +1 -1
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +44 -0
- package/dist/tracing/PerformanceTracer.js +102 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -0
- package/dist/tracing/TraceWriter.d.ts +22 -0
- package/dist/tracing/TraceWriter.js +63 -0
- package/dist/tracing/TraceWriter.js.map +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +1 -1
- package/dist/util/config/compound-config-collector.js +2 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config.js +1 -1
- package/dist/util/config.js.map +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/errors.d.ts +3 -0
- package/dist/util/errors.js +15 -0
- package/dist/util/errors.js.map +1 -0
- package/dist/util/protocol-types.d.ts +3 -3
- package/dist/util/protocol-types.js +1 -1
- package/dist/util/util-index.d.ts +1 -1
- package/dist/util/util-index.js +1 -1
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +20 -3
- package/src/api/api-index.ts +1 -1
- package/src/api/api-metrics.ts +1 -1
- package/src/api/diagnostics.ts +42 -20
- package/src/auth/CachedKeyCollector.ts +2 -3
- package/src/auth/CompoundKeyCollector.ts +2 -3
- package/src/auth/KeyStore.ts +1 -1
- package/src/auth/RemoteJWKSCollector.ts +0 -1
- package/src/auth/StaticKeyCollector.ts +1 -1
- package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -1
- package/src/events/EventsEngine.ts +1 -1
- package/src/index.ts +2 -0
- package/src/metrics/MetricsEngine.ts +1 -1
- package/src/metrics/metrics-index.ts +3 -3
- package/src/metrics/open-telemetry/util.ts +1 -1
- package/src/metrics/register-metrics.ts +3 -3
- package/src/modules/AbstractModule.ts +2 -2
- package/src/modules/modules-index.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +2 -2
- package/src/replication/AbstractReplicator.ts +23 -23
- package/src/replication/replication-index.ts +3 -3
- package/src/replication/replication-metrics.ts +1 -1
- package/src/routes/endpoints/admin.ts +7 -3
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +10 -12
- package/src/routes/endpoints/sync-stream.ts +2 -2
- package/src/routes/hooks.ts +2 -2
- package/src/routes/route-register.ts +2 -10
- package/src/runner/teardown.ts +4 -4
- package/src/storage/BucketStorage.ts +9 -9
- package/src/storage/BucketStorageBatch.ts +1 -1
- package/src/storage/BucketStorageFactory.ts +45 -34
- package/src/storage/ChecksumCache.ts +1 -1
- package/src/storage/PersistedSyncRulesContent.ts +30 -6
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/SourceTable.ts +4 -4
- package/src/storage/StorageVersionConfig.ts +1 -1
- package/src/storage/SyncRulesBucketStorage.ts +46 -7
- package/src/storage/WriteCheckpointAPI.ts +6 -6
- package/src/storage/bson.ts +0 -5
- package/src/storage/storage-index.ts +8 -8
- package/src/storage/storage-metrics.ts +2 -2
- package/src/streams/streams-index.ts +2 -2
- package/src/sync/BucketChecksumState.ts +141 -93
- package/src/sync/RequestTracker.ts +1 -1
- package/src/sync/sync-index.ts +2 -2
- package/src/sync/sync.ts +2 -8
- package/src/sync/util.ts +1 -1
- package/src/system/ServiceContext.ts +1 -1
- package/src/tracing/PerformanceTracer.ts +126 -0
- package/src/tracing/TraceWriter.ts +67 -0
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
- package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
- package/src/util/config/compound-config-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
- package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
- package/src/util/config.ts +1 -1
- package/src/util/env.ts +1 -1
- package/src/util/errors.ts +21 -0
- package/src/util/protocol-types.ts +1 -1
- package/src/util/util-index.ts +1 -1
- package/src/util/utils.ts +1 -1
- package/test/src/auth.test.ts +115 -7
- package/test/src/diagnostics.test.ts +151 -0
- package/test/src/module-loader.test.ts +1 -1
- package/test/src/routes/mocks.ts +1 -1
- package/test/src/routes/stream.test.ts +1 -2
- package/test/src/sync/BucketChecksumState.test.ts +223 -67
- package/test/src/util/protocol_types.test.ts +1 -1
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +1 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
|
|
2
2
|
|
|
3
|
+
import { EventsEngine } from '../events/EventsEngine.js';
|
|
3
4
|
import { framework } from '../index.js';
|
|
4
5
|
import * as metrics from '../metrics/MetricsEngine.js';
|
|
5
6
|
import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
|
|
@@ -8,7 +9,6 @@ import * as routes from '../routes/routes-index.js';
|
|
|
8
9
|
import * as storage from '../storage/storage-index.js';
|
|
9
10
|
import { SyncContext } from '../sync/SyncContext.js';
|
|
10
11
|
import * as utils from '../util/util-index.js';
|
|
11
|
-
import { EventsEngine } from '../events/EventsEngine.js';
|
|
12
12
|
|
|
13
13
|
export interface ServiceContext {
|
|
14
14
|
configuration: utils.ResolvedPowerSyncConfig;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { traceWriter } from './TraceWriter.js';
|
|
2
|
+
|
|
3
|
+
export interface Span extends Disposable {
|
|
4
|
+
name: string;
|
|
5
|
+
startAt: number;
|
|
6
|
+
endAt: number;
|
|
7
|
+
selfDuration: number;
|
|
8
|
+
nestedSince: number | undefined;
|
|
9
|
+
subtrackFromSelf: number;
|
|
10
|
+
nestedDurations: Record<string, number>;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* End the span - same as [Symbol.dispose]().
|
|
14
|
+
*
|
|
15
|
+
* Safe to call multiple times. Any nested spans will automatically end as well.
|
|
16
|
+
*
|
|
17
|
+
* Returns an aggregate record of category -> "selfDuration".
|
|
18
|
+
*/
|
|
19
|
+
end(): Record<string, number>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function now() {
|
|
23
|
+
return Number(process.hrtime.bigint() / 1000n);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let nextThreadId = 1;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Lightweight tracing helper, with two main goals:
|
|
30
|
+
* 1. Generate aggregate timing info with low overhead.
|
|
31
|
+
* 2. Optional support for generating trace files during development.
|
|
32
|
+
*
|
|
33
|
+
* This is only intended for a single "thread" - concurrent operations on the same instance have undefined behavior.
|
|
34
|
+
* To trace concurrent operations, use separate instances of PerformanceTracer.
|
|
35
|
+
*
|
|
36
|
+
* Spans cannot be overlapping: If a parent span is ended, all nested spans are automatically ended.
|
|
37
|
+
*/
|
|
38
|
+
export class PerformanceTracer<K extends string> {
|
|
39
|
+
stack: Span[] = [];
|
|
40
|
+
threadId: number;
|
|
41
|
+
|
|
42
|
+
constructor(traceName: string) {
|
|
43
|
+
this.threadId = nextThreadId;
|
|
44
|
+
nextThreadId += 1;
|
|
45
|
+
traceWriter?.write({
|
|
46
|
+
ph: 'M',
|
|
47
|
+
cat: '__metadata',
|
|
48
|
+
name: 'thread_name',
|
|
49
|
+
pid: process.pid,
|
|
50
|
+
tid: this.threadId,
|
|
51
|
+
args: { name: `PowerSync ${traceName}` }
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Recommended usage:
|
|
57
|
+
*
|
|
58
|
+
* using _ = tracer.span('cat', 'details');
|
|
59
|
+
*
|
|
60
|
+
* The above automatically ends the span when it goes out of scope. Alternatively, call
|
|
61
|
+
* .end() on the span to end it earlier.
|
|
62
|
+
*
|
|
63
|
+
* @param category one of the defined categories
|
|
64
|
+
* @param subcat optional subcategory. Not used for calculating "self" durations in the aggregate API.
|
|
65
|
+
*/
|
|
66
|
+
span(category: K, subcat?: string): Span {
|
|
67
|
+
const stack = this.stack;
|
|
68
|
+
const index = this.stack.length;
|
|
69
|
+
const parent = this.stack[this.stack.length - 1];
|
|
70
|
+
const threadId = this.threadId;
|
|
71
|
+
const startAt = now();
|
|
72
|
+
if (parent != null) {
|
|
73
|
+
parent.nestedSince ??= startAt;
|
|
74
|
+
}
|
|
75
|
+
let name: string = category;
|
|
76
|
+
if (subcat) {
|
|
77
|
+
name += ':' + subcat;
|
|
78
|
+
}
|
|
79
|
+
const s: Span = {
|
|
80
|
+
name,
|
|
81
|
+
startAt: now(),
|
|
82
|
+
selfDuration: 0,
|
|
83
|
+
endAt: 0,
|
|
84
|
+
nestedSince: undefined,
|
|
85
|
+
subtrackFromSelf: 0,
|
|
86
|
+
nestedDurations: {},
|
|
87
|
+
end() {
|
|
88
|
+
if (this.endAt != 0) {
|
|
89
|
+
return this.nestedDurations;
|
|
90
|
+
}
|
|
91
|
+
while (stack.length - 1 > index) {
|
|
92
|
+
stack[stack.length - 1].end();
|
|
93
|
+
}
|
|
94
|
+
const endAt = now();
|
|
95
|
+
this.endAt = endAt;
|
|
96
|
+
const endTime = this.nestedSince ?? endAt;
|
|
97
|
+
this.selfDuration = endTime - startAt - this.subtrackFromSelf;
|
|
98
|
+
traceWriter?.write({
|
|
99
|
+
name,
|
|
100
|
+
cat: 'powersync',
|
|
101
|
+
ph: 'X',
|
|
102
|
+
ts: this.startAt,
|
|
103
|
+
dur: endAt - startAt,
|
|
104
|
+
pid: process.pid,
|
|
105
|
+
tid: threadId
|
|
106
|
+
});
|
|
107
|
+
stack.pop();
|
|
108
|
+
if (parent != null) {
|
|
109
|
+
parent.subtrackFromSelf += endAt - parent.nestedSince!;
|
|
110
|
+
for (let key in this.nestedDurations) {
|
|
111
|
+
parent.nestedDurations[key] = (parent.nestedDurations[key] ?? 0) + this.nestedDurations[key];
|
|
112
|
+
}
|
|
113
|
+
parent.nestedDurations[category] = (parent.nestedDurations[category] ?? 0) + this.selfDuration;
|
|
114
|
+
parent.nestedSince = undefined;
|
|
115
|
+
}
|
|
116
|
+
return this.nestedDurations;
|
|
117
|
+
},
|
|
118
|
+
[Symbol.dispose]() {
|
|
119
|
+
this.end();
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
this.stack.push(s);
|
|
123
|
+
|
|
124
|
+
return s;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Mutex } from 'async-mutex';
|
|
2
|
+
import * as fs from 'node:fs/promises';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Write traces in the Chrome JSON Trace Format.
|
|
6
|
+
*
|
|
7
|
+
* View at https://ui.perfetto.dev/
|
|
8
|
+
*/
|
|
9
|
+
class TraceWriter {
|
|
10
|
+
handle: fs.FileHandle | null = null;
|
|
11
|
+
length = 0;
|
|
12
|
+
queue: any[] = [];
|
|
13
|
+
private mutex = new Mutex();
|
|
14
|
+
|
|
15
|
+
constructor(public readonly path: string) {
|
|
16
|
+
this.open().catch((e) => {
|
|
17
|
+
console.error(`Failed to open trace file at ${path}`, e);
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async open() {
|
|
22
|
+
await this.mutex.runExclusive(async () => {
|
|
23
|
+
this.handle = await fs.open(this.path, 'w+');
|
|
24
|
+
this.handle.truncate(0);
|
|
25
|
+
await this.handle.write('[]');
|
|
26
|
+
this.length = 2;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
write(...traceEvents: any[]) {
|
|
31
|
+
this.writeAsync(...traceEvents).catch((e) => {
|
|
32
|
+
console.error(`Failed to write trace file`, e);
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async writeAsync(...traceEvents: any[]) {
|
|
37
|
+
this.queue.push(...traceEvents);
|
|
38
|
+
await this.mutex.runExclusive(async () => {
|
|
39
|
+
if (this.queue.length > 0) {
|
|
40
|
+
// Write queued events.
|
|
41
|
+
// After each write, we end the file as a valid JSON array.
|
|
42
|
+
// On the next write, we overwrite the last character to extend the array.
|
|
43
|
+
const buffer = Buffer.from(JSON.stringify(this.queue));
|
|
44
|
+
await this.handle?.write(buffer, 1, buffer.length - 1, this.length - 1);
|
|
45
|
+
this.queue = [];
|
|
46
|
+
this.length += buffer.length - 2;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const traceFile = process.env.POWERSYNC_TRACE_FILE;
|
|
53
|
+
/**
|
|
54
|
+
* traceWriter, only present if POWERSYNC_TRACE_FILE env var is configured.
|
|
55
|
+
*/
|
|
56
|
+
export const traceWriter = traceFile ? new TraceWriter(traceFile) : null;
|
|
57
|
+
|
|
58
|
+
if (traceWriter) {
|
|
59
|
+
traceWriter.write({
|
|
60
|
+
ph: 'M',
|
|
61
|
+
cat: '__metadata',
|
|
62
|
+
name: 'process_name',
|
|
63
|
+
pid: process.pid,
|
|
64
|
+
tid: 1000,
|
|
65
|
+
args: { name: 'powersync' }
|
|
66
|
+
});
|
|
67
|
+
}
|
|
@@ -13,7 +13,7 @@ export async function createWriteCheckpoint(options: CreateWriteCheckpointOption
|
|
|
13
13
|
|
|
14
14
|
const syncBucketStorage = await options.storage.getActiveStorage();
|
|
15
15
|
if (!syncBucketStorage) {
|
|
16
|
-
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync
|
|
16
|
+
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
|
|
4
|
-
import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
|
|
5
|
-
import { RunnerConfig } from '../../types.js';
|
|
6
4
|
import { logger } from '@powersync/lib-services-framework';
|
|
5
|
+
import { RunnerConfig } from '../../types.js';
|
|
6
|
+
import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
|
|
7
7
|
|
|
8
8
|
export class FileSystemConfigCollector extends ConfigCollector {
|
|
9
9
|
get name(): string {
|
|
@@ -26,7 +26,7 @@ export type CompoundConfigCollectorOptions = {
|
|
|
26
26
|
*/
|
|
27
27
|
configCollectors: ConfigCollector[];
|
|
28
28
|
/**
|
|
29
|
-
* Collectors for PowerSync sync
|
|
29
|
+
* Collectors for PowerSync sync config content.
|
|
30
30
|
* The configuration from first collector to provide a configuration
|
|
31
31
|
* is used. The order of the collectors specifies precedence
|
|
32
32
|
*/
|
|
@@ -236,11 +236,11 @@ export class CompoundConfigCollector {
|
|
|
236
236
|
return config;
|
|
237
237
|
}
|
|
238
238
|
logger.debug(
|
|
239
|
-
`Could not collect sync
|
|
239
|
+
`Could not collect sync config with ${collector.name} method. Moving on to next method if available.`
|
|
240
240
|
);
|
|
241
241
|
} catch (ex) {
|
|
242
242
|
// An error in a collector is a hard stop
|
|
243
|
-
throw new Error(`Could not collect sync
|
|
243
|
+
throw new Error(`Could not collect sync config using ${collector.name} method. Caught exception: ${ex}`);
|
|
244
244
|
}
|
|
245
245
|
}
|
|
246
246
|
return {
|
|
@@ -16,7 +16,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
|
|
|
16
16
|
|
|
17
17
|
const { config_path } = runnerConfig;
|
|
18
18
|
|
|
19
|
-
// Depending on the container, the sync
|
|
19
|
+
// Depending on the container, the sync config may not actually be present.
|
|
20
20
|
// Only persist the path here, and load on demand using `loadSyncRules()`.
|
|
21
21
|
return {
|
|
22
22
|
present: true,
|
package/src/util/config.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
2
|
import winston from 'winston';
|
|
3
3
|
|
|
4
|
-
import { container,
|
|
4
|
+
import { container, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, LogFormat, logger } from '@powersync/lib-services-framework';
|
|
5
5
|
import { configFile } from '@powersync/service-types';
|
|
6
6
|
import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
|
|
7
7
|
import { CompoundConfigCollector } from './util-index.js';
|
package/src/util/env.ts
CHANGED
|
@@ -13,7 +13,7 @@ export const env = utils.collectEnvironmentVariables({
|
|
|
13
13
|
POWERSYNC_CONFIG_B64: utils.type.string.optional(),
|
|
14
14
|
/**
|
|
15
15
|
* @deprecated use POWERSYNC_SYNC_CONFIG_B64 instead.
|
|
16
|
-
* Base64 encoded contents of sync
|
|
16
|
+
* Base64 encoded contents of sync config YAML
|
|
17
17
|
*/
|
|
18
18
|
POWERSYNC_SYNC_RULES_B64: utils.type.string.optional(),
|
|
19
19
|
/**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { YamlError } from '@powersync/service-sync-rules';
|
|
2
|
+
import { ReplicationError } from '@powersync/service-types';
|
|
3
|
+
|
|
4
|
+
export function syncConfigYamlErrorToReplicationError(
|
|
5
|
+
{ type, message, location }: YamlError,
|
|
6
|
+
ts?: string
|
|
7
|
+
): ReplicationError {
|
|
8
|
+
const error: ReplicationError = {
|
|
9
|
+
level: type,
|
|
10
|
+
message,
|
|
11
|
+
ts
|
|
12
|
+
};
|
|
13
|
+
if (location != null) {
|
|
14
|
+
error.location = {
|
|
15
|
+
start_offset: location.start,
|
|
16
|
+
end_offset: location.end
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return error;
|
|
21
|
+
}
|
package/src/util/util-index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from './alerting.js';
|
|
2
|
+
export * from './checkpointing.js';
|
|
2
3
|
export * from './env.js';
|
|
3
4
|
export * from './lsn.js';
|
|
4
5
|
export * from './memory-tracking.js';
|
|
@@ -6,7 +7,6 @@ export * from './Mutex.js';
|
|
|
6
7
|
export * from './protocol-types.js';
|
|
7
8
|
export * from './secs.js';
|
|
8
9
|
export * from './utils.js';
|
|
9
|
-
export * from './checkpointing.js';
|
|
10
10
|
export * from './version.js';
|
|
11
11
|
|
|
12
12
|
export * from './config.js';
|
package/src/util/utils.ts
CHANGED
|
@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
|
|
|
2
2
|
import * as bson from 'bson';
|
|
3
3
|
import crypto from 'crypto';
|
|
4
4
|
import * as uuid from 'uuid';
|
|
5
|
-
import { BucketChecksum,
|
|
5
|
+
import { BucketChecksum, OplogEntry, ProtocolOpId } from './protocol-types.js';
|
|
6
6
|
|
|
7
7
|
import * as storage from '../storage/storage-index.js';
|
|
8
8
|
|
package/test/src/auth.test.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { StaticSupabaseKeyCollector } from '@/index.js';
|
|
2
|
+
import { configFile } from '@powersync/service-types';
|
|
3
3
|
import * as jose from 'jose';
|
|
4
|
+
import { describe, expect, test } from 'vitest';
|
|
5
|
+
import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
|
|
6
|
+
import { KeyResult } from '../../src/auth/KeyCollector.js';
|
|
7
|
+
import {
|
|
8
|
+
EC_ALGORITHMS,
|
|
9
|
+
HS_ALGORITHMS,
|
|
10
|
+
KeySpec,
|
|
11
|
+
OKP_ALGORITHMS,
|
|
12
|
+
RSA_ALGORITHMS,
|
|
13
|
+
SUPPORTED_ALGORITHMS
|
|
14
|
+
} from '../../src/auth/KeySpec.js';
|
|
4
15
|
import { KeyStore } from '../../src/auth/KeyStore.js';
|
|
5
|
-
import { KeySpec } from '../../src/auth/KeySpec.js';
|
|
6
16
|
import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
|
|
10
|
-
import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
|
|
17
|
+
import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
|
|
18
|
+
import { getSupabaseJwksUrl } from '../../src/auth/utils.js';
|
|
11
19
|
|
|
12
20
|
const publicKeyRSA: jose.JWK = {
|
|
13
21
|
use: 'sig',
|
|
@@ -52,6 +60,47 @@ const privateKeyECDSA: jose.JWK = {
|
|
|
52
60
|
alg: 'ES256'
|
|
53
61
|
};
|
|
54
62
|
|
|
63
|
+
const EC_ALGORITHM_CURVES = [
|
|
64
|
+
['ES256', 'P-256'],
|
|
65
|
+
['ES384', 'P-384'],
|
|
66
|
+
['ES512', 'P-521']
|
|
67
|
+
] satisfies [string, string][];
|
|
68
|
+
|
|
69
|
+
const EDDSA_CURVES = ['Ed25519', 'Ed448'];
|
|
70
|
+
|
|
71
|
+
function roundTripJwkThroughPowerSyncConfig(key: jose.JWK): jose.JWK {
|
|
72
|
+
const encoded = configFile.strictJwks.encode({
|
|
73
|
+
keys: [key as configFile.StrictJwk]
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const decoded = configFile.strictJwks.decode(encoded);
|
|
77
|
+
return decoded.keys[0] as jose.JWK;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function signAndVerifyWithKey(alg: string, key: jose.JWK, signKey: jose.KeyLike | Uint8Array) {
|
|
81
|
+
const parsedKey = roundTripJwkThroughPowerSyncConfig(key);
|
|
82
|
+
expect(parsedKey).toEqual(key);
|
|
83
|
+
|
|
84
|
+
const keys = await StaticKeyCollector.importKeys([parsedKey]);
|
|
85
|
+
const store = new KeyStore(keys);
|
|
86
|
+
|
|
87
|
+
const signedJwt = await new jose.SignJWT({ claim: alg })
|
|
88
|
+
.setProtectedHeader({ alg, kid: key.kid })
|
|
89
|
+
.setSubject('f1')
|
|
90
|
+
.setIssuedAt()
|
|
91
|
+
.setIssuer('tester')
|
|
92
|
+
.setAudience('tests')
|
|
93
|
+
.setExpirationTime('5m')
|
|
94
|
+
.sign(signKey);
|
|
95
|
+
|
|
96
|
+
const verified = await store.verifyJwt(signedJwt, {
|
|
97
|
+
defaultAudiences: ['tests'],
|
|
98
|
+
maxAge: '6m'
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
expect(verified.parsedPayload.claim).toEqual(alg);
|
|
102
|
+
}
|
|
103
|
+
|
|
55
104
|
describe('JWT Auth', () => {
|
|
56
105
|
test('KeyStore basics', async () => {
|
|
57
106
|
const keys = await StaticKeyCollector.importKeys([sharedKey]);
|
|
@@ -208,6 +257,65 @@ describe('JWT Auth', () => {
|
|
|
208
257
|
).rejects.toThrow('Unexpected token algorithm HS256');
|
|
209
258
|
});
|
|
210
259
|
|
|
260
|
+
describe('supported JWT algorithms', () => {
|
|
261
|
+
test('covers every declared supported algorithm', () => {
|
|
262
|
+
const testedAlgorithms = new Set([
|
|
263
|
+
...HS_ALGORITHMS,
|
|
264
|
+
...RSA_ALGORITHMS,
|
|
265
|
+
...EC_ALGORITHM_CURVES.map(([alg]) => alg),
|
|
266
|
+
...OKP_ALGORITHMS
|
|
267
|
+
]);
|
|
268
|
+
|
|
269
|
+
expect([...testedAlgorithms].sort()).toEqual([...SUPPORTED_ALGORITHMS].sort());
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
test.each(HS_ALGORITHMS)('verifies %s tokens', async (alg) => {
|
|
273
|
+
const secret = await jose.generateSecret(alg);
|
|
274
|
+
const key = await jose.exportJWK(secret);
|
|
275
|
+
key.kid = `test-${alg}`;
|
|
276
|
+
key.alg = alg;
|
|
277
|
+
|
|
278
|
+
await signAndVerifyWithKey(alg, key, secret);
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
test.each(RSA_ALGORITHMS)('verifies %s tokens', async (alg) => {
|
|
282
|
+
const { privateKey, publicKey } = await jose.generateKeyPair(alg);
|
|
283
|
+
const key = await jose.exportJWK(publicKey);
|
|
284
|
+
key.kid = `test-${alg}`;
|
|
285
|
+
key.alg = alg;
|
|
286
|
+
key.use = 'sig';
|
|
287
|
+
|
|
288
|
+
await signAndVerifyWithKey(alg, key, privateKey);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
test.each(EC_ALGORITHM_CURVES)('verifies %s tokens with curve %s', async (alg, crv) => {
|
|
292
|
+
expect(EC_ALGORITHMS).toContain(alg);
|
|
293
|
+
|
|
294
|
+
const { privateKey, publicKey } = await jose.generateKeyPair(alg);
|
|
295
|
+
const key = await jose.exportJWK(publicKey);
|
|
296
|
+
key.kid = `test-${alg}`;
|
|
297
|
+
key.alg = alg;
|
|
298
|
+
key.use = 'sig';
|
|
299
|
+
|
|
300
|
+
expect(key.crv).toEqual(crv);
|
|
301
|
+
await signAndVerifyWithKey(alg, key, privateKey);
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
test.each(EDDSA_CURVES)('verifies EdDSA tokens with curve %s', async (crv) => {
|
|
305
|
+
const alg = 'EdDSA';
|
|
306
|
+
expect(OKP_ALGORITHMS).toContain(alg);
|
|
307
|
+
|
|
308
|
+
const { privateKey, publicKey } = await jose.generateKeyPair(alg, { crv });
|
|
309
|
+
const key = await jose.exportJWK(publicKey);
|
|
310
|
+
key.kid = `test-${alg}-${crv}`;
|
|
311
|
+
key.alg = alg;
|
|
312
|
+
key.use = 'sig';
|
|
313
|
+
|
|
314
|
+
expect(key.crv).toEqual(crv);
|
|
315
|
+
await signAndVerifyWithKey(alg, key, privateKey);
|
|
316
|
+
});
|
|
317
|
+
});
|
|
318
|
+
|
|
211
319
|
test('key selection for key with kid', async () => {
|
|
212
320
|
const keys = await StaticKeyCollector.importKeys([publicKeyRSA, sharedKey, sharedKey2]);
|
|
213
321
|
const store = new KeyStore(keys);
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { DiagnosticsOptions, getSyncRulesStatus } from '@/api/diagnostics.js';
|
|
2
|
+
import { RouteAPI, SlotWalBudgetInfo } from '@/api/RouteAPI.js';
|
|
3
|
+
import { BucketStorageFactory } from '@/index.js';
|
|
4
|
+
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
5
|
+
import { describe, expect, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
const GB = 1024 * 1024 * 1024;
|
|
8
|
+
|
|
9
|
+
const MINIMAL_SYNC_RULES = `
|
|
10
|
+
bucket_definitions:
|
|
11
|
+
global:
|
|
12
|
+
data:
|
|
13
|
+
- SELECT id FROM test_table
|
|
14
|
+
`;
|
|
15
|
+
|
|
16
|
+
function makeSyncRulesContent(overrides?: { slot_name?: string }) {
|
|
17
|
+
return {
|
|
18
|
+
id: 1,
|
|
19
|
+
slot_name: overrides?.slot_name ?? 'test_slot',
|
|
20
|
+
sync_rules_content: MINIMAL_SYNC_RULES,
|
|
21
|
+
compiled_plan: null,
|
|
22
|
+
active: true,
|
|
23
|
+
storageVersion: 1,
|
|
24
|
+
last_checkpoint_lsn: 'some_lsn',
|
|
25
|
+
last_fatal_error: null,
|
|
26
|
+
last_fatal_error_ts: null,
|
|
27
|
+
last_keepalive_ts: new Date(),
|
|
28
|
+
last_checkpoint_ts: new Date(),
|
|
29
|
+
parsed(options?: any) {
|
|
30
|
+
const syncRules = SqlSyncRules.fromYaml(MINIMAL_SYNC_RULES, {
|
|
31
|
+
...options,
|
|
32
|
+
defaultSchema: 'public'
|
|
33
|
+
});
|
|
34
|
+
return {
|
|
35
|
+
sync_rules: syncRules
|
|
36
|
+
};
|
|
37
|
+
},
|
|
38
|
+
lock() {
|
|
39
|
+
throw new Error('Not implemented in mock');
|
|
40
|
+
},
|
|
41
|
+
current_lock: undefined
|
|
42
|
+
} as any;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function makeBucketStorage() {
|
|
46
|
+
return {
|
|
47
|
+
getInstance() {
|
|
48
|
+
return {
|
|
49
|
+
async getStatus() {
|
|
50
|
+
return {
|
|
51
|
+
snapshot_done: true,
|
|
52
|
+
checkpoint_lsn: 'some_lsn',
|
|
53
|
+
active: true
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
} as unknown as BucketStorageFactory;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function makeRouteAPI(walBudget?: SlotWalBudgetInfo | undefined): RouteAPI {
|
|
62
|
+
return {
|
|
63
|
+
getParseSyncRulesOptions() {
|
|
64
|
+
return { defaultSchema: 'public' };
|
|
65
|
+
},
|
|
66
|
+
async getSourceConfig() {
|
|
67
|
+
return { tag: 'test', id: 'test', type: 'postgresql' };
|
|
68
|
+
},
|
|
69
|
+
async getConnectionStatus() {
|
|
70
|
+
return { connected: true };
|
|
71
|
+
},
|
|
72
|
+
async getDebugTablesInfo() {
|
|
73
|
+
return [];
|
|
74
|
+
},
|
|
75
|
+
async getReplicationLagBytes() {
|
|
76
|
+
return 0;
|
|
77
|
+
},
|
|
78
|
+
...(walBudget !== undefined
|
|
79
|
+
? {
|
|
80
|
+
async getSlotWalBudget() {
|
|
81
|
+
return walBudget;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
: {})
|
|
85
|
+
} as unknown as RouteAPI;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const OPTIONS: DiagnosticsOptions = {
|
|
89
|
+
live_status: true,
|
|
90
|
+
check_connection: true,
|
|
91
|
+
include_content: false
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
95
|
+
test('warns when WAL budget is at 40%', async () => {
|
|
96
|
+
const api = makeRouteAPI({
|
|
97
|
+
wal_status: 'extended',
|
|
98
|
+
safe_wal_size: 4 * GB,
|
|
99
|
+
max_slot_wal_keep_size: 10 * GB
|
|
100
|
+
});
|
|
101
|
+
const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
|
|
102
|
+
const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
|
|
103
|
+
expect(walWarnings).toHaveLength(1);
|
|
104
|
+
expect(walWarnings[0].level).toBe('warning');
|
|
105
|
+
expect(walWarnings[0].message).toContain('40%');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('no warning when WAL budget is at 80%', async () => {
|
|
109
|
+
const api = makeRouteAPI({
|
|
110
|
+
wal_status: 'reserved',
|
|
111
|
+
safe_wal_size: 8 * GB,
|
|
112
|
+
max_slot_wal_keep_size: 10 * GB
|
|
113
|
+
});
|
|
114
|
+
const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
|
|
115
|
+
const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
|
|
116
|
+
expect(walWarnings).toHaveLength(0);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('clamps negative safe_wal_size to 0%', async () => {
|
|
120
|
+
const api = makeRouteAPI({
|
|
121
|
+
wal_status: 'unreserved',
|
|
122
|
+
safe_wal_size: -2.4 * GB,
|
|
123
|
+
max_slot_wal_keep_size: 1 * 1024 * 1024 // 1MB
|
|
124
|
+
});
|
|
125
|
+
const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
|
|
126
|
+
const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
|
|
127
|
+
expect(walWarnings).toHaveLength(1);
|
|
128
|
+
expect(walWarnings[0].message).toContain('0%');
|
|
129
|
+
expect(walWarnings[0].message).not.toMatch(/-\d+%/);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test('no WAL budget error when slot status is lost', async () => {
|
|
133
|
+
const api = makeRouteAPI({
|
|
134
|
+
wal_status: 'lost'
|
|
135
|
+
});
|
|
136
|
+
const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
|
|
137
|
+
const walErrors = result!.errors.filter(
|
|
138
|
+
(e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146')
|
|
139
|
+
);
|
|
140
|
+
expect(walErrors).toHaveLength(0);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test('no WAL error when getSlotWalBudget is not defined', async () => {
|
|
144
|
+
const api = makeRouteAPI();
|
|
145
|
+
const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS);
|
|
146
|
+
const walErrors = result!.errors.filter(
|
|
147
|
+
(e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146')
|
|
148
|
+
);
|
|
149
|
+
expect(walErrors).toHaveLength(0);
|
|
150
|
+
});
|
|
151
|
+
});
|
package/test/src/routes/mocks.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EventsEngine } from '@/events/EventsEngine.js';
|
|
1
2
|
import {
|
|
2
3
|
BucketStorageFactory,
|
|
3
4
|
createCoreAPIMetrics,
|
|
@@ -11,7 +12,6 @@ import {
|
|
|
11
12
|
SyncRulesBucketStorage
|
|
12
13
|
} from '@/index.js';
|
|
13
14
|
import { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
14
|
-
import { EventsEngine } from '@/events/EventsEngine.js';
|
|
15
15
|
|
|
16
16
|
export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
|
|
17
17
|
// This is very incomplete - just enough to get the current tests passing.
|