@powersync/service-core 1.25.0 → 1.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/api/diagnostics.js +2 -0
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +28 -24
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +5 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +7 -3
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +16 -6
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +9 -0
- package/dist/routes/endpoints/admin.d.ts +18 -0
- package/dist/routes/endpoints/admin.js +3 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +3 -1
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +2 -1
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +28 -4
- package/dist/storage/BucketStorageFactory.js +8 -4
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.d.ts +2 -0
- package/dist/storage/PersistedSyncConfigContent.js +23 -10
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +6 -1
- package/dist/storage/SourceTable.d.ts +13 -5
- package/dist/storage/SourceTable.js +8 -4
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +4 -0
- package/dist/storage/StorageVersionConfig.js +10 -1
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +39 -7
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +25 -3
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +23 -7
- package/dist/storage/implementation/BucketDefinitionMapping.js +88 -14
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +9 -1
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -1
- package/dist/storage/storage-metrics.js +120 -4
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +1 -1
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +37 -9
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -1
- package/dist/sync/util.js +13 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +5 -0
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/dist/util/protocol-types.d.ts +4 -4
- package/dist/util/protocol-types.js +3 -2
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +1 -0
- package/dist/util/utils.js +15 -0
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -5
- package/src/api/diagnostics.ts +2 -0
- package/src/entry/commands/compact-action.ts +32 -24
- package/src/metrics/metrics-interfaces.ts +6 -1
- package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
- package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
- package/src/metrics/open-telemetry/util.ts +7 -3
- package/src/replication/AbstractReplicator.ts +22 -6
- package/src/routes/endpoints/admin.ts +3 -1
- package/src/routes/endpoints/sync-rules.ts +3 -1
- package/src/storage/BucketStorageBatch.ts +2 -1
- package/src/storage/BucketStorageFactory.ts +44 -8
- package/src/storage/PersistedSyncConfigContent.ts +26 -11
- package/src/storage/ReplicationEventPayload.ts +6 -1
- package/src/storage/SourceTable.ts +17 -4
- package/src/storage/StorageVersionConfig.ts +11 -1
- package/src/storage/SyncRulesBucketStorage.ts +47 -7
- package/src/storage/WriteCheckpointAPI.ts +32 -3
- package/src/storage/implementation/BucketDefinitionMapping.ts +120 -17
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +22 -5
- package/src/storage/storage-metrics.ts +154 -8
- package/src/sync/BucketChecksumState.ts +3 -1
- package/src/sync/sync.ts +40 -11
- package/src/sync/util.ts +25 -9
- package/src/util/config/compound-config-collector.ts +6 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
- package/src/util/config/types.ts +1 -0
- package/src/util/protocol-types.ts +3 -2
- package/src/util/utils.ts +14 -0
- package/test/src/AbstractReplicator.test.ts +30 -0
- package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
- package/test/src/PersistedSyncConfigContent.test.ts +152 -0
- package/test/src/config.test.ts +40 -0
- package/test/src/diagnostics.test.ts +25 -0
- package/test/src/routes/admin.test.ts +8 -2
- package/test/src/routes/stream.test.ts +123 -2
- package/test/src/storage/SourceTable.test.ts +4 -1
- package/test/src/storage-metrics.test.ts +148 -0
- package/test/src/sync/util.test.ts +34 -0
- package/test/src/util/utils.test.ts +23 -0
- package/tsconfig.tsbuildinfo +1 -1
package/src/sync/sync.ts
CHANGED
|
@@ -119,9 +119,14 @@ async function* streamResponseInner(
|
|
|
119
119
|
syncRequest: params,
|
|
120
120
|
logger: logger
|
|
121
121
|
});
|
|
122
|
+
// The checkpoint iterator can have a pending next() while a data batch fails.
|
|
123
|
+
// Aborting this on cleanup makes that pending next() settle, so the watcher subscription
|
|
124
|
+
// is released instead of staying open until another checkpoint arrives. See the cleanup
|
|
125
|
+
// in the finally block below.
|
|
126
|
+
const checkpointWatchController = new AbortController();
|
|
122
127
|
const stream = bucketStorage.watchCheckpointChanges({
|
|
123
128
|
user_id: checkpointUserId,
|
|
124
|
-
signal
|
|
129
|
+
signal: AbortSignal.any([signal, checkpointWatchController.signal])
|
|
125
130
|
});
|
|
126
131
|
const newCheckpoints = stream[Symbol.asyncIterator]();
|
|
127
132
|
|
|
@@ -170,6 +175,16 @@ async function* streamResponseInner(
|
|
|
170
175
|
}
|
|
171
176
|
}
|
|
172
177
|
|
|
178
|
+
function checkpointLogDetails(cp: storage.ReplicationCheckpoint) {
|
|
179
|
+
return {
|
|
180
|
+
checkpoint: cp.checkpoint,
|
|
181
|
+
user_id: tokenPayload.userIdJson,
|
|
182
|
+
...tracker.getIncrementalCheckpointStats()
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let activeCheckpoint: { checkpoint: storage.ReplicationCheckpoint; trace: ActiveCheckpointTrace } | null = null;
|
|
187
|
+
|
|
173
188
|
try {
|
|
174
189
|
let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
|
|
175
190
|
|
|
@@ -195,6 +210,7 @@ async function* streamResponseInner(
|
|
|
195
210
|
const trace = next.value.value.trace!;
|
|
196
211
|
const checkpoint = next.value.value.checkpoint;
|
|
197
212
|
const tracer = trace.tracer;
|
|
213
|
+
activeCheckpoint = { checkpoint, trace };
|
|
198
214
|
|
|
199
215
|
const { checkpointLine, bucketsToFetch, bucketDataRequestHint } = line;
|
|
200
216
|
|
|
@@ -266,14 +282,6 @@ async function* streamResponseInner(
|
|
|
266
282
|
maybeRaceForNewCheckpoint();
|
|
267
283
|
}
|
|
268
284
|
|
|
269
|
-
function checkpointLogDetails(cp: storage.ReplicationCheckpoint) {
|
|
270
|
-
return {
|
|
271
|
-
checkpoint: cp.checkpoint,
|
|
272
|
-
user_id: tokenPayload.userIdJson,
|
|
273
|
-
...tracker.getIncrementalCheckpointStats()
|
|
274
|
-
};
|
|
275
|
-
}
|
|
276
|
-
|
|
277
285
|
// This incrementally updates dataBuckets with each individual bucket position.
|
|
278
286
|
// At the end of this, we can be sure that all buckets have data up to the checkpoint.
|
|
279
287
|
for (const [priority, buckets] of priorityBatches) {
|
|
@@ -328,13 +336,34 @@ async function* streamResponseInner(
|
|
|
328
336
|
ms: getCheckpointTraceTimings(trace.span)
|
|
329
337
|
});
|
|
330
338
|
}
|
|
339
|
+
if (checkpointResult != null) {
|
|
340
|
+
activeCheckpoint = null;
|
|
341
|
+
}
|
|
331
342
|
|
|
332
343
|
if (!abortCheckpointSignal.aborted) {
|
|
333
344
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
334
345
|
}
|
|
335
346
|
} while (!signal.aborted);
|
|
336
347
|
} finally {
|
|
337
|
-
|
|
348
|
+
if (activeCheckpoint != null) {
|
|
349
|
+
const { checkpoint, trace } = activeCheckpoint;
|
|
350
|
+
logger.info(`checkpoint_interrupted: ${checkpoint.checkpoint}`, {
|
|
351
|
+
...checkpointLogDetails(checkpoint),
|
|
352
|
+
ms: getCheckpointTraceTimings(trace.span)
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// The abort makes a pending next() settle as soon as the storage watcher observes it,
|
|
357
|
+
// releasing the subscription. return() is still needed for the case where there is no
|
|
358
|
+
// pending next(): the watcher is then suspended at a yield and never observes the abort.
|
|
359
|
+
// Neither is awaited: async generators queue return() behind a pending next(), so
|
|
360
|
+
// awaiting it here would keep the connection open until the next checkpoint arrives.
|
|
361
|
+
checkpointWatchController.abort();
|
|
362
|
+
newCheckpoints.return?.().catch((e) => {
|
|
363
|
+
if (!isAbortError(e)) {
|
|
364
|
+
logger.warn('Error while closing the checkpoint watcher', e);
|
|
365
|
+
}
|
|
366
|
+
});
|
|
338
367
|
}
|
|
339
368
|
}
|
|
340
369
|
|
|
@@ -437,7 +466,6 @@ async function* bucketDataBatch(
|
|
|
437
466
|
} = request;
|
|
438
467
|
|
|
439
468
|
const tracer = request.tracer;
|
|
440
|
-
|
|
441
469
|
let checkpointInvalidated = false;
|
|
442
470
|
|
|
443
471
|
const acquired = await tracer.span('acquiring_lock').with(async () => {
|
|
@@ -456,6 +484,7 @@ async function* bucketDataBatch(
|
|
|
456
484
|
const filteredBuckets = checkpointLine.getFilteredBucketPositions(bucketsToFetch);
|
|
457
485
|
const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, {
|
|
458
486
|
requestHint,
|
|
487
|
+
tracer,
|
|
459
488
|
// Checkpoint supersession is a cooperative batch handoff. Only abort
|
|
460
489
|
// in-flight storage work when the connection itself is closed.
|
|
461
490
|
signal: abort_connection
|
package/src/sync/util.ts
CHANGED
|
@@ -144,9 +144,19 @@ export async function* transformToBytesTracked(
|
|
|
144
144
|
|
|
145
145
|
export function acquireSemaphoreAbortable(
|
|
146
146
|
semaphone: SemaphoreInterface,
|
|
147
|
-
abort
|
|
147
|
+
abort?: AbortSignal
|
|
148
148
|
): Promise<[number, SemaphoreInterface.Releaser] | 'aborted'> {
|
|
149
|
+
if (abort == null) {
|
|
150
|
+
return semaphone.acquire();
|
|
151
|
+
}
|
|
149
152
|
return new Promise((resolve, reject) => {
|
|
153
|
+
// An already-aborted signal never fires its listener, and we would wait for the semaphore
|
|
154
|
+
// indefinitely.
|
|
155
|
+
if (abort.aborted) {
|
|
156
|
+
resolve('aborted');
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
|
|
150
160
|
let aborted = false;
|
|
151
161
|
let hasSemaphore = false;
|
|
152
162
|
|
|
@@ -159,16 +169,22 @@ export function acquireSemaphoreAbortable(
|
|
|
159
169
|
};
|
|
160
170
|
abort.addEventListener('abort', listener);
|
|
161
171
|
|
|
162
|
-
semaphone.acquire().then(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
172
|
+
semaphone.acquire().then(
|
|
173
|
+
(acquired) => {
|
|
174
|
+
hasSemaphore = true;
|
|
175
|
+
if (aborted) {
|
|
176
|
+
// Release semaphore, already aborted
|
|
177
|
+
acquired[1]();
|
|
178
|
+
} else {
|
|
179
|
+
abort.removeEventListener('abort', listener);
|
|
180
|
+
resolve(acquired);
|
|
181
|
+
}
|
|
182
|
+
},
|
|
183
|
+
(error) => {
|
|
168
184
|
abort.removeEventListener('abort', listener);
|
|
169
|
-
|
|
185
|
+
reject(error);
|
|
170
186
|
}
|
|
171
|
-
|
|
187
|
+
);
|
|
172
188
|
});
|
|
173
189
|
}
|
|
174
190
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { logger, LookupOptions } from '@powersync/lib-services-framework';
|
|
2
|
+
import { validateStorageVersion } from '@powersync/service-sync-rules';
|
|
2
3
|
import { configFile } from '@powersync/service-types';
|
|
3
4
|
import * as auth from '../../auth/auth-index.js';
|
|
4
5
|
import { ConfigCollector } from './collectors/config-collector.js';
|
|
@@ -62,6 +63,11 @@ export class CompoundConfigCollector {
|
|
|
62
63
|
async collectConfig(runnerConfig: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
|
|
63
64
|
const baseConfig = await this.collectBaseConfig(runnerConfig);
|
|
64
65
|
|
|
66
|
+
const defaultStorageVersion = baseConfig.storage?.default_storage_version;
|
|
67
|
+
if (defaultStorageVersion != null && validateStorageVersion(defaultStorageVersion) == null) {
|
|
68
|
+
throw new Error(`Storage version ${defaultStorageVersion} is not supported`);
|
|
69
|
+
}
|
|
70
|
+
|
|
65
71
|
const dataSources = baseConfig.replication?.connections ?? [];
|
|
66
72
|
if (dataSources.length > 1) {
|
|
67
73
|
throw new Error('Only a single replication data source is supported currently');
|
|
@@ -15,6 +15,7 @@ export class Base64SyncRulesCollector extends SyncRulesCollector {
|
|
|
15
15
|
|
|
16
16
|
return {
|
|
17
17
|
present: true,
|
|
18
|
+
version_label: baseConfig.sync_config?.version_label,
|
|
18
19
|
exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
|
|
19
20
|
content: Buffer.from(sync_config_base64, 'base64').toString()
|
|
20
21
|
};
|
|
@@ -20,6 +20,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
|
|
|
20
20
|
// Only persist the path here, and load on demand using `loadSyncRules()`.
|
|
21
21
|
return {
|
|
22
22
|
present: true,
|
|
23
|
+
version_label: baseConfig.sync_config?.version_label,
|
|
23
24
|
exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
|
|
24
25
|
path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
|
|
25
26
|
};
|
|
@@ -5,6 +5,7 @@ export interface SyncRulesProvider {
|
|
|
5
5
|
get(): Promise<string | undefined>;
|
|
6
6
|
|
|
7
7
|
readonly exitOnError: boolean;
|
|
8
|
+
readonly versionLabel: string | undefined;
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
@@ -21,4 +22,8 @@ export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
|
21
22
|
get exitOnError() {
|
|
22
23
|
return this.config.exit_on_error;
|
|
23
24
|
}
|
|
25
|
+
|
|
26
|
+
get versionLabel() {
|
|
27
|
+
return this.config.version_label;
|
|
28
|
+
}
|
|
24
29
|
}
|
package/src/util/config/types.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { JsonContainer } from '@powersync/service-jsonbig';
|
|
2
2
|
import { BucketPriority, SqliteJsonRow } from '@powersync/service-sync-rules';
|
|
3
|
+
import { orNull } from '@powersync/service-types';
|
|
3
4
|
import * as t from 'ts-codec';
|
|
4
5
|
|
|
5
6
|
export const BucketRequest = t.object({
|
|
@@ -24,14 +25,14 @@ export const RequestedStreamSubscription = t.object({
|
|
|
24
25
|
/**
|
|
25
26
|
* An optional dictionary of parameters to pass to this specific stream.
|
|
26
27
|
*/
|
|
27
|
-
parameters:
|
|
28
|
+
parameters: orNull(t.record(t.any)),
|
|
28
29
|
/**
|
|
29
30
|
* Set when the client wishes to re-assign a different priority to this stream.
|
|
30
31
|
*
|
|
31
32
|
* Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
|
|
32
33
|
* useful when the priority for partial syncs depends on e.g. the current page opened in a client.
|
|
33
34
|
*/
|
|
34
|
-
override_priority:
|
|
35
|
+
override_priority: orNull(t.number)
|
|
35
36
|
});
|
|
36
37
|
|
|
37
38
|
export type RequestedStreamSubscription = t.Decoded<typeof RequestedStreamSubscription>;
|
package/src/util/utils.ts
CHANGED
|
@@ -309,3 +309,17 @@ export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefine
|
|
|
309
309
|
}
|
|
310
310
|
return size;
|
|
311
311
|
}
|
|
312
|
+
|
|
313
|
+
export function formatBytes(bytes: number | bigint): string {
|
|
314
|
+
if (typeof bytes == 'bigint') {
|
|
315
|
+
bytes = Number(bytes); // We round either way
|
|
316
|
+
}
|
|
317
|
+
if (bytes >= 1024 * 1024 * 1024) {
|
|
318
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
|
319
|
+
} else if (bytes >= 1024 * 1024) {
|
|
320
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
321
|
+
} else if (bytes >= 1024) {
|
|
322
|
+
return `${(bytes / 1024).toFixed(1)}KB`;
|
|
323
|
+
}
|
|
324
|
+
return `${bytes}B`;
|
|
325
|
+
}
|
|
@@ -44,6 +44,14 @@ class TestReplicator extends AbstractReplicator {
|
|
|
44
44
|
get heartbeatIntervalNanosForTest(): bigint | null {
|
|
45
45
|
return (this as any).heartbeatIntervalNanos;
|
|
46
46
|
}
|
|
47
|
+
|
|
48
|
+
shouldHandleStreamForTest(
|
|
49
|
+
replicationStream: PersistedReplicationStream,
|
|
50
|
+
loadedSyncRules: string | undefined,
|
|
51
|
+
loadedVersionLabel: string | undefined
|
|
52
|
+
): boolean {
|
|
53
|
+
return (this as any).shouldHandleReplicationStream(replicationStream, loadedSyncRules, loadedVersionLabel);
|
|
54
|
+
}
|
|
47
55
|
}
|
|
48
56
|
|
|
49
57
|
describe('AbstractReplicator heartbeat interval', () => {
|
|
@@ -145,3 +153,25 @@ describe('AbstractReplicator stopped stream cleanup', () => {
|
|
|
145
153
|
expect(stopped).toBe(true);
|
|
146
154
|
});
|
|
147
155
|
});
|
|
156
|
+
|
|
157
|
+
describe('AbstractReplicator rolling deployment config matching', () => {
|
|
158
|
+
const processingStream = (version_label: string | undefined): PersistedReplicationStream =>
|
|
159
|
+
({
|
|
160
|
+
syncConfigContent: [
|
|
161
|
+
{
|
|
162
|
+
syncConfigState: 'PROCESSING',
|
|
163
|
+
sync_rules_content: 'bucket_definitions: {}',
|
|
164
|
+
version_label
|
|
165
|
+
}
|
|
166
|
+
]
|
|
167
|
+
}) as unknown as PersistedReplicationStream;
|
|
168
|
+
|
|
169
|
+
it('requires both YAML and version label to match a processing config', () => {
|
|
170
|
+
const replicator = new TestReplicator(async () => {});
|
|
171
|
+
const stream = processingStream('v2');
|
|
172
|
+
|
|
173
|
+
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v2')).toBe(true);
|
|
174
|
+
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v1')).toBe(false);
|
|
175
|
+
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', undefined)).toBe(false);
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { MetricAttributeFilteringExporter } from '@/metrics/open-telemetry/MetricAttributeFilteringExporter.js';
|
|
2
|
+
import { Attributes } from '@opentelemetry/api';
|
|
3
|
+
import { ExportResult } from '@opentelemetry/core';
|
|
4
|
+
import { DataPointType, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
|
|
5
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
6
|
+
|
|
7
|
+
type ExportMetrics = (metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void) => void;
|
|
8
|
+
|
|
9
|
+
describe('MetricAttributeFilteringExporter', () => {
|
|
10
|
+
it('filters configured attributes without changing the collected metrics', () => {
|
|
11
|
+
const exportMetrics = vi.fn<ExportMetrics>();
|
|
12
|
+
const delegate = {
|
|
13
|
+
export: exportMetrics,
|
|
14
|
+
forceFlush: vi.fn(async () => {}),
|
|
15
|
+
shutdown: vi.fn(async () => {})
|
|
16
|
+
} satisfies PushMetricExporter;
|
|
17
|
+
const exporter = new MetricAttributeFilteringExporter(delegate, new Set(['version_label']));
|
|
18
|
+
const attributes: Attributes = {
|
|
19
|
+
sync_config_id: 'config-a',
|
|
20
|
+
sync_config_state: 'ACTIVE',
|
|
21
|
+
version_label: 'customer-version'
|
|
22
|
+
};
|
|
23
|
+
const metrics = {
|
|
24
|
+
resource: {},
|
|
25
|
+
scopeMetrics: [
|
|
26
|
+
{
|
|
27
|
+
scope: {},
|
|
28
|
+
metrics: [
|
|
29
|
+
{
|
|
30
|
+
dataPointType: DataPointType.GAUGE,
|
|
31
|
+
dataPoints: [{ attributes, value: 100 }]
|
|
32
|
+
}
|
|
33
|
+
]
|
|
34
|
+
}
|
|
35
|
+
]
|
|
36
|
+
} as unknown as ResourceMetrics;
|
|
37
|
+
const callback = vi.fn();
|
|
38
|
+
|
|
39
|
+
exporter.export(metrics, callback);
|
|
40
|
+
|
|
41
|
+
expect(exportMetrics).toHaveBeenCalledWith(
|
|
42
|
+
expect.objectContaining({
|
|
43
|
+
scopeMetrics: [
|
|
44
|
+
expect.objectContaining({
|
|
45
|
+
metrics: [
|
|
46
|
+
expect.objectContaining({
|
|
47
|
+
dataPoints: [
|
|
48
|
+
expect.objectContaining({
|
|
49
|
+
attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' }
|
|
50
|
+
})
|
|
51
|
+
]
|
|
52
|
+
})
|
|
53
|
+
]
|
|
54
|
+
})
|
|
55
|
+
]
|
|
56
|
+
}),
|
|
57
|
+
callback
|
|
58
|
+
);
|
|
59
|
+
expect(metrics.scopeMetrics[0].metrics[0].dataPoints[0].attributes).toBe(attributes);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('forwards rather than merges data points whose attributes become identical', () => {
|
|
63
|
+
const exportMetrics = vi.fn<ExportMetrics>();
|
|
64
|
+
const delegate = {
|
|
65
|
+
export: exportMetrics,
|
|
66
|
+
forceFlush: vi.fn(async () => {}),
|
|
67
|
+
shutdown: vi.fn(async () => {})
|
|
68
|
+
} satisfies PushMetricExporter;
|
|
69
|
+
const exporter = new MetricAttributeFilteringExporter(delegate, new Set(['filtered']));
|
|
70
|
+
const metrics = {
|
|
71
|
+
resource: {},
|
|
72
|
+
scopeMetrics: [
|
|
73
|
+
{
|
|
74
|
+
scope: {},
|
|
75
|
+
metrics: [
|
|
76
|
+
{
|
|
77
|
+
dataPointType: DataPointType.GAUGE,
|
|
78
|
+
dataPoints: [
|
|
79
|
+
{ attributes: { retained: 'same', filtered: 'a' }, value: 10 },
|
|
80
|
+
{ attributes: { retained: 'same', filtered: 'b' }, value: 20 }
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
]
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
} as unknown as ResourceMetrics;
|
|
87
|
+
|
|
88
|
+
exporter.export(metrics, vi.fn());
|
|
89
|
+
|
|
90
|
+
const exported = exportMetrics.mock.calls[0][0];
|
|
91
|
+
expect(exported.scopeMetrics[0].metrics[0].dataPoints).toMatchObject([
|
|
92
|
+
{ attributes: { retained: 'same' }, value: 10 },
|
|
93
|
+
{ attributes: { retained: 'same' }, value: 20 }
|
|
94
|
+
]);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_HYDRATION_STATE,
|
|
3
|
+
DEFAULT_TAG,
|
|
4
|
+
nodeSqlite,
|
|
5
|
+
PrecompiledSyncConfig,
|
|
6
|
+
serializeSyncPlan,
|
|
7
|
+
SqlSyncRules
|
|
8
|
+
} from '@powersync/service-sync-rules';
|
|
9
|
+
import * as sqlite from 'node:sqlite';
|
|
10
|
+
import { describe, expect, test } from 'vitest';
|
|
11
|
+
import {
|
|
12
|
+
parsePersistedSyncConfigContent,
|
|
13
|
+
SerializedSyncPlan,
|
|
14
|
+
updateSyncRulesFromConfig
|
|
15
|
+
} from '../../src/storage/storage-index.js';
|
|
16
|
+
|
|
17
|
+
const EVENT_QUERY = 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true';
|
|
18
|
+
const yamlWithoutEvents = `
|
|
19
|
+
config:
|
|
20
|
+
edition: 3
|
|
21
|
+
|
|
22
|
+
streams:
|
|
23
|
+
checkpoints:
|
|
24
|
+
query: SELECT * FROM checkpoints
|
|
25
|
+
`;
|
|
26
|
+
const yamlWithEvents = `${yamlWithoutEvents}
|
|
27
|
+
event_definitions:
|
|
28
|
+
write_checkpoints:
|
|
29
|
+
payloads:
|
|
30
|
+
- ${EVENT_QUERY}
|
|
31
|
+
`;
|
|
32
|
+
|
|
33
|
+
describe('persisted compiled replication events', () => {
|
|
34
|
+
test('dual-writes additive compiled events and restores the compiled evaluator', () => {
|
|
35
|
+
const parsed = SqlSyncRules.fromYaml(yamlWithEvents, { defaultSchema: 'test_schema' });
|
|
36
|
+
const update = updateSyncRulesFromConfig(parsed);
|
|
37
|
+
const compiled = update.config.plan!;
|
|
38
|
+
const originalEvent = compiled.plan.events![0];
|
|
39
|
+
|
|
40
|
+
expect(compiled.plan.version).toBeLessThanOrEqual(2);
|
|
41
|
+
expect(compiled.plan.events).toHaveLength(1);
|
|
42
|
+
expect(originalEvent.name).toBe('write_checkpoints');
|
|
43
|
+
expect(compiled.eventDescriptors).toEqual({ write_checkpoints: [EVENT_QUERY] });
|
|
44
|
+
|
|
45
|
+
const restored = restore(compiled);
|
|
46
|
+
expect(restored.config).toBeInstanceOf(PrecompiledSyncConfig);
|
|
47
|
+
expect(restored.config).not.toHaveProperty('eventDescriptors');
|
|
48
|
+
expect(restored.config.eventDefinitions).toHaveLength(1);
|
|
49
|
+
expect(restored.config.eventDefinitions[0].name).toBe('write_checkpoints');
|
|
50
|
+
|
|
51
|
+
const hydrated = restored.config.hydrate({
|
|
52
|
+
hydrationState: DEFAULT_HYDRATION_STATE,
|
|
53
|
+
sqlite: nodeSqlite(sqlite)
|
|
54
|
+
});
|
|
55
|
+
expect(
|
|
56
|
+
hydrated.eventDescriptors[0].evaluateRowWithErrors({
|
|
57
|
+
sourceTable: { connectionTag: DEFAULT_TAG, schema: 'test_schema', name: 'checkpoints' },
|
|
58
|
+
record: { user_id: 'user-1', checkpoint: 4n, active: 1 }
|
|
59
|
+
})
|
|
60
|
+
).toEqual({ results: [{ data: { user_id: 'user-1', checkpoint: 4n } }], errors: [] });
|
|
61
|
+
|
|
62
|
+
// This represents what an older compiler sees after ignoring the additive plan.events field. The raw descriptor
|
|
63
|
+
// mirror is normalized into the compiled representation by the new loading boundary, retaining the legacy behavior
|
|
64
|
+
// of ignoring payload filters until the sync config is redeployed.
|
|
65
|
+
const { events: _ignored, ...planWithoutCompiledEvents } = compiled.plan;
|
|
66
|
+
const legacyView = restore({ ...compiled, plan: planWithoutCompiledEvents });
|
|
67
|
+
expect(legacyView.config).not.toHaveProperty('eventDescriptors');
|
|
68
|
+
expect(legacyView.config.eventDefinitions).toHaveLength(1);
|
|
69
|
+
expect(legacyView.config.eventDefinitions[0].name).toBe('write_checkpoints');
|
|
70
|
+
expect(legacyView.errors).toHaveLength(1);
|
|
71
|
+
expect(legacyView.errors[0]).toMatchObject({ type: 'warning' });
|
|
72
|
+
|
|
73
|
+
const legacyEvent = serializeSyncPlan((legacyView.config as PrecompiledSyncConfig).plan).events![0];
|
|
74
|
+
expect(legacyEvent.sourceQueries[0].variants[0].filters).toEqual([]);
|
|
75
|
+
expect(originalEvent.sourceQueries[0].variants[0].filters).not.toEqual([]);
|
|
76
|
+
|
|
77
|
+
const legacyHydrated = legacyView.config.hydrate({
|
|
78
|
+
hydrationState: DEFAULT_HYDRATION_STATE,
|
|
79
|
+
sqlite: nodeSqlite(sqlite)
|
|
80
|
+
});
|
|
81
|
+
expect(
|
|
82
|
+
legacyHydrated.eventDescriptors[0].evaluateRowWithErrors({
|
|
83
|
+
sourceTable: { connectionTag: DEFAULT_TAG, schema: 'test_schema', name: 'checkpoints' },
|
|
84
|
+
record: { user_id: 'user-1', checkpoint: 4n, active: 0 }
|
|
85
|
+
})
|
|
86
|
+
).toEqual({ results: [{ data: { user_id: 'user-1', checkpoint: 4n } }], errors: [] });
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('restores raw event descriptors attached to version 1 and 2 plans', () => {
|
|
90
|
+
const parsed = SqlSyncRules.fromYaml(yamlWithoutEvents, { defaultSchema: 'test_schema' });
|
|
91
|
+
const update = updateSyncRulesFromConfig(parsed);
|
|
92
|
+
const legacy: SerializedSyncPlan = {
|
|
93
|
+
...update.config.plan!,
|
|
94
|
+
eventDescriptors: { write_checkpoints: [EVENT_QUERY] }
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
expect(legacy.plan.version).toBeLessThanOrEqual(2);
|
|
98
|
+
const restored = restore(legacy);
|
|
99
|
+
expect(restored.config).not.toHaveProperty('eventDescriptors');
|
|
100
|
+
expect(restored.config.eventDefinitions).toHaveLength(1);
|
|
101
|
+
expect((restored.config as PrecompiledSyncConfig).plan.events).toMatchObject([
|
|
102
|
+
{ name: 'write_checkpoints', sourceQueries: [{ sql: EVENT_QUERY }] }
|
|
103
|
+
]);
|
|
104
|
+
expect(restored.errors).toHaveLength(1);
|
|
105
|
+
expect(restored.errors[0]).toMatchObject({ type: 'warning' });
|
|
106
|
+
expect(
|
|
107
|
+
serializeSyncPlan((restored.config as PrecompiledSyncConfig).plan).events![0].sourceQueries[0].variants[0].filters
|
|
108
|
+
).toEqual([]);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Event SQL accepted by the legacy evaluator must remain loadable after an upgrade, even when its ignored filter uses
|
|
113
|
+
* expressions that the compiled event evaluator intentionally rejects for new deployments.
|
|
114
|
+
*/
|
|
115
|
+
test.each([
|
|
116
|
+
['request parameters', 'SELECT user_id FROM checkpoints WHERE user_id = request.user_id()'],
|
|
117
|
+
['unsupported expressions', 'SELECT user_id FROM checkpoints WHERE active IN (1, 2)']
|
|
118
|
+
])('loads legacy event descriptors without validating ignored %s', (_description, eventQuery) => {
|
|
119
|
+
const parsed = SqlSyncRules.fromYaml(yamlWithoutEvents, { defaultSchema: 'test_schema' });
|
|
120
|
+
const update = updateSyncRulesFromConfig(parsed);
|
|
121
|
+
const legacy: SerializedSyncPlan = {
|
|
122
|
+
...update.config.plan!,
|
|
123
|
+
eventDescriptors: {
|
|
124
|
+
write_checkpoints: [eventQuery]
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const restored = restore(legacy);
|
|
129
|
+
expect(restored.errors).toHaveLength(1);
|
|
130
|
+
expect(restored.errors[0]).toMatchObject({ type: 'warning' });
|
|
131
|
+
|
|
132
|
+
const event = restored.config.hydrate({
|
|
133
|
+
hydrationState: DEFAULT_HYDRATION_STATE,
|
|
134
|
+
sqlite: nodeSqlite(sqlite)
|
|
135
|
+
}).eventDescriptors[0];
|
|
136
|
+
expect(
|
|
137
|
+
event.evaluateRowWithErrors({
|
|
138
|
+
sourceTable: { connectionTag: DEFAULT_TAG, schema: 'test_schema', name: 'checkpoints' },
|
|
139
|
+
record: { user_id: 'user-1' }
|
|
140
|
+
})
|
|
141
|
+
).toEqual({ results: [{ data: { user_id: 'user-1' } }], errors: [] });
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
function restore(compiledPlan: SerializedSyncPlan) {
|
|
146
|
+
return parsePersistedSyncConfigContent({
|
|
147
|
+
content: yamlWithEvents,
|
|
148
|
+
compiledPlan,
|
|
149
|
+
storageVersion: 1,
|
|
150
|
+
parseOptions: { defaultSchema: 'test_schema' }
|
|
151
|
+
});
|
|
152
|
+
}
|
package/test/src/config.test.ts
CHANGED
|
@@ -70,6 +70,44 @@ describe('Config', () => {
|
|
|
70
70
|
expect(config.api_parameters.max_buckets_per_connection).toBe(1);
|
|
71
71
|
});
|
|
72
72
|
|
|
73
|
+
it('should resolve the storage default version', {}, async () => {
|
|
74
|
+
const yamlConfig = /* yaml */ `
|
|
75
|
+
# PowerSync config
|
|
76
|
+
replication:
|
|
77
|
+
connections: []
|
|
78
|
+
storage:
|
|
79
|
+
type: mongodb
|
|
80
|
+
default_storage_version: 4
|
|
81
|
+
`;
|
|
82
|
+
|
|
83
|
+
const collector = new CompoundConfigCollector();
|
|
84
|
+
|
|
85
|
+
const config = await collector.collectConfig({
|
|
86
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
expect(config.storage.default_storage_version).toBe(4);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it.each([1, 1.5, 5])('should reject unsupported storage default version %s', async (defaultStorageVersion) => {
|
|
93
|
+
const yamlConfig = /* yaml */ `
|
|
94
|
+
# PowerSync config
|
|
95
|
+
replication:
|
|
96
|
+
connections: []
|
|
97
|
+
storage:
|
|
98
|
+
type: mongodb
|
|
99
|
+
default_storage_version: ${defaultStorageVersion}
|
|
100
|
+
`;
|
|
101
|
+
|
|
102
|
+
const collector = new CompoundConfigCollector();
|
|
103
|
+
|
|
104
|
+
await expect(
|
|
105
|
+
collector.collectConfig({
|
|
106
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
107
|
+
})
|
|
108
|
+
).rejects.toThrow(`Storage version ${defaultStorageVersion} is not supported`);
|
|
109
|
+
});
|
|
110
|
+
|
|
73
111
|
it('should resolve checkpoint request retention config', {}, async () => {
|
|
74
112
|
const yamlConfig = /* yaml */ `
|
|
75
113
|
# PowerSync config
|
|
@@ -198,6 +236,7 @@ describe('Config', () => {
|
|
|
198
236
|
storage:
|
|
199
237
|
type: mongodb
|
|
200
238
|
sync_config:
|
|
239
|
+
version_label: v6
|
|
201
240
|
content: |
|
|
202
241
|
config:
|
|
203
242
|
edition: 2
|
|
@@ -214,6 +253,7 @@ describe('Config', () => {
|
|
|
214
253
|
|
|
215
254
|
expect(result.sync_rules).toEqual({
|
|
216
255
|
present: true,
|
|
256
|
+
version_label: 'v6',
|
|
217
257
|
exit_on_error: true,
|
|
218
258
|
content: expect.stringContaining('edition: 2')
|
|
219
259
|
});
|
|
@@ -15,6 +15,7 @@ bucket_definitions:
|
|
|
15
15
|
|
|
16
16
|
function makeSyncRulesContent(overrides?: {
|
|
17
17
|
slot_name?: string;
|
|
18
|
+
version_label?: string;
|
|
18
19
|
status?: storage.PersistedSyncConfigStatus;
|
|
19
20
|
}): storage.PersistedSyncConfigContent {
|
|
20
21
|
// We don't implement the entire interface correctly here - just enough to test the diagnostics logic.
|
|
@@ -37,6 +38,7 @@ function makeSyncRulesContent(overrides?: {
|
|
|
37
38
|
compiled_plan: null,
|
|
38
39
|
storageVersion: 1,
|
|
39
40
|
syncConfigState: status.state,
|
|
41
|
+
version_label: overrides?.version_label,
|
|
40
42
|
parsed(options?: any) {
|
|
41
43
|
const syncRules = SqlSyncRules.fromYaml(MINIMAL_SYNC_RULES, {
|
|
42
44
|
...options,
|
|
@@ -180,4 +182,27 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
180
182
|
expect(result!.connections[0].last_keepalive_ts).toBe('2026-01-01T00:01:00.000Z');
|
|
181
183
|
expect(result!.errors.some((error) => error.message == 'config failed')).toBe(true);
|
|
182
184
|
});
|
|
185
|
+
|
|
186
|
+
test('includes the sync config version label', async () => {
|
|
187
|
+
const result = await getSyncRulesStatus(
|
|
188
|
+
makeRouteAPI(),
|
|
189
|
+
makeSyncRulesContent({ version_label: 'v6' }),
|
|
190
|
+
OPTIONS,
|
|
191
|
+
makeSystemStorage()
|
|
192
|
+
);
|
|
193
|
+
|
|
194
|
+
expect(result!.version_label).toBe('v6');
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
test('includes the sync config version label when parsing fails', async () => {
|
|
198
|
+
const content = makeSyncRulesContent({ version_label: 'v6' });
|
|
199
|
+
content.parsed = () => {
|
|
200
|
+
throw new Error('Invalid sync config');
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
const result = await getSyncRulesStatus(makeRouteAPI(), content, OPTIONS, makeSystemStorage());
|
|
204
|
+
|
|
205
|
+
expect(result!.version_label).toBe('v6');
|
|
206
|
+
expect(result!.errors).toEqual([expect.objectContaining({ level: 'fatal', message: 'Invalid sync config' })]);
|
|
207
|
+
});
|
|
183
208
|
});
|