@powersync/service-core 1.21.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/dist/api/RouteAPI.d.ts +2 -2
- package/dist/api/diagnostics.d.ts +12 -1
- package/dist/api/diagnostics.js +34 -22
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js +3 -2
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/entry/commands/compact-action.js +1 -1
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +6 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +2 -2
- package/dist/replication/AbstractReplicator.js +24 -23
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/RelationCache.d.ts +9 -2
- package/dist/replication/RelationCache.js +21 -2
- package/dist/replication/RelationCache.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +3 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +12 -12
- package/dist/routes/configure-fastify.js +3 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +24 -24
- package/dist/routes/endpoints/admin.js +49 -35
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +22 -13
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +1 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +2 -0
- package/dist/routes/route-register.js +65 -3
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +8 -8
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +29 -6
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +26 -27
- package/dist/storage/BucketStorageFactory.js +5 -19
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
- package/dist/storage/ParsedSyncConfigSet.js +2 -0
- package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/PersistedReplicationStream.d.ts +32 -0
- package/dist/storage/PersistedReplicationStream.js +26 -0
- package/dist/storage/PersistedReplicationStream.js.map +1 -0
- package/dist/storage/PersistedSyncConfigContent.d.ts +60 -0
- package/dist/storage/PersistedSyncConfigContent.js +123 -0
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
- package/dist/storage/PersistedSyncConfigStatus.js +2 -0
- package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
- package/dist/storage/SourceEntity.d.ts +8 -1
- package/dist/storage/SourceTable.d.ts +37 -10
- package/dist/storage/SourceTable.js +50 -14
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +80 -24
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +117 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js +277 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +62 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
- package/dist/storage/storage-index.d.ts +8 -1
- package/dist/storage/storage-index.js +8 -1
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +4 -4
- package/dist/sync/BucketChecksumState.js +1 -1
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -2
- package/dist/sync/sync.js +5 -0
- package/dist/sync/sync.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +17 -1
- package/dist/tracing/PerformanceTracer.js +3 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -1
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -1
- package/dist/util/lsn.d.ts +4 -0
- package/dist/util/lsn.js +10 -0
- package/dist/util/lsn.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +5 -0
- package/dist/util/utils.js +7 -0
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -7
- package/src/api/RouteAPI.ts +2 -2
- package/src/api/diagnostics.ts +45 -28
- package/src/auth/RemoteJWKSCollector.ts +3 -1
- package/src/entry/commands/compact-action.ts +1 -1
- package/src/modules/AbstractModule.ts +6 -2
- package/src/replication/AbstractReplicator.ts +27 -26
- package/src/replication/RelationCache.ts +23 -4
- package/src/replication/ReplicationModule.ts +2 -3
- package/src/routes/configure-fastify.ts +8 -1
- package/src/routes/endpoints/admin.ts +75 -49
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +23 -13
- package/src/routes/endpoints/sync-stream.ts +1 -1
- package/src/routes/route-register.ts +73 -4
- package/src/runner/teardown.ts +8 -8
- package/src/storage/BucketStorageBatch.ts +32 -7
- package/src/storage/BucketStorageFactory.ts +36 -40
- package/src/storage/ParsedSyncConfigSet.ts +25 -0
- package/src/storage/PersistedReplicationStream.ts +55 -0
- package/src/storage/PersistedSyncConfigContent.ts +188 -0
- package/src/storage/PersistedSyncConfigStatus.ts +14 -0
- package/src/storage/SourceEntity.ts +9 -1
- package/src/storage/SourceTable.ts +72 -21
- package/src/storage/SyncRulesBucketStorage.ts +87 -26
- package/src/storage/implementation/BucketDefinitionMapping.ts +436 -0
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +109 -0
- package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
- package/src/storage/storage-index.ts +9 -1
- package/src/sync/BucketChecksumState.ts +5 -5
- package/src/sync/sync.ts +8 -3
- package/src/tracing/PerformanceTracer.ts +24 -1
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/types.ts +1 -1
- package/src/util/lsn.ts +9 -0
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +8 -0
- package/test/src/auth.test.ts +11 -0
- package/test/src/diagnostics.test.ts +64 -33
- package/test/src/module-loader.test.ts +33 -1
- package/test/src/routes/admin.test.ts +180 -18
- package/test/src/routes/error-handler.integration.test.ts +275 -0
- package/test/src/routes/mocks.ts +6 -2
- package/test/src/routes/stream.test.ts +15 -4
- package/test/src/storage/SourceTable.test.ts +89 -0
- package/test/src/sync/BucketChecksumState.test.ts +25 -17
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
- package/dist/storage/PersistedSyncRulesContent.js +0 -100
- package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
- package/src/storage/PersistedSyncRulesContent.ts +0 -174
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules';
|
|
2
|
+
import {
|
|
3
|
+
IncrementalMappingChanges,
|
|
4
|
+
IncrementalMappingDefinitionChange,
|
|
5
|
+
SingleSyncConfigBucketDefinitionMapping
|
|
6
|
+
} from './BucketDefinitionMapping.js';
|
|
7
|
+
|
|
8
|
+
export interface IncrementalDefinitionLogEntry extends IncrementalMappingDefinitionChange {
|
|
9
|
+
sourceTables?: string[];
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface IncrementalSyncConfigUpdateLog {
|
|
13
|
+
reusedDefinitions: IncrementalDefinitionLogEntry[];
|
|
14
|
+
addedDefinitions: IncrementalDefinitionLogEntry[];
|
|
15
|
+
droppedDefinitions: IncrementalDefinitionLogEntry[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function formatIncrementalSyncConfigUpdateLog(log: IncrementalSyncConfigUpdateLog) {
|
|
19
|
+
return [
|
|
20
|
+
formatDefinitionSection('Reused definitions', log.reusedDefinitions),
|
|
21
|
+
formatDefinitionSection('New definitions', log.addedDefinitions, { includeSnapshotTables: true }),
|
|
22
|
+
formatDefinitionSection('Definitions to drop after switching', log.droppedDefinitions)
|
|
23
|
+
].join('\n');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function describeIncrementalSyncConfigUpdate(options: {
|
|
27
|
+
activeMappings: SingleSyncConfigBucketDefinitionMapping[];
|
|
28
|
+
newMapping: SingleSyncConfigBucketDefinitionMapping;
|
|
29
|
+
newSyncConfig: SyncConfigWithErrors;
|
|
30
|
+
mappingChanges: IncrementalMappingChanges;
|
|
31
|
+
}): IncrementalSyncConfigUpdateLog {
|
|
32
|
+
const { activeMappings, newMapping, newSyncConfig, mappingChanges } = options;
|
|
33
|
+
const newDefinitionKeys = new Set(newMapping.allDefinitionEntries().map(definitionKey));
|
|
34
|
+
const activeDefinitions = uniqueDefinitions(activeMappings.flatMap((mapping) => mapping.allDefinitionEntries()));
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
reusedDefinitions: mappingChanges.reusedDefinitions,
|
|
38
|
+
addedDefinitions: mappingChanges.addedDefinitions.map((definition) => {
|
|
39
|
+
const sourceTables = sourceTablesForDefinition(newSyncConfig, newMapping, definition);
|
|
40
|
+
return {
|
|
41
|
+
...definition,
|
|
42
|
+
sourceTables
|
|
43
|
+
};
|
|
44
|
+
}),
|
|
45
|
+
droppedDefinitions: activeDefinitions.filter((definition) => !newDefinitionKeys.has(definitionKey(definition)))
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function sourceTablesForDefinition(
|
|
50
|
+
syncConfig: SyncConfigWithErrors,
|
|
51
|
+
mapping: SingleSyncConfigBucketDefinitionMapping,
|
|
52
|
+
definition: IncrementalMappingDefinitionChange
|
|
53
|
+
) {
|
|
54
|
+
if (definition.type == 'bucket_data') {
|
|
55
|
+
return sourceTablesForSources(
|
|
56
|
+
syncConfig.config.bucketDataSources.filter((source) => mapping.bucketSourceId(source) == definition.id)
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return sourceTablesForSources(
|
|
61
|
+
syncConfig.config.bucketParameterLookupSources.filter(
|
|
62
|
+
(source) => mapping.parameterLookupId(source) == definition.id
|
|
63
|
+
)
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function sourceTablesForSources(sources: Array<BucketDataSource | ParameterIndexLookupCreator>) {
|
|
68
|
+
return uniqueSorted(sources.flatMap((source) => [...source.getSourceTables()].map((table) => table.tablePattern)));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function definitionKey(definition: IncrementalMappingDefinitionChange) {
|
|
72
|
+
return `${definition.type}:${definition.id}`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function uniqueSorted(values: string[]) {
|
|
76
|
+
return [...new Set(values)].sort();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function uniqueDefinitions(definitions: IncrementalMappingDefinitionChange[]) {
|
|
80
|
+
const byKey = new Map<string, IncrementalMappingDefinitionChange>();
|
|
81
|
+
for (const definition of definitions) {
|
|
82
|
+
byKey.set(definitionKey(definition), definition);
|
|
83
|
+
}
|
|
84
|
+
return [...byKey.values()];
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function formatDefinitionSection(
|
|
88
|
+
title: string,
|
|
89
|
+
definitions: IncrementalDefinitionLogEntry[],
|
|
90
|
+
options: { includeSnapshotTables?: boolean } = {}
|
|
91
|
+
) {
|
|
92
|
+
if (definitions.length == 0) {
|
|
93
|
+
return `${title}: none`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return [`${title}:`, ...definitions.map((definition) => ` - ${formatDefinition(definition, options)}`)].join('\n');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function formatDefinition(definition: IncrementalDefinitionLogEntry, options: { includeSnapshotTables?: boolean }) {
|
|
100
|
+
const details = [`type=${definition.type}`, `id=${definition.id}`, `name=${definition.name}`];
|
|
101
|
+
if (definition.sourceTables != null) {
|
|
102
|
+
details.push(`tables=${formatList(definition.sourceTables)}`);
|
|
103
|
+
}
|
|
104
|
+
return details.join(', ');
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function formatList(values: string[]) {
|
|
108
|
+
return values.length == 0 ? 'none' : values.join(', ');
|
|
109
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
|
|
2
|
+
import { SingleSyncConfigBucketDefinitionMapping } from './BucketDefinitionMapping.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Persisted storage ids for a replication stream, derived from the rule_mapping documents
|
|
6
|
+
* of all its sync configs.
|
|
7
|
+
*
|
|
8
|
+
* This requires no parsed sync configs, and deliberately does not resolve individual
|
|
9
|
+
* sources - use the parsed-set-bound {@link BucketDefinitionMapping} for that.
|
|
10
|
+
*/
|
|
11
|
+
export class ReplicationStreamStorageIds {
|
|
12
|
+
constructor(private readonly mappings: SingleSyncConfigBucketDefinitionMapping[]) {}
|
|
13
|
+
|
|
14
|
+
get bucketDefinitionIds(): BucketDefinitionId[] {
|
|
15
|
+
return [...new Set(this.mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
get parameterIndexIds(): ParameterIndexId[] {
|
|
19
|
+
return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))];
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -3,7 +3,10 @@ export * from './BucketStorage.js';
|
|
|
3
3
|
export * from './BucketStorageBatch.js';
|
|
4
4
|
export * from './BucketStorageFactory.js';
|
|
5
5
|
export * from './ChecksumCache.js';
|
|
6
|
-
export * from './
|
|
6
|
+
export * from './ParsedSyncConfigSet.js';
|
|
7
|
+
export * from './PersistedReplicationStream.js';
|
|
8
|
+
export * from './PersistedSyncConfigContent.js';
|
|
9
|
+
export * from './PersistedSyncConfigStatus.js';
|
|
7
10
|
export * from './ReplicationEventPayload.js';
|
|
8
11
|
export * from './ReplicationLock.js';
|
|
9
12
|
export * from './ReportStorage.js';
|
|
@@ -15,3 +18,8 @@ export * from './StorageProvider.js';
|
|
|
15
18
|
export * from './StorageVersionConfig.js';
|
|
16
19
|
export * from './SyncRulesBucketStorage.js';
|
|
17
20
|
export * from './WriteCheckpointAPI.js';
|
|
21
|
+
|
|
22
|
+
export * from './implementation/BucketDefinitionMapping.js';
|
|
23
|
+
export * from './implementation/IncrementalReprocessingCompatibility.js';
|
|
24
|
+
export * from './implementation/IncrementalReprocessingSyncConfigLog.js';
|
|
25
|
+
export * from './implementation/ReplicationStreamStorageIds.js';
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
BucketParameterQuerier,
|
|
3
3
|
BucketPriority,
|
|
4
4
|
BucketSource,
|
|
5
|
-
|
|
5
|
+
HydratedSyncConfig,
|
|
6
6
|
mergeBuckets,
|
|
7
7
|
QuerierError,
|
|
8
8
|
RequestedStream,
|
|
@@ -28,7 +28,7 @@ import { getIntersection, hasIntersection } from './util.js';
|
|
|
28
28
|
export interface BucketChecksumStateOptions {
|
|
29
29
|
syncContext: SyncContext;
|
|
30
30
|
bucketStorage: BucketChecksumStateStorage;
|
|
31
|
-
syncRules:
|
|
31
|
+
syncRules: HydratedSyncConfig;
|
|
32
32
|
tokenPayload: JwtPayload;
|
|
33
33
|
syncRequest: util.StreamingSyncRequest;
|
|
34
34
|
logger?: Logger;
|
|
@@ -285,7 +285,7 @@ export class BucketChecksumState {
|
|
|
285
285
|
const streamNameToIndex = new Map<string, number>();
|
|
286
286
|
this.streamNameToIndex = streamNameToIndex;
|
|
287
287
|
|
|
288
|
-
for (const source of this.parameterState.syncRules.
|
|
288
|
+
for (const source of this.parameterState.syncRules.bucketSourceDefinitions) {
|
|
289
289
|
if (this.parameterState.isSubscribedToStream(source)) {
|
|
290
290
|
streamNameToIndex.set(source.name, subscriptions.length);
|
|
291
291
|
|
|
@@ -416,7 +416,7 @@ export interface CheckpointUpdate {
|
|
|
416
416
|
export class BucketParameterState {
|
|
417
417
|
private readonly context: SyncContext;
|
|
418
418
|
public readonly bucketStorage: BucketChecksumStateStorage;
|
|
419
|
-
public readonly syncRules:
|
|
419
|
+
public readonly syncRules: HydratedSyncConfig;
|
|
420
420
|
public readonly syncParams: RequestParameters;
|
|
421
421
|
private readonly querier: BucketParameterQuerier;
|
|
422
422
|
/**
|
|
@@ -439,7 +439,7 @@ export class BucketParameterState {
|
|
|
439
439
|
constructor(
|
|
440
440
|
context: SyncContext,
|
|
441
441
|
bucketStorage: BucketChecksumStateStorage,
|
|
442
|
-
syncRules:
|
|
442
|
+
syncRules: HydratedSyncConfig,
|
|
443
443
|
tokenPayload: JwtPayload,
|
|
444
444
|
request: util.StreamingSyncRequest,
|
|
445
445
|
logger: Logger
|
package/src/sync/sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
|
-
import { BucketPriority,
|
|
2
|
+
import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
3
|
|
|
4
4
|
import { AbortError } from 'ix/aborterror.js';
|
|
5
5
|
|
|
@@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr
|
|
|
17
17
|
export interface SyncStreamParameters {
|
|
18
18
|
syncContext: SyncContext;
|
|
19
19
|
bucketStorage: storage.SyncRulesBucketStorage;
|
|
20
|
-
syncRules:
|
|
20
|
+
syncRules: HydratedSyncConfig;
|
|
21
21
|
params: util.StreamingSyncRequest;
|
|
22
22
|
token: auth.JwtPayload;
|
|
23
23
|
logger?: Logger;
|
|
@@ -94,7 +94,7 @@ export async function* streamResponse(
|
|
|
94
94
|
async function* streamResponseInner(
|
|
95
95
|
syncContext: SyncContext,
|
|
96
96
|
bucketStorage: storage.SyncRulesBucketStorage,
|
|
97
|
-
syncRules:
|
|
97
|
+
syncRules: HydratedSyncConfig,
|
|
98
98
|
params: util.StreamingSyncRequest,
|
|
99
99
|
tokenPayload: auth.JwtPayload,
|
|
100
100
|
tracker: RequestTracker,
|
|
@@ -367,6 +367,11 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
367
367
|
checkpointInvalidated = true;
|
|
368
368
|
}
|
|
369
369
|
if (r.data.length == 0) {
|
|
370
|
+
// An empty chunk is not sent to the client, but may still report progress:
|
|
371
|
+
// storage can emit an empty chunk for a bucket that has no operations in the
|
|
372
|
+
// requested range, and its position must advance so the bucket is not
|
|
373
|
+
// re-requested indefinitely.
|
|
374
|
+
checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
|
|
370
375
|
continue;
|
|
371
376
|
}
|
|
372
377
|
logger.debug(`Sending data for ${r.bucket}`);
|
|
@@ -2,19 +2,38 @@ import { traceWriter } from './TraceWriter.js';
|
|
|
2
2
|
|
|
3
3
|
export interface Span extends Disposable {
|
|
4
4
|
name: string;
|
|
5
|
+
/**
|
|
6
|
+
* Start time in microseconds since an arbitrary epoch.
|
|
7
|
+
*/
|
|
5
8
|
startAt: number;
|
|
9
|
+
/**
|
|
10
|
+
* End time in microseconds since an arbitrary epoch.
|
|
11
|
+
*/
|
|
6
12
|
endAt: number;
|
|
13
|
+
/**
|
|
14
|
+
* Time spent not in nested spans.
|
|
15
|
+
*/
|
|
7
16
|
selfDuration: number;
|
|
17
|
+
|
|
8
18
|
nestedSince: number | undefined;
|
|
9
19
|
subtrackFromSelf: number;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Durations spent in nested spans, in microseconds.
|
|
23
|
+
*/
|
|
10
24
|
nestedDurations: Record<string, number>;
|
|
11
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Total duration of this span in milliseconds, rounded up. Only valid after the span has ended.
|
|
28
|
+
*/
|
|
29
|
+
durationMillis: number;
|
|
30
|
+
|
|
12
31
|
/**
|
|
13
32
|
* End the span - same as [Symbol.dispose]().
|
|
14
33
|
*
|
|
15
34
|
* Safe to call multiple times. Any nested spans will automatically end as well.
|
|
16
35
|
*
|
|
17
|
-
* Returns an aggregate record of category -> "selfDuration".
|
|
36
|
+
* Returns an aggregate record of category -> "selfDuration", in microseconds.
|
|
18
37
|
*/
|
|
19
38
|
end(): Record<string, number>;
|
|
20
39
|
}
|
|
@@ -115,6 +134,10 @@ export class PerformanceTracer<K extends string> {
|
|
|
115
134
|
}
|
|
116
135
|
return this.nestedDurations;
|
|
117
136
|
},
|
|
137
|
+
|
|
138
|
+
get durationMillis() {
|
|
139
|
+
return Math.ceil((this.endAt - this.startAt) / 1000);
|
|
140
|
+
},
|
|
118
141
|
[Symbol.dispose]() {
|
|
119
142
|
this.end();
|
|
120
143
|
}
|
|
@@ -11,7 +11,7 @@ export interface CreateWriteCheckpointOptions {
|
|
|
11
11
|
export async function createWriteCheckpoint(options: CreateWriteCheckpointOptions) {
|
|
12
12
|
const full_user_id = checkpointUserId(options.userId, options.clientId);
|
|
13
13
|
|
|
14
|
-
const syncBucketStorage = await options.storage.
|
|
14
|
+
const syncBucketStorage = (await options.storage.getActiveSyncConfig())?.storage;
|
|
15
15
|
if (!syncBucketStorage) {
|
|
16
16
|
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
17
17
|
}
|
package/src/util/config/types.ts
CHANGED
|
@@ -55,7 +55,7 @@ export type ResolvedPowerSyncConfig = {
|
|
|
55
55
|
max_parameter_query_results: number;
|
|
56
56
|
};
|
|
57
57
|
|
|
58
|
-
/** Prefix for postgres replication slot names
|
|
58
|
+
/** Prefix for postgres replication slot names, as well as replication stream names. */
|
|
59
59
|
slot_name_prefix: string;
|
|
60
60
|
|
|
61
61
|
healthcheck: {
|
package/src/util/lsn.ts
CHANGED
|
@@ -6,3 +6,12 @@ export function maxLsn(a: string | null | undefined, b: string | null | undefine
|
|
|
6
6
|
if (b == null) return a;
|
|
7
7
|
return a > b ? a : b;
|
|
8
8
|
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Return the smaller of two LSNs, ignoring nulls.
|
|
12
|
+
*/
|
|
13
|
+
export function minLsn(a: string | null | undefined, b: string | null | undefined): string | null {
|
|
14
|
+
if (a == null) return b ?? null;
|
|
15
|
+
if (b == null) return a;
|
|
16
|
+
return a < b ? a : b;
|
|
17
|
+
}
|
package/src/util/util-index.ts
CHANGED
|
@@ -17,6 +17,7 @@ export * from './config/collectors/config-collector.js';
|
|
|
17
17
|
export * from './config/collectors/impl/base64-config-collector.js';
|
|
18
18
|
export * from './config/collectors/impl/fallback-config-collector.js';
|
|
19
19
|
export * from './config/collectors/impl/filesystem-config-collector.js';
|
|
20
|
+
export * from './config/collectors/impl/yaml-env.js';
|
|
20
21
|
|
|
21
22
|
export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
|
|
22
23
|
export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';
|
package/src/util/utils.ts
CHANGED
|
@@ -39,6 +39,14 @@ export function escapeIdentifier(identifier: string) {
|
|
|
39
39
|
return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Sanitized name of the entity in the format of "{schema}.{entity name}"
|
|
44
|
+
* Suitable for safe use in Postgres queries and in log output.
|
|
45
|
+
*/
|
|
46
|
+
export function qualifiedName(ref: sync_rules.SourceTableRef) {
|
|
47
|
+
return `${escapeIdentifier(ref.schema)}.${escapeIdentifier(ref.name)}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
42
50
|
export function hashData(type: string, id: string, data: string): number {
|
|
43
51
|
const hash = crypto.createHash('sha256');
|
|
44
52
|
hash.update(`put.${type}.${id}.${data}`);
|
package/test/src/auth.test.ts
CHANGED
|
@@ -484,6 +484,17 @@ describe('JWT Auth', () => {
|
|
|
484
484
|
}
|
|
485
485
|
})
|
|
486
486
|
).toThrowError('IPs in this range are not supported');
|
|
487
|
+
|
|
488
|
+
// `URL.hostname` exposes IPv6 literals wrapped in brackets; ensure they are
|
|
489
|
+
// handled like other direct-IP cases.
|
|
490
|
+
expect(
|
|
491
|
+
() =>
|
|
492
|
+
new RemoteJWKSCollector('https://[::1]/.well-known/jwks.json', {
|
|
493
|
+
lookupOptions: {
|
|
494
|
+
reject_ip_ranges: ['local']
|
|
495
|
+
}
|
|
496
|
+
})
|
|
497
|
+
).toThrowError('IPs in this range are not supported');
|
|
487
498
|
});
|
|
488
499
|
|
|
489
500
|
test('http not blocking local IPs', async () => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DiagnosticsOptions, getSyncRulesStatus } from '@/api/diagnostics.js';
|
|
2
2
|
import { RouteAPI, SlotWalBudgetInfo } from '@/api/RouteAPI.js';
|
|
3
|
-
import {
|
|
3
|
+
import { ParsedSyncConfigSet, storage } from '@/index.js';
|
|
4
4
|
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
5
5
|
import { describe, expect, test } from 'vitest';
|
|
6
6
|
|
|
@@ -13,49 +13,56 @@ bucket_definitions:
|
|
|
13
13
|
- SELECT id FROM test_table
|
|
14
14
|
`;
|
|
15
15
|
|
|
16
|
-
function makeSyncRulesContent(overrides?: {
|
|
16
|
+
function makeSyncRulesContent(overrides?: {
|
|
17
|
+
slot_name?: string;
|
|
18
|
+
status?: storage.PersistedSyncConfigStatus;
|
|
19
|
+
}): storage.PersistedSyncConfigContent {
|
|
20
|
+
// We don't implement the entire interface correctly here - just enough to test the diagnostics logic.
|
|
17
21
|
return {
|
|
18
|
-
|
|
19
|
-
|
|
22
|
+
replicationStreamId: 1,
|
|
23
|
+
syncConfigId: null,
|
|
24
|
+
replicationStreamName: overrides?.slot_name ?? 'test_slot',
|
|
20
25
|
sync_rules_content: MINIMAL_SYNC_RULES,
|
|
21
26
|
compiled_plan: null,
|
|
22
|
-
active: true,
|
|
23
27
|
storageVersion: 1,
|
|
24
|
-
last_checkpoint_lsn: 'some_lsn',
|
|
25
|
-
last_fatal_error: null,
|
|
26
|
-
last_fatal_error_ts: null,
|
|
27
|
-
last_keepalive_ts: new Date(),
|
|
28
|
-
last_checkpoint_ts: new Date(),
|
|
29
28
|
parsed(options?: any) {
|
|
30
29
|
const syncRules = SqlSyncRules.fromYaml(MINIMAL_SYNC_RULES, {
|
|
31
30
|
...options,
|
|
32
31
|
defaultSchema: 'public'
|
|
33
32
|
});
|
|
34
33
|
return {
|
|
35
|
-
|
|
36
|
-
};
|
|
37
|
-
},
|
|
38
|
-
lock() {
|
|
39
|
-
throw new Error('Not implemented in mock');
|
|
34
|
+
syncConfigs: [syncRules]
|
|
35
|
+
} as ParsedSyncConfigSet;
|
|
40
36
|
},
|
|
41
|
-
|
|
42
|
-
|
|
37
|
+
logger: null as any,
|
|
38
|
+
asUpdateOptions: null as any,
|
|
39
|
+
getStorageConfig: null as any,
|
|
40
|
+
async getSyncConfigStatus() {
|
|
41
|
+
return (
|
|
42
|
+
overrides?.status ?? {
|
|
43
|
+
id: '1',
|
|
44
|
+
replicationStreamId: 1,
|
|
45
|
+
state: storage.SyncRuleState.ACTIVE,
|
|
46
|
+
last_checkpoint_lsn: 'some_lsn',
|
|
47
|
+
last_fatal_error: null,
|
|
48
|
+
last_fatal_error_ts: null,
|
|
49
|
+
last_keepalive_ts: new Date(),
|
|
50
|
+
last_checkpoint_ts: new Date()
|
|
51
|
+
}
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
} as storage.PersistedSyncConfigContent;
|
|
43
55
|
}
|
|
44
56
|
|
|
45
|
-
function
|
|
57
|
+
function makeSystemStorage() {
|
|
46
58
|
return {
|
|
47
|
-
|
|
59
|
+
async getStatus() {
|
|
48
60
|
return {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
snapshot_done: true,
|
|
52
|
-
checkpoint_lsn: 'some_lsn',
|
|
53
|
-
active: true
|
|
54
|
-
};
|
|
55
|
-
}
|
|
61
|
+
snapshotDone: true,
|
|
62
|
+
resumeLsn: 'some_lsn'
|
|
56
63
|
};
|
|
57
64
|
}
|
|
58
|
-
} as
|
|
65
|
+
} as storage.SyncRulesBucketStorage;
|
|
59
66
|
}
|
|
60
67
|
|
|
61
68
|
function makeRouteAPI(walBudget?: SlotWalBudgetInfo | undefined): RouteAPI {
|
|
@@ -88,7 +95,8 @@ function makeRouteAPI(walBudget?: SlotWalBudgetInfo | undefined): RouteAPI {
|
|
|
88
95
|
const OPTIONS: DiagnosticsOptions = {
|
|
89
96
|
live_status: true,
|
|
90
97
|
check_connection: true,
|
|
91
|
-
include_content: false
|
|
98
|
+
include_content: false,
|
|
99
|
+
active: true
|
|
92
100
|
};
|
|
93
101
|
|
|
94
102
|
describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
@@ -98,7 +106,7 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
98
106
|
safe_wal_size: 4 * GB,
|
|
99
107
|
max_slot_wal_keep_size: 10 * GB
|
|
100
108
|
});
|
|
101
|
-
const result = await getSyncRulesStatus(
|
|
109
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
102
110
|
const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
|
|
103
111
|
expect(walWarnings).toHaveLength(1);
|
|
104
112
|
expect(walWarnings[0].level).toBe('warning');
|
|
@@ -111,7 +119,7 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
111
119
|
safe_wal_size: 8 * GB,
|
|
112
120
|
max_slot_wal_keep_size: 10 * GB
|
|
113
121
|
});
|
|
114
|
-
const result = await getSyncRulesStatus(
|
|
122
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
115
123
|
const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
|
|
116
124
|
expect(walWarnings).toHaveLength(0);
|
|
117
125
|
});
|
|
@@ -122,7 +130,7 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
122
130
|
safe_wal_size: -2.4 * GB,
|
|
123
131
|
max_slot_wal_keep_size: 1 * 1024 * 1024 // 1MB
|
|
124
132
|
});
|
|
125
|
-
const result = await getSyncRulesStatus(
|
|
133
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
126
134
|
const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget'));
|
|
127
135
|
expect(walWarnings).toHaveLength(1);
|
|
128
136
|
expect(walWarnings[0].message).toContain('0%');
|
|
@@ -133,7 +141,7 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
133
141
|
const api = makeRouteAPI({
|
|
134
142
|
wal_status: 'lost'
|
|
135
143
|
});
|
|
136
|
-
const result = await getSyncRulesStatus(
|
|
144
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
137
145
|
const walErrors = result!.errors.filter(
|
|
138
146
|
(e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146')
|
|
139
147
|
);
|
|
@@ -142,10 +150,33 @@ describe('getSyncRulesStatus WAL budget warnings', () => {
|
|
|
142
150
|
|
|
143
151
|
test('no WAL error when getSlotWalBudget is not defined', async () => {
|
|
144
152
|
const api = makeRouteAPI();
|
|
145
|
-
const result = await getSyncRulesStatus(
|
|
153
|
+
const result = await getSyncRulesStatus(api, makeSyncRulesContent(), OPTIONS, makeSystemStorage());
|
|
146
154
|
const walErrors = result!.errors.filter(
|
|
147
155
|
(e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146')
|
|
148
156
|
);
|
|
149
157
|
expect(walErrors).toHaveLength(0);
|
|
150
158
|
});
|
|
159
|
+
|
|
160
|
+
test('uses sync config status for status-derived diagnostics fields', async () => {
|
|
161
|
+
const configStatus: storage.PersistedSyncConfigStatus = {
|
|
162
|
+
id: 'config-a',
|
|
163
|
+
replicationStreamId: 1,
|
|
164
|
+
state: storage.SyncRuleState.ACTIVE,
|
|
165
|
+
last_checkpoint_lsn: 'config_lsn',
|
|
166
|
+
last_fatal_error: 'config failed',
|
|
167
|
+
last_fatal_error_ts: new Date('2026-01-01T00:00:00.000Z'),
|
|
168
|
+
last_keepalive_ts: new Date('2026-01-01T00:01:00.000Z'),
|
|
169
|
+
last_checkpoint_ts: new Date('2026-01-01T00:02:00.000Z')
|
|
170
|
+
};
|
|
171
|
+
const result = await getSyncRulesStatus(
|
|
172
|
+
makeRouteAPI(),
|
|
173
|
+
makeSyncRulesContent({ status: configStatus }),
|
|
174
|
+
OPTIONS,
|
|
175
|
+
makeSystemStorage()
|
|
176
|
+
);
|
|
177
|
+
|
|
178
|
+
expect(result!.connections[0].last_checkpoint_ts).toBe('2026-01-01T00:02:00.000Z');
|
|
179
|
+
expect(result!.connections[0].last_keepalive_ts).toBe('2026-01-01T00:01:00.000Z');
|
|
180
|
+
expect(result!.errors.some((error) => error.message == 'config failed')).toBe(true);
|
|
181
|
+
});
|
|
151
182
|
});
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
AbstractModule,
|
|
3
|
+
loadModules,
|
|
4
|
+
ModuleManager,
|
|
5
|
+
ServiceContextContainer,
|
|
6
|
+
storage,
|
|
7
|
+
TearDownOptions
|
|
8
|
+
} from '@/index.js';
|
|
2
9
|
import { describe, expect, it } from 'vitest';
|
|
3
10
|
|
|
4
11
|
interface MockConfig {
|
|
@@ -27,6 +34,19 @@ class MockPostgresStorageModule extends AbstractModule {
|
|
|
27
34
|
async initialize(context: ServiceContextContainer): Promise<void> {}
|
|
28
35
|
async teardown(options: TearDownOptions): Promise<void> {}
|
|
29
36
|
}
|
|
37
|
+
class RecordingModule extends AbstractModule {
|
|
38
|
+
teardownOptions: TearDownOptions | undefined;
|
|
39
|
+
|
|
40
|
+
constructor(name: string) {
|
|
41
|
+
super({ name });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async initialize(context: ServiceContextContainer): Promise<void> {}
|
|
45
|
+
|
|
46
|
+
async teardown(options: TearDownOptions): Promise<void> {
|
|
47
|
+
this.teardownOptions = options;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
30
50
|
const mockLoaders = {
|
|
31
51
|
connection: {
|
|
32
52
|
mysql: async () => {
|
|
@@ -99,4 +119,16 @@ describe('module loader', () => {
|
|
|
99
119
|
|
|
100
120
|
await expect(loadModules(config as any, loaders)).rejects.toThrowError('Failed to load MySQL module');
|
|
101
121
|
});
|
|
122
|
+
|
|
123
|
+
it('passes replication streams through teardown options', async () => {
|
|
124
|
+
const module = new RecordingModule('RecordingModule');
|
|
125
|
+
const moduleManager = new ModuleManager();
|
|
126
|
+
const replicationStreams = [{ replicationStreamId: 1 }] as storage.PersistedReplicationStream[];
|
|
127
|
+
|
|
128
|
+
moduleManager.register([module]);
|
|
129
|
+
await moduleManager.tearDown({ replicationStreams, syncRules: replicationStreams });
|
|
130
|
+
|
|
131
|
+
expect(module.teardownOptions?.replicationStreams).toBe(replicationStreams);
|
|
132
|
+
expect(module.teardownOptions?.syncRules).toBe(replicationStreams);
|
|
133
|
+
});
|
|
102
134
|
});
|