@powersync/service-core 1.25.0 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/api/diagnostics.js +2 -0
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +28 -24
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +5 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +7 -3
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +16 -6
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +9 -0
- package/dist/routes/endpoints/admin.d.ts +18 -0
- package/dist/routes/endpoints/admin.js +3 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +3 -1
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +2 -1
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +28 -4
- package/dist/storage/BucketStorageFactory.js +8 -4
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.d.ts +2 -0
- package/dist/storage/PersistedSyncConfigContent.js +23 -10
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +6 -1
- package/dist/storage/SourceTable.d.ts +13 -5
- package/dist/storage/SourceTable.js +8 -4
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +4 -0
- package/dist/storage/StorageVersionConfig.js +10 -1
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +39 -7
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +25 -3
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +23 -7
- package/dist/storage/implementation/BucketDefinitionMapping.js +88 -14
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +9 -1
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -1
- package/dist/storage/storage-metrics.js +120 -4
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +1 -1
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +37 -9
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js +10 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +5 -0
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/dist/util/protocol-types.d.ts +4 -4
- package/dist/util/protocol-types.js +3 -2
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +1 -0
- package/dist/util/utils.js +15 -0
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -5
- package/src/api/diagnostics.ts +2 -0
- package/src/entry/commands/compact-action.ts +32 -24
- package/src/metrics/metrics-interfaces.ts +6 -1
- package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
- package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
- package/src/metrics/open-telemetry/util.ts +7 -3
- package/src/replication/AbstractReplicator.ts +22 -6
- package/src/routes/endpoints/admin.ts +3 -1
- package/src/routes/endpoints/sync-rules.ts +3 -1
- package/src/storage/BucketStorageBatch.ts +2 -1
- package/src/storage/BucketStorageFactory.ts +44 -8
- package/src/storage/PersistedSyncConfigContent.ts +26 -11
- package/src/storage/ReplicationEventPayload.ts +6 -1
- package/src/storage/SourceTable.ts +17 -4
- package/src/storage/StorageVersionConfig.ts +11 -1
- package/src/storage/SyncRulesBucketStorage.ts +47 -7
- package/src/storage/WriteCheckpointAPI.ts +32 -3
- package/src/storage/implementation/BucketDefinitionMapping.ts +120 -17
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +22 -5
- package/src/storage/storage-metrics.ts +154 -8
- package/src/sync/BucketChecksumState.ts +3 -1
- package/src/sync/sync.ts +40 -11
- package/src/sync/util.ts +21 -8
- package/src/util/config/compound-config-collector.ts +6 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
- package/src/util/config/types.ts +1 -0
- package/src/util/protocol-types.ts +3 -2
- package/src/util/utils.ts +14 -0
- package/test/src/AbstractReplicator.test.ts +30 -0
- package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
- package/test/src/PersistedSyncConfigContent.test.ts +152 -0
- package/test/src/config.test.ts +40 -0
- package/test/src/diagnostics.test.ts +25 -0
- package/test/src/routes/admin.test.ts +8 -2
- package/test/src/routes/stream.test.ts +123 -2
- package/test/src/storage/SourceTable.test.ts +4 -1
- package/test/src/storage-metrics.test.ts +148 -0
- package/test/src/sync/util.test.ts +15 -0
- package/test/src/util/utils.test.ts +23 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -2,11 +2,14 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
|
2
2
|
import {
|
|
3
3
|
BucketDataSource,
|
|
4
4
|
BucketDefinitionId,
|
|
5
|
+
EventDefinitionId,
|
|
5
6
|
HashMap,
|
|
6
7
|
ParameterIndexId,
|
|
7
8
|
ParameterIndexLookupCreator,
|
|
8
9
|
ParameterLookupDefinitionId,
|
|
9
10
|
SerializedBucketDataSourceWithDataSources,
|
|
11
|
+
serializedEventDefinitionEquality,
|
|
12
|
+
SerializedEventDescriptor,
|
|
10
13
|
SerializedParameterIndexLookupCreator,
|
|
11
14
|
serializedStreamBucketDataSourceEquality,
|
|
12
15
|
serializedStreamParameterIndexLookupCreatorEquality,
|
|
@@ -15,17 +18,22 @@ import {
|
|
|
15
18
|
SyncConfigWithErrors
|
|
16
19
|
} from '@powersync/service-sync-rules';
|
|
17
20
|
|
|
21
|
+
/** Minimal shape required to resolve an event's assigned id: its name is the per-config key. */
|
|
22
|
+
export interface NamedEventDefinition {
|
|
23
|
+
readonly name: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
18
26
|
export interface SerializedSyncConfigWithMapping {
|
|
19
27
|
plan: SerializedSyncPlan;
|
|
20
28
|
mapping: SingleSyncConfigBucketDefinitionMapping;
|
|
21
29
|
}
|
|
22
30
|
|
|
23
|
-
export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup';
|
|
31
|
+
export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup' | 'event';
|
|
24
32
|
|
|
25
33
|
export interface IncrementalMappingDefinitionChange {
|
|
26
34
|
type: IncrementalMappingDefinitionType;
|
|
27
35
|
name: string;
|
|
28
|
-
id: BucketDefinitionId | ParameterIndexId;
|
|
36
|
+
id: BucketDefinitionId | ParameterIndexId | EventDefinitionId;
|
|
29
37
|
}
|
|
30
38
|
|
|
31
39
|
export interface IncrementalMappingChanges {
|
|
@@ -58,6 +66,10 @@ export interface PersistedDefinitionMapping {
|
|
|
58
66
|
* Map of (lookupName, queryId) -> id, unique per replication stream.
|
|
59
67
|
*/
|
|
60
68
|
parameter_indexes: Record<string, string>;
|
|
69
|
+
/**
|
|
70
|
+
* Map of event name -> id, unique per replication stream. Absent for mappings persisted before events were tracked.
|
|
71
|
+
*/
|
|
72
|
+
events?: Record<string, string>;
|
|
61
73
|
}
|
|
62
74
|
|
|
63
75
|
/**
|
|
@@ -80,11 +92,14 @@ export interface BucketDefinitionMapping {
|
|
|
80
92
|
|
|
81
93
|
allParameterIndexIds(): ParameterIndexId[];
|
|
82
94
|
|
|
95
|
+
allEventDefinitionIds(): EventDefinitionId[];
|
|
96
|
+
|
|
83
97
|
syncConfigIdsForSourceTable(
|
|
84
98
|
selectedSyncConfigIds: string[],
|
|
85
99
|
table: SourceTableRef,
|
|
86
100
|
bucketDataSourceIds: BucketDefinitionId[],
|
|
87
|
-
parameterLookupSourceIds: ParameterIndexId[]
|
|
101
|
+
parameterLookupSourceIds: ParameterIndexId[],
|
|
102
|
+
eventDefinitionIds: EventDefinitionId[]
|
|
88
103
|
): string[];
|
|
89
104
|
|
|
90
105
|
snapshotBlockingSourceTablesFilter(syncConfigId: string): Record<string, unknown>;
|
|
@@ -100,7 +115,11 @@ export interface BucketDefinitionMapping {
|
|
|
100
115
|
*/
|
|
101
116
|
export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping {
|
|
102
117
|
static fromPersistedMapping(mapping: PersistedDefinitionMapping): SingleSyncConfigBucketDefinitionMapping {
|
|
103
|
-
return new SingleSyncConfigBucketDefinitionMapping(
|
|
118
|
+
return new SingleSyncConfigBucketDefinitionMapping(
|
|
119
|
+
mapping.definitions ?? {},
|
|
120
|
+
mapping.parameter_indexes ?? {},
|
|
121
|
+
mapping.events ?? {}
|
|
122
|
+
);
|
|
104
123
|
}
|
|
105
124
|
|
|
106
125
|
static fromParsedSyncConfig(syncConfig: SyncConfigWithErrors): SingleSyncConfigBucketDefinitionMapping {
|
|
@@ -108,9 +127,11 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
108
127
|
const parameterKeys = syncConfig.config.bucketParameterLookupSources
|
|
109
128
|
.map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`)
|
|
110
129
|
.sort();
|
|
130
|
+
const eventNames = syncConfig.config.eventDefinitions.map((event) => event.name).sort();
|
|
111
131
|
|
|
112
132
|
const definitions: Record<string, BucketDefinitionId> = {};
|
|
113
133
|
const parameterLookups: Record<string, ParameterIndexId> = {};
|
|
134
|
+
const events: Record<string, EventDefinitionId> = {};
|
|
114
135
|
|
|
115
136
|
for (const [index, uniqueName] of definitionNames.entries()) {
|
|
116
137
|
definitions[uniqueName] = (index + 1).toString(16);
|
|
@@ -118,8 +139,11 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
118
139
|
for (const [index, key] of parameterKeys.entries()) {
|
|
119
140
|
parameterLookups[key] = (index + 1).toString(16);
|
|
120
141
|
}
|
|
142
|
+
for (const [index, name] of eventNames.entries()) {
|
|
143
|
+
events[name] = (index + 1).toString(16);
|
|
144
|
+
}
|
|
121
145
|
|
|
122
|
-
return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups);
|
|
146
|
+
return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups, events);
|
|
123
147
|
}
|
|
124
148
|
|
|
125
149
|
/**
|
|
@@ -162,9 +186,25 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
162
186
|
nextParameterIndexId++;
|
|
163
187
|
return id;
|
|
164
188
|
}
|
|
189
|
+
// Event ids are deliberately allocated per replication stream instead of being hashes or UUIDs derived from a
|
|
190
|
+
// compiled plan. Compatibility decides whether to reuse an existing id, keeping the storage identifier separate
|
|
191
|
+
// from event content and allowing the comparison to evolve independently. That comparison must remain
|
|
192
|
+
// conservative: a false negative only causes a new id and resnapshot, while a false positive could incorrectly
|
|
193
|
+
// reuse existing event state.
|
|
194
|
+
let nextEventDefinitionId =
|
|
195
|
+
reservedMappings
|
|
196
|
+
.map((mapping) => mapping.allEventDefinitionIds())
|
|
197
|
+
.flat()
|
|
198
|
+
.reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1;
|
|
199
|
+
function generateNewEventDefinitionId(): EventDefinitionId {
|
|
200
|
+
const id = nextEventDefinitionId.toString(16);
|
|
201
|
+
nextEventDefinitionId++;
|
|
202
|
+
return id;
|
|
203
|
+
}
|
|
165
204
|
|
|
166
205
|
const definitions: Record<string, BucketDefinitionId> = {};
|
|
167
206
|
const parameterLookups: Record<string, ParameterIndexId> = {};
|
|
207
|
+
const events: Record<string, EventDefinitionId> = {};
|
|
168
208
|
const changes: IncrementalMappingChanges = {
|
|
169
209
|
reusedDefinitions: [],
|
|
170
210
|
addedDefinitions: []
|
|
@@ -175,6 +215,9 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
175
215
|
const compatibleParameterLookups = new HashMap<SerializedParameterIndexLookupCreator, ParameterIndexId>(
|
|
176
216
|
serializedStreamParameterIndexLookupCreatorEquality
|
|
177
217
|
);
|
|
218
|
+
const compatibleEvents = new HashMap<SerializedEventDescriptor, EventDefinitionId>(
|
|
219
|
+
serializedEventDefinitionEquality
|
|
220
|
+
);
|
|
178
221
|
|
|
179
222
|
for (const config of compatibleConfigs) {
|
|
180
223
|
for (const bucket of config.plan.buckets) {
|
|
@@ -188,6 +231,10 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
188
231
|
config.mapping.parameterLookupIdByKey(parameterLookupKey(parameterLookup.lookupScope))
|
|
189
232
|
);
|
|
190
233
|
}
|
|
234
|
+
|
|
235
|
+
for (const event of config.plan.events ?? []) {
|
|
236
|
+
compatibleEvents.putIfAbsent(event, () => config.mapping.eventDefinitionIdByName(event.name));
|
|
237
|
+
}
|
|
191
238
|
}
|
|
192
239
|
|
|
193
240
|
for (const bucket of newPlan.buckets) {
|
|
@@ -223,15 +270,32 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
223
270
|
}
|
|
224
271
|
}
|
|
225
272
|
|
|
273
|
+
for (const event of newPlan.events ?? []) {
|
|
274
|
+
const compatibleId = compatibleEvents.get(event);
|
|
275
|
+
const id = compatibleId ?? generateNewEventDefinitionId();
|
|
276
|
+
events[event.name] = id;
|
|
277
|
+
const change: IncrementalMappingDefinitionChange = {
|
|
278
|
+
type: 'event',
|
|
279
|
+
name: event.name,
|
|
280
|
+
id
|
|
281
|
+
};
|
|
282
|
+
if (compatibleId == null) {
|
|
283
|
+
changes.addedDefinitions.push(change);
|
|
284
|
+
} else {
|
|
285
|
+
changes.reusedDefinitions.push(change);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
226
289
|
return {
|
|
227
|
-
mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups),
|
|
290
|
+
mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups, events),
|
|
228
291
|
changes
|
|
229
292
|
};
|
|
230
293
|
}
|
|
231
294
|
|
|
232
295
|
constructor(
|
|
233
296
|
private definitions: Record<string, BucketDefinitionId> = {},
|
|
234
|
-
private parameterLookupMapping: Record<string, ParameterIndexId> = {}
|
|
297
|
+
private parameterLookupMapping: Record<string, ParameterIndexId> = {},
|
|
298
|
+
private events: Record<string, EventDefinitionId> = {}
|
|
235
299
|
) {}
|
|
236
300
|
|
|
237
301
|
bucketSourceId(source: BucketDataSource): BucketDefinitionId {
|
|
@@ -270,8 +334,20 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
270
334
|
}));
|
|
271
335
|
}
|
|
272
336
|
|
|
337
|
+
allEventDefinitionIds(): EventDefinitionId[] {
|
|
338
|
+
return Object.values(this.events);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
eventDefinitionEntries(): IncrementalMappingDefinitionChange[] {
|
|
342
|
+
return Object.entries(this.events).map(([name, id]) => ({
|
|
343
|
+
type: 'event',
|
|
344
|
+
name,
|
|
345
|
+
id
|
|
346
|
+
}));
|
|
347
|
+
}
|
|
348
|
+
|
|
273
349
|
allDefinitionEntries(): IncrementalMappingDefinitionChange[] {
|
|
274
|
-
return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries()];
|
|
350
|
+
return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries(), ...this.eventDefinitionEntries()];
|
|
275
351
|
}
|
|
276
352
|
|
|
277
353
|
parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
|
|
@@ -286,10 +362,23 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
286
362
|
return defId;
|
|
287
363
|
}
|
|
288
364
|
|
|
365
|
+
eventId(event: NamedEventDefinition): EventDefinitionId {
|
|
366
|
+
return this.eventDefinitionIdByName(event.name);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
eventDefinitionIdByName(name: string): EventDefinitionId {
|
|
370
|
+
const id = this.events[name];
|
|
371
|
+
if (id == null) {
|
|
372
|
+
throw new ServiceAssertionError(`No mapping found for event definition ${name}`);
|
|
373
|
+
}
|
|
374
|
+
return id;
|
|
375
|
+
}
|
|
376
|
+
|
|
289
377
|
serialize(): PersistedDefinitionMapping {
|
|
290
378
|
return {
|
|
291
379
|
definitions: { ...this.definitions },
|
|
292
|
-
parameter_indexes: { ...this.parameterLookupMapping }
|
|
380
|
+
parameter_indexes: { ...this.parameterLookupMapping },
|
|
381
|
+
events: { ...this.events }
|
|
293
382
|
};
|
|
294
383
|
}
|
|
295
384
|
|
|
@@ -297,9 +386,12 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
297
386
|
selectedSyncConfigIds: string[],
|
|
298
387
|
_table: SourceTableRef,
|
|
299
388
|
bucketDataSourceIds: BucketDefinitionId[],
|
|
300
|
-
parameterLookupSourceIds: ParameterIndexId[]
|
|
389
|
+
parameterLookupSourceIds: ParameterIndexId[],
|
|
390
|
+
eventDefinitionIds: EventDefinitionId[]
|
|
301
391
|
): string[] {
|
|
302
|
-
return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0
|
|
392
|
+
return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0 || eventDefinitionIds.length > 0
|
|
393
|
+
? selectedSyncConfigIds
|
|
394
|
+
: [];
|
|
303
395
|
}
|
|
304
396
|
|
|
305
397
|
snapshotBlockingSourceTablesFilter(_syncConfigId: string): Record<string, unknown> {
|
|
@@ -312,6 +404,10 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
|
|
|
312
404
|
if (parameterLookupSourceIds.length > 0) {
|
|
313
405
|
clauses.push({ parameter_lookup_source_ids: { $in: parameterLookupSourceIds } });
|
|
314
406
|
}
|
|
407
|
+
const eventDefinitionIds = this.allEventDefinitionIds();
|
|
408
|
+
if (eventDefinitionIds.length > 0) {
|
|
409
|
+
clauses.push({ event_definition_ids: { $in: eventDefinitionIds } });
|
|
410
|
+
}
|
|
315
411
|
if (clauses.length == 0) {
|
|
316
412
|
return {
|
|
317
413
|
snapshot_done: false,
|
|
@@ -337,6 +433,7 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM
|
|
|
337
433
|
private bucketDataSourceSyncConfigIdsById = new Map<BucketDefinitionId, Set<string>>();
|
|
338
434
|
private parameterLookupMappings = new WeakMap<ParameterIndexLookupCreator, SingleSyncConfigBucketDefinitionMapping>();
|
|
339
435
|
private parameterLookupSyncConfigIdsById = new Map<ParameterIndexId, Set<string>>();
|
|
436
|
+
private eventDefinitionSyncConfigIdsById = new Map<EventDefinitionId, Set<string>>();
|
|
340
437
|
private syncConfigsById = new Map<string, SyncConfigWithRequiredMapping>();
|
|
341
438
|
private mappings: SingleSyncConfigBucketDefinitionMapping[];
|
|
342
439
|
|
|
@@ -357,6 +454,9 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM
|
|
|
357
454
|
config.syncConfigId
|
|
358
455
|
);
|
|
359
456
|
}
|
|
457
|
+
for (const event of config.syncConfig.config.eventDefinitions) {
|
|
458
|
+
addSetEntry(this.eventDefinitionSyncConfigIdsById, config.mapping.eventId(event), config.syncConfigId);
|
|
459
|
+
}
|
|
360
460
|
}
|
|
361
461
|
}
|
|
362
462
|
|
|
@@ -388,11 +488,16 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM
|
|
|
388
488
|
return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))];
|
|
389
489
|
}
|
|
390
490
|
|
|
491
|
+
allEventDefinitionIds(): EventDefinitionId[] {
|
|
492
|
+
return [...new Set(this.mappings.flatMap((mapping) => mapping.allEventDefinitionIds()))];
|
|
493
|
+
}
|
|
494
|
+
|
|
391
495
|
syncConfigIdsForSourceTable(
|
|
392
496
|
_selectedSyncConfigIds: string[],
|
|
393
|
-
|
|
497
|
+
_table: SourceTableRef,
|
|
394
498
|
bucketDataSourceIds: BucketDefinitionId[],
|
|
395
|
-
parameterLookupSourceIds: ParameterIndexId[]
|
|
499
|
+
parameterLookupSourceIds: ParameterIndexId[],
|
|
500
|
+
eventDefinitionIds: EventDefinitionId[]
|
|
396
501
|
): string[] {
|
|
397
502
|
const ids = new Set<string>();
|
|
398
503
|
for (const sourceId of bucketDataSourceIds) {
|
|
@@ -401,10 +506,8 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM
|
|
|
401
506
|
for (const sourceId of parameterLookupSourceIds) {
|
|
402
507
|
addAll(ids, this.parameterLookupSyncConfigIdsById.get(sourceId));
|
|
403
508
|
}
|
|
404
|
-
for (const
|
|
405
|
-
|
|
406
|
-
ids.add(syncConfigId);
|
|
407
|
-
}
|
|
509
|
+
for (const eventDefinitionId of eventDefinitionIds) {
|
|
510
|
+
addAll(ids, this.eventDefinitionSyncConfigIdsById.get(eventDefinitionId));
|
|
408
511
|
}
|
|
409
512
|
return [...ids];
|
|
410
513
|
}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
BucketDataSource,
|
|
3
|
+
EventDefinition,
|
|
4
|
+
ParameterIndexLookupCreator,
|
|
5
|
+
SyncConfigWithErrors
|
|
6
|
+
} from '@powersync/service-sync-rules';
|
|
2
7
|
import {
|
|
3
8
|
IncrementalMappingChanges,
|
|
4
9
|
IncrementalMappingDefinitionChange,
|
|
@@ -30,6 +35,8 @@ export function describeIncrementalSyncConfigUpdate(options: {
|
|
|
30
35
|
mappingChanges: IncrementalMappingChanges;
|
|
31
36
|
}): IncrementalSyncConfigUpdateLog {
|
|
32
37
|
const { activeMappings, newMapping, newSyncConfig, mappingChanges } = options;
|
|
38
|
+
// Buckets, parameter lookups and events all flow through the mapping's assigned ids, so their changes are
|
|
39
|
+
// already present in mappingChanges and allDefinitionEntries().
|
|
33
40
|
const newDefinitionKeys = new Set(newMapping.allDefinitionEntries().map(definitionKey));
|
|
34
41
|
const activeDefinitions = uniqueDefinitions(activeMappings.flatMap((mapping) => mapping.allDefinitionEntries()));
|
|
35
42
|
|
|
@@ -57,10 +64,16 @@ function sourceTablesForDefinition(
|
|
|
57
64
|
);
|
|
58
65
|
}
|
|
59
66
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
67
|
+
if (definition.type == 'parameter_lookup') {
|
|
68
|
+
return sourceTablesForSources(
|
|
69
|
+
syncConfig.config.bucketParameterLookupSources.filter(
|
|
70
|
+
(source) => mapping.parameterLookupId(source) == definition.id
|
|
71
|
+
)
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return sourceTablesForEvents(
|
|
76
|
+
syncConfig.config.eventDefinitions.filter((event) => mapping.eventId(event) == definition.id)
|
|
64
77
|
);
|
|
65
78
|
}
|
|
66
79
|
|
|
@@ -68,6 +81,10 @@ function sourceTablesForSources(sources: Array<BucketDataSource | ParameterIndex
|
|
|
68
81
|
return uniqueSorted(sources.flatMap((source) => [...source.getSourceTables()].map((table) => table.tablePattern)));
|
|
69
82
|
}
|
|
70
83
|
|
|
84
|
+
function sourceTablesForEvents(events: EventDefinition[]) {
|
|
85
|
+
return uniqueSorted(events.flatMap((event) => [...event.getSourceTables()].map((table) => table.tablePattern)));
|
|
86
|
+
}
|
|
87
|
+
|
|
71
88
|
function definitionKey(definition: IncrementalMappingDefinitionChange) {
|
|
72
89
|
return `${definition.type}:${definition.id}`;
|
|
73
90
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { StorageMetric } from '@powersync/service-types';
|
|
3
3
|
import { MetricsEngine } from '../metrics/MetricsEngine.js';
|
|
4
|
-
import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
|
|
4
|
+
import { BucketStorageFactory, StorageMetrics, StorageSyncConfigMetrics } from './BucketStorageFactory.js';
|
|
5
5
|
|
|
6
6
|
export function createCoreStorageMetrics(engine: MetricsEngine): void {
|
|
7
7
|
engine.createObservableGauge({
|
|
@@ -9,59 +9,205 @@ export function createCoreStorageMetrics(engine: MetricsEngine): void {
|
|
|
9
9
|
description: 'Size of current data stored in PowerSync',
|
|
10
10
|
unit: 'bytes'
|
|
11
11
|
});
|
|
12
|
+
engine.createObservableGauge({
|
|
13
|
+
name: StorageMetric.ATTRIBUTED_SOURCE_RECORDS_BYTES,
|
|
14
|
+
description: 'Size of current data stored in PowerSync by sync config',
|
|
15
|
+
unit: 'bytes'
|
|
16
|
+
});
|
|
12
17
|
|
|
13
18
|
engine.createObservableGauge({
|
|
14
19
|
name: StorageMetric.OPERATION_SIZE_BYTES,
|
|
15
20
|
description: 'Size of operations stored in PowerSync',
|
|
16
21
|
unit: 'bytes'
|
|
17
22
|
});
|
|
23
|
+
engine.createObservableGauge({
|
|
24
|
+
name: StorageMetric.ATTRIBUTED_BUCKET_DATA_BYTES,
|
|
25
|
+
description: 'Size of operations stored in PowerSync by sync config',
|
|
26
|
+
unit: 'bytes'
|
|
27
|
+
});
|
|
18
28
|
|
|
19
29
|
engine.createObservableGauge({
|
|
20
30
|
name: StorageMetric.PARAMETER_SIZE_BYTES,
|
|
21
31
|
description: 'Size of parameter data stored in PowerSync',
|
|
22
32
|
unit: 'bytes'
|
|
23
33
|
});
|
|
34
|
+
engine.createObservableGauge({
|
|
35
|
+
name: StorageMetric.ATTRIBUTED_PARAMETER_INDEXES_BYTES,
|
|
36
|
+
description: 'Size of parameter data stored in PowerSync by sync config',
|
|
37
|
+
unit: 'bytes'
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
engine.createObservableGauge({
|
|
41
|
+
name: StorageMetric.OBJECT_STORAGE_SIZE_BYTES,
|
|
42
|
+
description: 'Size of active object-storage references stored in PowerSync',
|
|
43
|
+
unit: 'bytes'
|
|
44
|
+
});
|
|
45
|
+
engine.createObservableGauge({
|
|
46
|
+
name: StorageMetric.ATTRIBUTED_OBJECT_STORAGE_BYTES,
|
|
47
|
+
description: 'Size of active object-storage references by sync config',
|
|
48
|
+
unit: 'bytes'
|
|
49
|
+
});
|
|
24
50
|
}
|
|
25
51
|
|
|
26
52
|
export function initializeCoreStorageMetrics(engine: MetricsEngine, storage: BucketStorageFactory): void {
|
|
27
53
|
const replication_storage_size_bytes = engine.getObservableGauge(StorageMetric.REPLICATION_SIZE_BYTES);
|
|
54
|
+
const attributed_source_records_bytes = engine.getObservableGauge(StorageMetric.ATTRIBUTED_SOURCE_RECORDS_BYTES);
|
|
28
55
|
const operation_storage_size_bytes = engine.getObservableGauge(StorageMetric.OPERATION_SIZE_BYTES);
|
|
56
|
+
const attributed_bucket_data_bytes = engine.getObservableGauge(StorageMetric.ATTRIBUTED_BUCKET_DATA_BYTES);
|
|
29
57
|
const parameter_storage_size_bytes = engine.getObservableGauge(StorageMetric.PARAMETER_SIZE_BYTES);
|
|
58
|
+
const attributed_parameter_indexes_bytes = engine.getObservableGauge(
|
|
59
|
+
StorageMetric.ATTRIBUTED_PARAMETER_INDEXES_BYTES
|
|
60
|
+
);
|
|
61
|
+
const object_storage_size_bytes = engine.getObservableGauge(StorageMetric.OBJECT_STORAGE_SIZE_BYTES);
|
|
62
|
+
const attributed_object_storage_bytes = engine.getObservableGauge(StorageMetric.ATTRIBUTED_OBJECT_STORAGE_BYTES);
|
|
30
63
|
|
|
31
64
|
const MINIMUM_INTERVAL = 60_000;
|
|
32
65
|
|
|
66
|
+
/**
|
|
67
|
+
* How long a sync config that is no longer present keeps being reported as zero.
|
|
68
|
+
*
|
|
69
|
+
* This must comfortably exceed the longest metric export interval, so that every configured
|
|
70
|
+
* reader observes the zero at least once. The OTLP reader exports every 5 minutes.
|
|
71
|
+
*/
|
|
72
|
+
const RETIRED_SYNC_CONFIG_INTERVAL = 15 * 60_000;
|
|
73
|
+
|
|
74
|
+
type SyncConfigAttributes = {
|
|
75
|
+
sync_config_id: string;
|
|
76
|
+
sync_config_state: string;
|
|
77
|
+
version_label?: string;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Attribute sets reported at least once, along with the last time each was present in the
|
|
82
|
+
* storage metrics.
|
|
83
|
+
*
|
|
84
|
+
* These gauges are exported with cumulative temporality, which means the metrics SDK keeps
|
|
85
|
+
* re-exporting the last reported value for every attribute set it has seen, also after we stop
|
|
86
|
+
* reporting that set. Without this, a sync config that has since been pruned would keep
|
|
87
|
+
* reporting its last non-zero size indefinitely, so sums over these gauges would drift upwards
|
|
88
|
+
* permanently. Reporting zero for a while instead leaves the SDK carrying a zero forward.
|
|
89
|
+
*/
|
|
90
|
+
const reportedSyncConfigs = new Map<string, { attributes: SyncConfigAttributes; lastSeen: number }>();
|
|
91
|
+
|
|
92
|
+
const syncConfigKey = (attributes: SyncConfigAttributes) =>
|
|
93
|
+
JSON.stringify([attributes.sync_config_id, attributes.sync_config_state, attributes.version_label]);
|
|
94
|
+
|
|
95
|
+
const syncConfigAttributes = (syncConfig: StorageSyncConfigMetrics): SyncConfigAttributes => ({
|
|
96
|
+
sync_config_id: syncConfig.sync_config_id,
|
|
97
|
+
sync_config_state: syncConfig.sync_config_state,
|
|
98
|
+
...(syncConfig.version_label == null ? {} : { version_label: syncConfig.version_label })
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const trackSyncConfigs = (metrics: StorageMetrics | null) => {
|
|
102
|
+
if (metrics == null) {
|
|
103
|
+
// Failed to read the metrics - this says nothing about which sync configs still exist.
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const now = Date.now();
|
|
107
|
+
for (const syncConfig of metrics.sync_config_metrics ?? []) {
|
|
108
|
+
const attributes = syncConfigAttributes(syncConfig);
|
|
109
|
+
reportedSyncConfigs.set(syncConfigKey(attributes), { attributes, lastSeen: now });
|
|
110
|
+
}
|
|
111
|
+
for (const [key, entry] of reportedSyncConfigs) {
|
|
112
|
+
if (now - entry.lastSeen > RETIRED_SYNC_CONFIG_INTERVAL) {
|
|
113
|
+
reportedSyncConfigs.delete(key);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
|
|
33
118
|
let cachedRequest: Promise<StorageMetrics | null> | undefined = undefined;
|
|
34
119
|
let cacheTimestamp = 0;
|
|
35
120
|
|
|
36
121
|
const getMetrics = () => {
|
|
37
122
|
if (!cachedRequest || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
|
|
38
|
-
cachedRequest = storage
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
123
|
+
cachedRequest = storage
|
|
124
|
+
.getStorageMetrics()
|
|
125
|
+
.catch((e) => {
|
|
126
|
+
logger.error(`Failed to get storage metrics`, e);
|
|
127
|
+
return null;
|
|
128
|
+
})
|
|
129
|
+
.then((metrics) => {
|
|
130
|
+
trackSyncConfigs(metrics);
|
|
131
|
+
return metrics;
|
|
132
|
+
});
|
|
42
133
|
cacheTimestamp = Date.now();
|
|
43
134
|
}
|
|
44
135
|
return cachedRequest;
|
|
45
136
|
};
|
|
46
137
|
|
|
138
|
+
type StorageSizeMetricKey =
|
|
139
|
+
| 'attributed_bucket_data_bytes'
|
|
140
|
+
| 'attributed_parameter_indexes_bytes'
|
|
141
|
+
| 'attributed_source_records_bytes'
|
|
142
|
+
| 'attributed_object_storage_bytes';
|
|
143
|
+
|
|
144
|
+
function nonNegative(value: number): number;
|
|
145
|
+
function nonNegative(value: number | undefined): number | undefined;
|
|
146
|
+
function nonNegative(value: number | undefined) {
|
|
147
|
+
return value == null ? value : Math.max(0, value);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const observationsForSyncConfigs = (metrics: StorageMetrics, key: StorageSizeMetricKey) => {
|
|
151
|
+
const observations: { value: number; attributes?: Record<string, string> }[] = [];
|
|
152
|
+
const present = new Set<string>();
|
|
153
|
+
for (const syncConfig of metrics.sync_config_metrics ?? []) {
|
|
154
|
+
const attributes = syncConfigAttributes(syncConfig);
|
|
155
|
+
present.add(syncConfigKey(attributes));
|
|
156
|
+
observations.push({ value: nonNegative(syncConfig[key]), attributes });
|
|
157
|
+
}
|
|
158
|
+
for (const entry of reportedSyncConfigs.values()) {
|
|
159
|
+
if (!present.has(syncConfigKey(entry.attributes))) {
|
|
160
|
+
observations.push({ value: 0, attributes: entry.attributes });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return observations;
|
|
164
|
+
};
|
|
165
|
+
|
|
47
166
|
replication_storage_size_bytes.setValueProvider(async () => {
|
|
167
|
+
const metrics = await getMetrics();
|
|
168
|
+
return nonNegative(metrics?.replication_size_bytes);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
attributed_source_records_bytes.setValueProvider(async () => {
|
|
48
172
|
const metrics = await getMetrics();
|
|
49
173
|
if (metrics) {
|
|
50
|
-
return metrics
|
|
174
|
+
return observationsForSyncConfigs(metrics, 'attributed_source_records_bytes');
|
|
51
175
|
}
|
|
52
176
|
});
|
|
53
177
|
|
|
54
178
|
operation_storage_size_bytes.setValueProvider(async () => {
|
|
179
|
+
const metrics = await getMetrics();
|
|
180
|
+
return nonNegative(metrics?.operations_size_bytes);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
attributed_bucket_data_bytes.setValueProvider(async () => {
|
|
55
184
|
const metrics = await getMetrics();
|
|
56
185
|
if (metrics) {
|
|
57
|
-
return metrics
|
|
186
|
+
return observationsForSyncConfigs(metrics, 'attributed_bucket_data_bytes');
|
|
58
187
|
}
|
|
59
188
|
});
|
|
60
189
|
|
|
61
190
|
parameter_storage_size_bytes.setValueProvider(async () => {
|
|
191
|
+
const metrics = await getMetrics();
|
|
192
|
+
return nonNegative(metrics?.parameters_size_bytes);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
attributed_parameter_indexes_bytes.setValueProvider(async () => {
|
|
196
|
+
const metrics = await getMetrics();
|
|
197
|
+
if (metrics) {
|
|
198
|
+
return observationsForSyncConfigs(metrics, 'attributed_parameter_indexes_bytes');
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
object_storage_size_bytes.setValueProvider(async () => {
|
|
203
|
+
const metrics = await getMetrics();
|
|
204
|
+
return nonNegative(metrics?.object_storage_size_bytes);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
attributed_object_storage_bytes.setValueProvider(async () => {
|
|
62
208
|
const metrics = await getMetrics();
|
|
63
209
|
if (metrics) {
|
|
64
|
-
return metrics
|
|
210
|
+
return observationsForSyncConfigs(metrics, 'attributed_object_storage_bytes');
|
|
65
211
|
}
|
|
66
212
|
});
|
|
67
213
|
}
|
|
@@ -39,7 +39,9 @@ export type SyncCheckpointTraceCategory =
|
|
|
39
39
|
// Data fetches. Holds one data lock for this period.
|
|
40
40
|
| 'bucket_data'
|
|
41
41
|
// Sending data to client, may include serialization overhead. Holds one data lock for this period in most cases.
|
|
42
|
-
| 'sending'
|
|
42
|
+
| 'sending'
|
|
43
|
+
// Time spent reading concurrent S3 requests, counted once per batch.
|
|
44
|
+
| 's3';
|
|
43
45
|
|
|
44
46
|
export interface BucketChecksumStateOptions {
|
|
45
47
|
syncContext: SyncContext;
|