@powersync/service-core 1.24.0 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/dist/api/diagnostics.js +2 -0
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +28 -24
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +5 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +7 -3
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +16 -6
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +9 -0
- package/dist/routes/endpoints/admin.d.ts +18 -0
- package/dist/routes/endpoints/admin.js +3 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +3 -1
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +2 -1
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +28 -4
- package/dist/storage/BucketStorageFactory.js +8 -4
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.d.ts +2 -0
- package/dist/storage/PersistedSyncConfigContent.js +23 -10
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +6 -1
- package/dist/storage/SourceEntity.d.ts +7 -2
- package/dist/storage/SourceTable.d.ts +37 -6
- package/dist/storage/SourceTable.js +42 -10
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SourceTableReconciler.d.ts +83 -0
- package/dist/storage/SourceTableReconciler.js +95 -0
- package/dist/storage/SourceTableReconciler.js.map +1 -0
- package/dist/storage/StorageVersionConfig.d.ts +4 -0
- package/dist/storage/StorageVersionConfig.js +10 -1
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +45 -7
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +25 -3
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +23 -7
- package/dist/storage/implementation/BucketDefinitionMapping.js +88 -14
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +9 -1
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js +120 -4
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +1 -1
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +37 -9
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js +10 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +5 -0
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/dist/util/protocol-types.d.ts +4 -4
- package/dist/util/protocol-types.js +3 -2
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +2 -1
- package/dist/util/utils.js +16 -1
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -5
- package/src/api/diagnostics.ts +2 -0
- package/src/entry/commands/compact-action.ts +32 -24
- package/src/metrics/metrics-interfaces.ts +6 -1
- package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
- package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
- package/src/metrics/open-telemetry/util.ts +7 -3
- package/src/replication/AbstractReplicator.ts +22 -6
- package/src/routes/endpoints/admin.ts +3 -1
- package/src/routes/endpoints/sync-rules.ts +3 -1
- package/src/storage/BucketStorageBatch.ts +2 -1
- package/src/storage/BucketStorageFactory.ts +44 -8
- package/src/storage/PersistedSyncConfigContent.ts +26 -11
- package/src/storage/ReplicationEventPayload.ts +6 -1
- package/src/storage/SourceEntity.ts +6 -2
- package/src/storage/SourceTable.ts +68 -11
- package/src/storage/SourceTableReconciler.ts +188 -0
- package/src/storage/StorageVersionConfig.ts +11 -1
- package/src/storage/SyncRulesBucketStorage.ts +53 -7
- package/src/storage/WriteCheckpointAPI.ts +32 -3
- package/src/storage/implementation/BucketDefinitionMapping.ts +120 -17
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +22 -5
- package/src/storage/storage-index.ts +1 -0
- package/src/storage/storage-metrics.ts +154 -8
- package/src/sync/BucketChecksumState.ts +3 -1
- package/src/sync/sync.ts +40 -11
- package/src/sync/util.ts +21 -8
- package/src/util/config/compound-config-collector.ts +6 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
- package/src/util/config/types.ts +1 -0
- package/src/util/protocol-types.ts +3 -2
- package/src/util/utils.ts +15 -1
- package/test/src/AbstractReplicator.test.ts +30 -0
- package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
- package/test/src/PersistedSyncConfigContent.test.ts +152 -0
- package/test/src/config.test.ts +40 -0
- package/test/src/diagnostics.test.ts +25 -0
- package/test/src/routes/admin.test.ts +8 -2
- package/test/src/routes/stream.test.ts +123 -2
- package/test/src/source-table-reconciler.test.ts +244 -0
- package/test/src/storage/SourceTable.test.ts +4 -1
- package/test/src/storage-metrics.test.ts +148 -0
- package/test/src/sync/util.test.ts +15 -0
- package/test/src/util/utils.test.ts +23 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -34,6 +34,7 @@ describe('admin routes', () => {
|
|
|
34
34
|
syncConfigId?: string;
|
|
35
35
|
active?: boolean;
|
|
36
36
|
content?: string;
|
|
37
|
+
version_label?: string;
|
|
37
38
|
}): storage.PersistedSyncConfigContent {
|
|
38
39
|
const id = options.id ?? 1;
|
|
39
40
|
const syncConfigId = options.syncConfigId ?? String(id);
|
|
@@ -66,6 +67,7 @@ bucket_definitions:
|
|
|
66
67
|
compiled_plan: null,
|
|
67
68
|
storageVersion: storage.LEGACY_STORAGE_VERSION,
|
|
68
69
|
syncConfigState: syncConfigStatus.state,
|
|
70
|
+
version_label: options.version_label,
|
|
69
71
|
parsed(options?: any) {
|
|
70
72
|
const syncRules = SqlSyncRules.fromYaml(content.sync_rules_content, {
|
|
71
73
|
...options,
|
|
@@ -113,7 +115,7 @@ bucket_definitions:
|
|
|
113
115
|
|
|
114
116
|
describe('diagnostics', () => {
|
|
115
117
|
it('returns deploying config status', async () => {
|
|
116
|
-
const active = makeSyncConfigContent({ id: 1, syncConfigId: 'active-config' });
|
|
118
|
+
const active = makeSyncConfigContent({ id: 1, syncConfigId: 'active-config', version_label: 'v5' });
|
|
117
119
|
const deploying = makeSyncConfigContent({ id: 2, syncConfigId: 'deploying-config', active: false });
|
|
118
120
|
const getInstance = vi.fn(() => ({
|
|
119
121
|
async getStatus() {
|
|
@@ -144,12 +146,13 @@ bucket_definitions:
|
|
|
144
146
|
|
|
145
147
|
expect(response.deploying_sync_rules?.connections[0].slot_name).toBe('slot_2');
|
|
146
148
|
expect(response.active_sync_rules?.connections[0].slot_name).toBe('slot_1');
|
|
149
|
+
expect(response.active_sync_rules?.version_label).toBe('v5');
|
|
147
150
|
});
|
|
148
151
|
});
|
|
149
152
|
|
|
150
153
|
describe('reprocess', () => {
|
|
151
154
|
it('reprocesses the active sync config', async () => {
|
|
152
|
-
const active = makeSyncConfigContent({ id: 7, syncConfigId: 'active-config' });
|
|
155
|
+
const active = makeSyncConfigContent({ id: 7, syncConfigId: 'active-config', version_label: 'v6' });
|
|
153
156
|
const updateSyncRules = vi.fn(async () => ({
|
|
154
157
|
replicationStreamId: 8,
|
|
155
158
|
replicationStreamName: 'new_slot',
|
|
@@ -178,6 +181,9 @@ bucket_definitions:
|
|
|
178
181
|
expect(activeBucketStorage.getActiveSyncConfig).toHaveBeenCalledTimes(1);
|
|
179
182
|
expect(activeBucketStorage.getSyncConfigContent).not.toHaveBeenCalled();
|
|
180
183
|
expect(updateSyncRules).toHaveBeenCalledTimes(1);
|
|
184
|
+
expect(updateSyncRules).toHaveBeenCalledWith(
|
|
185
|
+
expect.objectContaining({ version_label: 'v6', forceNewReplicationStream: true })
|
|
186
|
+
);
|
|
181
187
|
expect(response.connections[0].slot_name).toBe('new_slot');
|
|
182
188
|
});
|
|
183
189
|
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
BasicRouterRequest,
|
|
3
|
+
CHECKPOINT_INVALIDATE_ALL,
|
|
4
|
+
Context,
|
|
5
|
+
JwtPayload,
|
|
6
|
+
RequestTracker,
|
|
7
|
+
streamResponse,
|
|
8
|
+
SyncRulesBucketStorage
|
|
9
|
+
} from '@/index.js';
|
|
2
10
|
import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
|
|
3
11
|
import {
|
|
4
12
|
DEFAULT_HYDRATION_STATE,
|
|
@@ -9,7 +17,7 @@ import {
|
|
|
9
17
|
import * as sqlite from 'node:sqlite';
|
|
10
18
|
import { Readable, Writable } from 'stream';
|
|
11
19
|
import { pipeline } from 'stream/promises';
|
|
12
|
-
import { describe, expect, it } from 'vitest';
|
|
20
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
13
21
|
import winston from 'winston';
|
|
14
22
|
import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
|
|
15
23
|
import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
|
|
@@ -182,6 +190,119 @@ describe('Stream Route', () => {
|
|
|
182
190
|
);
|
|
183
191
|
});
|
|
184
192
|
});
|
|
193
|
+
|
|
194
|
+
it('closes after a data fetch error while waiting for a new checkpoint', async () => {
|
|
195
|
+
// After 1,000 operations, the sync stream concurrently calls next() on the
|
|
196
|
+
// checkpoint watcher. This test then fails the next data fetch without
|
|
197
|
+
// producing another checkpoint. Before the fix, awaiting watcher.return()
|
|
198
|
+
// during cleanup queued it behind that pending next(), leaving the response
|
|
199
|
+
// open until an unrelated checkpoint was written.
|
|
200
|
+
const syncRules = SqlSyncRules.fromYaml(
|
|
201
|
+
`
|
|
202
|
+
bucket_definitions:
|
|
203
|
+
global:
|
|
204
|
+
data: []
|
|
205
|
+
`,
|
|
206
|
+
{ defaultSchema: 'public' }
|
|
207
|
+
).config.hydrate(defaultHydrationOptions);
|
|
208
|
+
|
|
209
|
+
let checkpointWatcherAborted = false;
|
|
210
|
+
let dataFetches = 0;
|
|
211
|
+
const storage = {
|
|
212
|
+
watchCheckpointChanges: async function* ({ signal }) {
|
|
213
|
+
yield {
|
|
214
|
+
base: {
|
|
215
|
+
checkpoint: 1n,
|
|
216
|
+
lsn: '1',
|
|
217
|
+
getParameterSets: async () => []
|
|
218
|
+
},
|
|
219
|
+
writeCheckpoint: null,
|
|
220
|
+
update: CHECKPOINT_INVALIDATE_ALL
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
await new Promise<void>((_resolve, reject) => {
|
|
224
|
+
// There is intentionally no next checkpoint: cleanup must abort this
|
|
225
|
+
// pending wait rather than wait for it to complete naturally.
|
|
226
|
+
signal.addEventListener(
|
|
227
|
+
'abort',
|
|
228
|
+
() => {
|
|
229
|
+
checkpointWatcherAborted = true;
|
|
230
|
+
reject(new Error('Checkpoint watcher aborted'));
|
|
231
|
+
},
|
|
232
|
+
{ once: true }
|
|
233
|
+
);
|
|
234
|
+
});
|
|
235
|
+
},
|
|
236
|
+
getChecksums: async (_checkpoint, buckets) =>
|
|
237
|
+
new Map(buckets.map(({ bucket }) => [bucket, { bucket, checksum: 1, count: 1000 }])),
|
|
238
|
+
getBucketDataBatch: async function* () {
|
|
239
|
+
if (dataFetches++ > 0) {
|
|
240
|
+
throw new Error('Simulated data fetch failure');
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
yield {
|
|
244
|
+
chunkData: {
|
|
245
|
+
bucket: 'global[]',
|
|
246
|
+
data: Array.from({ length: 1000 }, () => ({ op: 'PUT' })),
|
|
247
|
+
has_more: true,
|
|
248
|
+
after: '0',
|
|
249
|
+
next_after: '1000'
|
|
250
|
+
},
|
|
251
|
+
targetOp: null
|
|
252
|
+
};
|
|
253
|
+
yield { hasMore: true };
|
|
254
|
+
}
|
|
255
|
+
} as Partial<SyncRulesBucketStorage>;
|
|
256
|
+
const serviceContext = mockServiceContext(storage);
|
|
257
|
+
const controller = new AbortController();
|
|
258
|
+
const checkpointLogger = logger.child({});
|
|
259
|
+
const checkpointLogSpy = vi.spyOn(checkpointLogger, 'info');
|
|
260
|
+
|
|
261
|
+
const response = (async () => {
|
|
262
|
+
for await (const _line of streamResponse({
|
|
263
|
+
syncContext: serviceContext.syncContext,
|
|
264
|
+
bucketStorage: storage as SyncRulesBucketStorage,
|
|
265
|
+
syncRules,
|
|
266
|
+
params: { client_id: 'test-client', raw_data: true },
|
|
267
|
+
token: new JwtPayload({ sub: 'test-user', exp: Date.now() / 1000 + 10_000 }),
|
|
268
|
+
tracker: new RequestTracker(serviceContext.metricsEngine),
|
|
269
|
+
isEncodingAsBson: false,
|
|
270
|
+
signal: controller.signal,
|
|
271
|
+
logger: checkpointLogger
|
|
272
|
+
})) {
|
|
273
|
+
// Consume the response until the simulated data fetch error is propagated.
|
|
274
|
+
}
|
|
275
|
+
})();
|
|
276
|
+
|
|
277
|
+
let timeout: NodeJS.Timeout | undefined;
|
|
278
|
+
const outcome = await Promise.race([
|
|
279
|
+
response.then(
|
|
280
|
+
() => new Error('Sync stream unexpectedly completed'),
|
|
281
|
+
(error) => error
|
|
282
|
+
),
|
|
283
|
+
new Promise((resolve) => {
|
|
284
|
+
timeout = setTimeout(() => resolve(new Error('Sync stream did not close')), 500);
|
|
285
|
+
})
|
|
286
|
+
]);
|
|
287
|
+
clearTimeout(timeout);
|
|
288
|
+
|
|
289
|
+
// Capture this before aborting below, which would set it regardless of the stream cleanup.
|
|
290
|
+
const abortedByCleanup = checkpointWatcherAborted;
|
|
291
|
+
|
|
292
|
+
controller.abort();
|
|
293
|
+
await response.catch(() => {});
|
|
294
|
+
|
|
295
|
+
expect(outcome).toBeInstanceOf(Error);
|
|
296
|
+
expect((outcome as Error).message).toContain('Simulated data fetch failure');
|
|
297
|
+
expect(abortedByCleanup).toBe(true);
|
|
298
|
+
expect(checkpointLogSpy).toHaveBeenCalledWith('checkpoint_interrupted: 1', {
|
|
299
|
+
checkpoint: 1n,
|
|
300
|
+
user_id: 'test-user',
|
|
301
|
+
operations_synced: 1000,
|
|
302
|
+
data_synced_bytes: 0,
|
|
303
|
+
ms: expect.any(Object)
|
|
304
|
+
});
|
|
305
|
+
});
|
|
185
306
|
});
|
|
186
307
|
|
|
187
308
|
export async function drainWithTimeout(readable: Readable, ms = 2_000) {
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { SourceEntityDescriptor } from '@/storage/SourceEntity.js';
|
|
2
|
+
import { SourceTable, sourceTableIdEquals } from '@/storage/SourceTable.js';
|
|
3
|
+
import {
|
|
4
|
+
defaultSourceTableReconciler,
|
|
5
|
+
diffSourceTableUpdates,
|
|
6
|
+
materializeSourceTableResolution,
|
|
7
|
+
sourceIdentityCompatible,
|
|
8
|
+
validateSourceTableCandidateResolution
|
|
9
|
+
} from '@/storage/SourceTableReconciler.js';
|
|
10
|
+
import * as bson from 'bson';
|
|
11
|
+
import { describe, expect, it } from 'vitest';
|
|
12
|
+
|
|
13
|
+
function descriptor(overrides: Partial<SourceEntityDescriptor> = {}): SourceEntityDescriptor {
|
|
14
|
+
return {
|
|
15
|
+
connectionTag: 'default',
|
|
16
|
+
schema: 'public',
|
|
17
|
+
name: 'users',
|
|
18
|
+
objectId: 100,
|
|
19
|
+
replicaIdColumns: [{ name: 'id', type: 'int', typeId: 23 }],
|
|
20
|
+
...overrides
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function candidate(overrides: Partial<ConstructorParameters<typeof SourceTable>[0]> = {}): SourceTable {
|
|
25
|
+
return new SourceTable({
|
|
26
|
+
id: overrides.id ?? 'table-1',
|
|
27
|
+
ref: overrides.ref ?? { connectionTag: 'default', schema: 'public', name: 'users' },
|
|
28
|
+
objectId: 'objectId' in overrides ? overrides.objectId! : 100,
|
|
29
|
+
replicaIdColumns: overrides.replicaIdColumns ?? [{ name: 'id', type: 'int', typeId: 23 }],
|
|
30
|
+
snapshotComplete: overrides.snapshotComplete ?? true,
|
|
31
|
+
bucketDataSources: overrides.bucketDataSources ?? [],
|
|
32
|
+
parameterLookupSources: overrides.parameterLookupSources ?? [],
|
|
33
|
+
sourceMetadata: overrides.sourceMetadata
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
describe('sourceIdentityCompatible', () => {
|
|
38
|
+
it('matches identical identity', () => {
|
|
39
|
+
expect(sourceIdentityCompatible(descriptor(), candidate())).toBe(true);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('rejects a different object id', () => {
|
|
43
|
+
expect(sourceIdentityCompatible(descriptor({ objectId: 200 }), candidate({ objectId: 100 }))).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('rejects a different schema/name', () => {
|
|
47
|
+
expect(sourceIdentityCompatible(descriptor({ name: 'accounts' }), candidate())).toBe(false);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('rejects changed replica-id columns', () => {
|
|
51
|
+
expect(
|
|
52
|
+
sourceIdentityCompatible(
|
|
53
|
+
descriptor(),
|
|
54
|
+
candidate({ replicaIdColumns: [{ name: 'id', type: 'bigint', typeId: 20 }] })
|
|
55
|
+
)
|
|
56
|
+
).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('treats an undefined descriptor object id as a wildcard on object id', () => {
|
|
60
|
+
expect(sourceIdentityCompatible(descriptor({ objectId: undefined }), candidate({ objectId: 999 }))).toBe(true);
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe('defaultSourceTableReconciler', () => {
|
|
65
|
+
it('returns all identity-compatible candidates and no metadata', async () => {
|
|
66
|
+
const a = candidate({ id: 'a' });
|
|
67
|
+
const b = candidate({ id: 'b', ref: { connectionTag: 'default', schema: 'public', name: 'accounts' } });
|
|
68
|
+
const resolution = await defaultSourceTableReconciler({ source: descriptor(), candidates: [a, b] });
|
|
69
|
+
expect(resolution.compatibleTables).toEqual([a]);
|
|
70
|
+
expect(resolution.incompatibleTables).toEqual([b]);
|
|
71
|
+
expect(resolution.newTableValues).toEqual({ sourceMetadata: null });
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('returns an empty set when nothing matches', async () => {
|
|
75
|
+
const resolution = await defaultSourceTableReconciler({
|
|
76
|
+
source: descriptor({ objectId: 200 }),
|
|
77
|
+
candidates: [candidate({ objectId: 100 })]
|
|
78
|
+
});
|
|
79
|
+
expect(resolution.compatibleTables).toHaveLength(0);
|
|
80
|
+
expect(resolution.incompatibleTables.map((table) => table.id)).toEqual(['table-1']);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
describe('validateSourceTableCandidateResolution', () => {
|
|
85
|
+
it('accepts an explicit compatible/incompatible partition', () => {
|
|
86
|
+
const a = candidate({ id: 'a' });
|
|
87
|
+
const b = candidate({ id: 'b' });
|
|
88
|
+
expect(() =>
|
|
89
|
+
validateSourceTableCandidateResolution([a, b], {
|
|
90
|
+
compatibleTables: [a],
|
|
91
|
+
incompatibleTables: [b],
|
|
92
|
+
newTableValues: { sourceMetadata: null }
|
|
93
|
+
})
|
|
94
|
+
).not.toThrow();
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it('rejects omitted, duplicate, and unknown candidates', () => {
|
|
98
|
+
const a = candidate({ id: 'a' });
|
|
99
|
+
expect(() =>
|
|
100
|
+
validateSourceTableCandidateResolution([a], {
|
|
101
|
+
compatibleTables: [],
|
|
102
|
+
incompatibleTables: [],
|
|
103
|
+
newTableValues: { sourceMetadata: null }
|
|
104
|
+
})
|
|
105
|
+
).toThrow(/exactly once/);
|
|
106
|
+
expect(() =>
|
|
107
|
+
validateSourceTableCandidateResolution([a], {
|
|
108
|
+
compatibleTables: [a],
|
|
109
|
+
incompatibleTables: [a],
|
|
110
|
+
newTableValues: { sourceMetadata: null }
|
|
111
|
+
})
|
|
112
|
+
).toThrow(/exactly once/);
|
|
113
|
+
expect(() =>
|
|
114
|
+
validateSourceTableCandidateResolution([a], {
|
|
115
|
+
compatibleTables: [a],
|
|
116
|
+
incompatibleTables: [candidate({ id: 'unknown' })],
|
|
117
|
+
newTableValues: { sourceMetadata: null }
|
|
118
|
+
})
|
|
119
|
+
).toThrow(/unknown candidate/);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
describe('diffSourceTableUpdates', () => {
|
|
124
|
+
function resolution(compatibleTables: SourceTable[]) {
|
|
125
|
+
return { compatibleTables, incompatibleTables: [], newTableValues: { sourceMetadata: null } };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
it('returns nothing when the reconciler returned the candidates untouched', () => {
|
|
129
|
+
const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
|
|
130
|
+
expect(diffSourceTableUpdates([a], resolution([a]))).toEqual([]);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it('compares by value, not by reference', () => {
|
|
134
|
+
// A new object with equal metadata should not cause a write.
|
|
135
|
+
const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
|
|
136
|
+
const rebuilt = a.withSourceMetadata({ captureTableObjectId: 7 });
|
|
137
|
+
|
|
138
|
+
expect(rebuilt.sourceMetadata).not.toBe(a.sourceMetadata);
|
|
139
|
+
expect(diffSourceTableUpdates([a], resolution([rebuilt]))).toEqual([]);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('returns changed metadata', () => {
|
|
143
|
+
const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
|
|
144
|
+
expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata({ captureTableObjectId: 8 })]))).toEqual([
|
|
145
|
+
{ id: 'a', sourceMetadata: { captureTableObjectId: 8 } }
|
|
146
|
+
]);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('returns metadata added to a legacy record', () => {
|
|
150
|
+
const legacy = candidate({ id: 'a' });
|
|
151
|
+
expect(legacy.sourceMetadata).toBeNull();
|
|
152
|
+
expect(
|
|
153
|
+
diffSourceTableUpdates([legacy], resolution([legacy.withSourceMetadata({ captureTableObjectId: 7 })]))
|
|
154
|
+
).toEqual([{ id: 'a', sourceMetadata: { captureTableObjectId: 7 } }]);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it('returns cleared metadata as null', () => {
|
|
158
|
+
const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
|
|
159
|
+
expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata(null)]))).toEqual([
|
|
160
|
+
{ id: 'a', sourceMetadata: null }
|
|
161
|
+
]);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it('materializes only source metadata from reconciler results', () => {
|
|
165
|
+
const persisted = candidate({ id: 'a', snapshotComplete: false });
|
|
166
|
+
const modified = persisted.withSourceMetadata({ captureTableObjectId: 8 });
|
|
167
|
+
modified.snapshotComplete = true;
|
|
168
|
+
modified.syncData = false;
|
|
169
|
+
|
|
170
|
+
const [materialized] = materializeSourceTableResolution([persisted], resolution([modified])).compatibleTables;
|
|
171
|
+
|
|
172
|
+
expect(materialized.sourceMetadata).toEqual({ captureTableObjectId: 8 });
|
|
173
|
+
expect(materialized.snapshotComplete).toBe(false);
|
|
174
|
+
expect(materialized.syncData).toBe(persisted.syncData);
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
it('does not trust incompatible candidate mutations made through a cast', () => {
|
|
178
|
+
const persisted = candidate({ id: 'a', snapshotComplete: false });
|
|
179
|
+
const exposed = persisted.clone();
|
|
180
|
+
// Simulate callback code bypassing the read-only type boundary.
|
|
181
|
+
(exposed as any).snapshotComplete = true;
|
|
182
|
+
|
|
183
|
+
const materialized = materializeSourceTableResolution([persisted], {
|
|
184
|
+
compatibleTables: [],
|
|
185
|
+
incompatibleTables: [exposed],
|
|
186
|
+
newTableValues: { sourceMetadata: null }
|
|
187
|
+
}).incompatibleTables[0];
|
|
188
|
+
|
|
189
|
+
expect(exposed.snapshotComplete).toBe(true);
|
|
190
|
+
expect(materialized).toBe(persisted);
|
|
191
|
+
expect(materialized.snapshotComplete).toBe(false);
|
|
192
|
+
expect(materialized.syncData).toBe(persisted.syncData);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('only reports the candidates that changed', () => {
|
|
196
|
+
const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
|
|
197
|
+
const b = candidate({ id: 'b', sourceMetadata: { captureTableObjectId: 7 } });
|
|
198
|
+
const updates = diffSourceTableUpdates([a, b], resolution([a, b.withSourceMetadata({ captureTableObjectId: 9 })]));
|
|
199
|
+
expect(updates).toEqual([{ id: 'b', sourceMetadata: { captureTableObjectId: 9 } }]);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it('rejects a compatible table that was never a candidate', () => {
|
|
203
|
+
expect(() => diffSourceTableUpdates([candidate({ id: 'a' })], resolution([candidate({ id: 'ghost' })]))).toThrow(
|
|
204
|
+
/unknown candidate/
|
|
205
|
+
);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
describe('SourceTable.clone', () => {
|
|
210
|
+
it('isolates mutable identity and metadata values', () => {
|
|
211
|
+
const persisted = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
|
|
212
|
+
const exposed = persisted.clone();
|
|
213
|
+
|
|
214
|
+
(exposed.ref as any).name = 'changed';
|
|
215
|
+
exposed.replicaIdColumns[0].name = 'changed';
|
|
216
|
+
(exposed.sourceMetadata as any).captureTableObjectId = 8;
|
|
217
|
+
|
|
218
|
+
expect(persisted.name).toBe('users');
|
|
219
|
+
expect(persisted.replicaIdColumns[0].name).toBe('id');
|
|
220
|
+
expect(persisted.sourceMetadata).toEqual({ captureTableObjectId: 7 });
|
|
221
|
+
});
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
describe('sourceTableIdEquals', () => {
|
|
225
|
+
it('compares string ids by value', () => {
|
|
226
|
+
expect(sourceTableIdEquals('table-1', 'table-1')).toBe(true);
|
|
227
|
+
expect(sourceTableIdEquals('table-1', 'table-2')).toBe(false);
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
it('compares BSON ObjectIds by value', () => {
|
|
231
|
+
const id = new bson.ObjectId();
|
|
232
|
+
const copy = new bson.ObjectId(id.toHexString());
|
|
233
|
+
|
|
234
|
+
expect(id).not.toBe(copy);
|
|
235
|
+
expect(sourceTableIdEquals(id, copy)).toBe(true);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it('does not mix string and BSON ObjectId representations', () => {
|
|
239
|
+
const id = new bson.ObjectId();
|
|
240
|
+
|
|
241
|
+
expect(sourceTableIdEquals(id.toHexString(), id)).toBe(false);
|
|
242
|
+
expect(sourceTableIdEquals(id, id.toHexString())).toBe(false);
|
|
243
|
+
});
|
|
244
|
+
});
|
|
@@ -43,7 +43,8 @@ describe('SourceTable', () => {
|
|
|
43
43
|
test('clone preserves all properties including storeCurrentData', () => {
|
|
44
44
|
const table = makeTable({
|
|
45
45
|
replicaIdColumns: [{ name: 'id', type: 'int4' }],
|
|
46
|
-
snapshotComplete: false
|
|
46
|
+
snapshotComplete: false,
|
|
47
|
+
eventDefinitionIds: new Set(['event-1'])
|
|
47
48
|
});
|
|
48
49
|
|
|
49
50
|
table.syncData = false;
|
|
@@ -63,6 +64,8 @@ describe('SourceTable', () => {
|
|
|
63
64
|
expect(cloned.syncEvent).toBe(false);
|
|
64
65
|
expect(cloned.storeCurrentData).toBe(false);
|
|
65
66
|
expect(cloned.snapshotStatus).toEqual(table.snapshotStatus);
|
|
67
|
+
expect(cloned.eventDefinitionIds).toEqual(new Set(['event-1']));
|
|
68
|
+
expect(cloned.eventDefinitionIds).not.toBe(table.eventDefinitionIds);
|
|
66
69
|
});
|
|
67
70
|
});
|
|
68
71
|
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { MetricsEngine } from '@/metrics/MetricsEngine.js';
|
|
2
|
+
import {
|
|
3
|
+
MetricMetadata,
|
|
4
|
+
MetricsFactory,
|
|
5
|
+
ObservableGauge,
|
|
6
|
+
ObservableGaugeObservation
|
|
7
|
+
} from '@/metrics/metrics-interfaces.js';
|
|
8
|
+
import { BucketStorageFactory, StorageMetrics, StorageSyncConfigMetrics } from '@/storage/BucketStorageFactory.js';
|
|
9
|
+
import { createCoreStorageMetrics, initializeCoreStorageMetrics } from '@/storage/storage-metrics.js';
|
|
10
|
+
import { StorageMetric } from '@powersync/service-types';
|
|
11
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
12
|
+
|
|
13
|
+
type ValueProvider = () => Promise<number | ObservableGaugeObservation[] | undefined>;
|
|
14
|
+
|
|
15
|
+
function testEngine() {
|
|
16
|
+
const providers = new Map<string, ValueProvider>();
|
|
17
|
+
const factory: MetricsFactory = {
|
|
18
|
+
createCounter: () => ({ add: () => {} }),
|
|
19
|
+
createUpDownCounter: () => ({ add: () => {} }),
|
|
20
|
+
createObservableGauge: (metadata: MetricMetadata): ObservableGauge => ({
|
|
21
|
+
setValueProvider: (valueProvider) => providers.set(metadata.name, valueProvider)
|
|
22
|
+
})
|
|
23
|
+
};
|
|
24
|
+
const engine = new MetricsEngine({ factory, disable_telemetry_sharing: true });
|
|
25
|
+
createCoreStorageMetrics(engine);
|
|
26
|
+
return { engine, providers };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function syncConfigMetrics(id: string, state: string, bytes: number, versionLabel?: string): StorageSyncConfigMetrics {
|
|
30
|
+
return {
|
|
31
|
+
sync_config_id: id,
|
|
32
|
+
sync_config_state: state,
|
|
33
|
+
version_label: versionLabel,
|
|
34
|
+
attributed_bucket_data_bytes: bytes,
|
|
35
|
+
attributed_parameter_indexes_bytes: bytes,
|
|
36
|
+
attributed_source_records_bytes: bytes,
|
|
37
|
+
attributed_object_storage_bytes: bytes
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function storageMetrics(sync_config_metrics: StorageSyncConfigMetrics[]): StorageMetrics {
|
|
42
|
+
return {
|
|
43
|
+
operations_size_bytes: 0,
|
|
44
|
+
parameters_size_bytes: 0,
|
|
45
|
+
replication_size_bytes: 0,
|
|
46
|
+
object_storage_size_bytes: 0,
|
|
47
|
+
sync_config_metrics
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
describe('storage metrics', () => {
|
|
52
|
+
beforeEach(() => {
|
|
53
|
+
vi.useFakeTimers();
|
|
54
|
+
vi.setSystemTime(0);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
afterEach(() => {
|
|
58
|
+
vi.useRealTimers();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('reports retired sync configs as zero, then stops reporting them', async () => {
|
|
62
|
+
let current = storageMetrics([syncConfigMetrics('config-a', 'ACTIVE', 100)]);
|
|
63
|
+
const storage = {
|
|
64
|
+
getStorageMetrics: async () => current
|
|
65
|
+
} as unknown as BucketStorageFactory;
|
|
66
|
+
|
|
67
|
+
const { engine, providers } = testEngine();
|
|
68
|
+
initializeCoreStorageMetrics(engine, storage);
|
|
69
|
+
const provider = providers.get(StorageMetric.ATTRIBUTED_BUCKET_DATA_BYTES)!;
|
|
70
|
+
|
|
71
|
+
expect(await provider()).toEqual([
|
|
72
|
+
{ value: 100, attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' } }
|
|
73
|
+
]);
|
|
74
|
+
|
|
75
|
+
// The sync config is pruned. Its series must go to zero rather than keep its last value,
|
|
76
|
+
// since the SDK carries the last reported value forward under cumulative temporality.
|
|
77
|
+
current = storageMetrics([syncConfigMetrics('config-b', 'ACTIVE', 50)]);
|
|
78
|
+
vi.setSystemTime(61_000);
|
|
79
|
+
|
|
80
|
+
expect(await provider()).toEqual([
|
|
81
|
+
{ value: 50, attributes: { sync_config_id: 'config-b', sync_config_state: 'ACTIVE' } },
|
|
82
|
+
{ value: 0, attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' } }
|
|
83
|
+
]);
|
|
84
|
+
|
|
85
|
+
// Once every reader has had a chance to observe the zero, stop reporting it entirely.
|
|
86
|
+
vi.setSystemTime(61_000 + 16 * 60_000);
|
|
87
|
+
|
|
88
|
+
expect(await provider()).toEqual([
|
|
89
|
+
{ value: 50, attributes: { sync_config_id: 'config-b', sync_config_state: 'ACTIVE' } }
|
|
90
|
+
]);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it('treats a state transition as a retired attribute set', async () => {
|
|
94
|
+
let current = storageMetrics([syncConfigMetrics('config-a', 'PROCESSING', 100, 'v1')]);
|
|
95
|
+
const storage = {
|
|
96
|
+
getStorageMetrics: async () => current
|
|
97
|
+
} as unknown as BucketStorageFactory;
|
|
98
|
+
|
|
99
|
+
const { engine, providers } = testEngine();
|
|
100
|
+
initializeCoreStorageMetrics(engine, storage);
|
|
101
|
+
const provider = providers.get(StorageMetric.ATTRIBUTED_SOURCE_RECORDS_BYTES)!;
|
|
102
|
+
|
|
103
|
+
await provider();
|
|
104
|
+
|
|
105
|
+
current = storageMetrics([syncConfigMetrics('config-a', 'ACTIVE', 120, 'v1')]);
|
|
106
|
+
vi.setSystemTime(61_000);
|
|
107
|
+
|
|
108
|
+
expect(await provider()).toEqual([
|
|
109
|
+
{
|
|
110
|
+
value: 120,
|
|
111
|
+
attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE', version_label: 'v1' }
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
value: 0,
|
|
115
|
+
attributes: { sync_config_id: 'config-a', sync_config_state: 'PROCESSING', version_label: 'v1' }
|
|
116
|
+
}
|
|
117
|
+
]);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('does not report anything as retired when reading the metrics fails', async () => {
|
|
121
|
+
let fail = false;
|
|
122
|
+
const storage = {
|
|
123
|
+
getStorageMetrics: async () => {
|
|
124
|
+
if (fail) {
|
|
125
|
+
throw new Error('storage unavailable');
|
|
126
|
+
}
|
|
127
|
+
return storageMetrics([syncConfigMetrics('config-a', 'ACTIVE', 100)]);
|
|
128
|
+
}
|
|
129
|
+
} as unknown as BucketStorageFactory;
|
|
130
|
+
|
|
131
|
+
const { engine, providers } = testEngine();
|
|
132
|
+
initializeCoreStorageMetrics(engine, storage);
|
|
133
|
+
const provider = providers.get(StorageMetric.ATTRIBUTED_OBJECT_STORAGE_BYTES)!;
|
|
134
|
+
|
|
135
|
+
await provider();
|
|
136
|
+
|
|
137
|
+
fail = true;
|
|
138
|
+
vi.setSystemTime(61_000);
|
|
139
|
+
expect(await provider()).toBeUndefined();
|
|
140
|
+
|
|
141
|
+
// The failure must not be interpreted as the sync config having been pruned.
|
|
142
|
+
fail = false;
|
|
143
|
+
vi.setSystemTime(122_000);
|
|
144
|
+
expect(await provider()).toEqual([
|
|
145
|
+
{ value: 100, attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' } }
|
|
146
|
+
]);
|
|
147
|
+
});
|
|
148
|
+
});
|
|
@@ -43,4 +43,19 @@ describe('acquireSemaphoreAbortable', () => {
|
|
|
43
43
|
// Releasing the semaphore should not invoke resolve again
|
|
44
44
|
release();
|
|
45
45
|
});
|
|
46
|
+
|
|
47
|
+
test('does not wait when the signal is already aborted', async () => {
|
|
48
|
+
const semaphore = new Semaphore(1);
|
|
49
|
+
const controller = new AbortController();
|
|
50
|
+
|
|
51
|
+
// Hold the only slot, so acquiring would block indefinitely.
|
|
52
|
+
const result = await acquireSemaphoreAbortable(semaphore, controller.signal);
|
|
53
|
+
expect(result).not.toBe('aborted');
|
|
54
|
+
const [, release] = result as [number, SemaphoreInterface.Releaser];
|
|
55
|
+
|
|
56
|
+
controller.abort();
|
|
57
|
+
expect(await acquireSemaphoreAbortable(semaphore, controller.signal)).toBe('aborted');
|
|
58
|
+
|
|
59
|
+
release();
|
|
60
|
+
});
|
|
46
61
|
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { formatBytes } from '@/index.js';
|
|
2
|
+
import { describe, expect, test } from 'vitest';
|
|
3
|
+
|
|
4
|
+
describe('formatBytes', () => {
|
|
5
|
+
test('formats bytes', () => {
|
|
6
|
+
expect(formatBytes(500)).toBe('500B');
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
test('formats kilobytes', () => {
|
|
10
|
+
expect(formatBytes(1024)).toBe('1.0KB');
|
|
11
|
+
expect(formatBytes(1536)).toBe('1.5KB');
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test('formats megabytes', () => {
|
|
15
|
+
expect(formatBytes(1024 * 1024)).toBe('1.0MB');
|
|
16
|
+
expect(formatBytes(1.5 * 1024 * 1024)).toBe('1.5MB');
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('formats gigabytes', () => {
|
|
20
|
+
expect(formatBytes(1024 * 1024 * 1024)).toBe('1.0GB');
|
|
21
|
+
expect(formatBytes(10.5 * 1024 * 1024 * 1024)).toBe('10.5GB');
|
|
22
|
+
});
|
|
23
|
+
});
|