@powersync/service-core 1.27.1 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +201 -67
- package/dist/api/RouteAPI.d.ts +6 -0
- package/dist/api/api-metrics.js +4 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +5 -0
- package/dist/api/diagnostics.js +17 -2
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/metrics/connection-metrics.d.ts +28 -0
- package/dist/metrics/connection-metrics.js +55 -0
- package/dist/metrics/connection-metrics.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +1 -0
- package/dist/metrics/metrics-index.js +1 -0
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +7 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/modules/loader.js +8 -2
- package/dist/modules/loader.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -0
- package/dist/replication/AbstractReplicationJob.js +17 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +8 -18
- package/dist/replication/AbstractReplicator.js +63 -64
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.js +0 -6
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.js +5 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.js +24 -22
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +19 -27
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +17 -40
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +25 -23
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.d.ts +1 -1
- package/dist/routes/hooks.js +2 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/sync-connection.d.ts +7 -0
- package/dist/routes/sync-connection.js +34 -0
- package/dist/routes/sync-connection.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +30 -22
- package/dist/storage/BucketStorageFactory.js +14 -14
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.js +2 -2
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +23 -2
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/streams/BroadcastIterable.js +8 -7
- package/dist/streams/BroadcastIterable.js.map +1 -1
- package/dist/streams/LastValueSink.d.ts +2 -2
- package/dist/streams/LastValueSink.js +8 -13
- package/dist/streams/LastValueSink.js.map +1 -1
- package/dist/streams/merge.d.ts +15 -31
- package/dist/streams/merge.js +136 -152
- package/dist/streams/merge.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +3 -2
- package/dist/sync/BucketChecksumState.js +12 -5
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +48 -38
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -2
- package/dist/sync/util.js +2 -5
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +7 -0
- package/dist/system/ServiceContext.js +3 -3
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/system/SyncConfigEngine.d.ts +135 -0
- package/dist/system/SyncConfigEngine.js +186 -0
- package/dist/system/SyncConfigEngine.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -0
- package/dist/system/system-index.js +1 -0
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/types.d.ts +4 -0
- package/package.json +8 -9
- package/src/api/RouteAPI.ts +6 -0
- package/src/api/api-metrics.ts +5 -0
- package/src/api/diagnostics.ts +22 -2
- package/src/metrics/connection-metrics.ts +75 -0
- package/src/metrics/metrics-index.ts +1 -0
- package/src/metrics/metrics-interfaces.ts +8 -1
- package/src/modules/loader.ts +16 -2
- package/src/replication/AbstractReplicationJob.ts +17 -1
- package/src/replication/AbstractReplicator.ts +83 -81
- package/src/replication/ReplicationModule.ts +0 -5
- package/src/routes/configure-fastify.ts +7 -1
- package/src/routes/endpoints/admin.ts +27 -20
- package/src/routes/endpoints/socket-route.ts +26 -38
- package/src/routes/endpoints/sync-rules.ts +22 -46
- package/src/routes/endpoints/sync-stream.ts +31 -27
- package/src/routes/hooks.ts +5 -1
- package/src/routes/sync-connection.ts +39 -0
- package/src/storage/BucketStorageFactory.ts +45 -29
- package/src/storage/PersistedSyncConfigContent.ts +2 -2
- package/src/storage/SyncRulesBucketStorage.ts +23 -2
- package/src/streams/BroadcastIterable.ts +8 -7
- package/src/streams/LastValueSink.ts +10 -16
- package/src/streams/merge.ts +149 -169
- package/src/sync/BucketChecksumState.ts +17 -5
- package/src/sync/sync.ts +71 -57
- package/src/sync/util.ts +2 -7
- package/src/system/ServiceContext.ts +8 -3
- package/src/system/SyncConfigEngine.ts +247 -0
- package/src/system/system-index.ts +1 -0
- package/src/util/config/types.ts +4 -0
- package/test/src/AbstractReplicator.test.ts +375 -45
- package/test/src/PersistedSyncConfigContent.test.ts +5 -5
- package/test/src/SyncConfigEngine.test.ts +208 -0
- package/test/src/broadcast_iterable.test.ts +59 -31
- package/test/src/config.test.ts +65 -3
- package/test/src/connection-metrics-export.test.ts +76 -0
- package/test/src/connection-metrics.test.ts +184 -0
- package/test/src/diagnostics.test.ts +3 -3
- package/test/src/last_value_sink.test.ts +34 -0
- package/test/src/merge_iterable.test.ts +304 -55
- package/test/src/module-loader.test.ts +14 -1
- package/test/src/recording-metrics.ts +36 -0
- package/test/src/routes/admin.test.ts +2 -2
- package/test/src/routes/concurrency-limit.test.ts +108 -0
- package/test/src/routes/connection-metrics.test.ts +269 -0
- package/test/src/routes/invalid-config.test.ts +184 -0
- package/test/src/routes/mocks.ts +26 -13
- package/test/src/routes/stream-disconnect.integration.test.ts +129 -0
- package/test/src/routes/stream.test.ts +200 -107
- package/test/src/routes/sync-rules.test.ts +19 -1
- package/test/src/source_table_config.test.ts +7 -7
- package/test/src/sync/BucketChecksumState.test.ts +10 -10
- package/test/src/sync/stream_response.test.ts +389 -0
- package/test/src/sync/util.test.ts +14 -3
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import type { RouteAPI } from '@/api/RouteAPI.js';
|
|
2
|
+
import { ServiceContextContainer, ServiceContextMode } from '@/system/ServiceContext.js';
|
|
3
|
+
import { SyncConfigEngine, SyncConfigRejectionReason, SyncConfigResultStatus } from '@/system/SyncConfigEngine.js';
|
|
4
|
+
import { CompoundConfigCollector } from '@/util/config/compound-config-collector.js';
|
|
5
|
+
import * as syncConfigParser from '@powersync/service-sync-rules';
|
|
6
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
7
|
+
import { tmpdir } from 'node:os';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
10
|
+
import { mockServiceContext } from './routes/mocks.js';
|
|
11
|
+
|
|
12
|
+
const CONTENT = /* yaml */ `
|
|
13
|
+
# sync config fixture.
|
|
14
|
+
config:
|
|
15
|
+
edition: 3
|
|
16
|
+
streams:
|
|
17
|
+
orders:
|
|
18
|
+
query: SELECT * FROM orders
|
|
19
|
+
`;
|
|
20
|
+
|
|
21
|
+
function source(): RouteAPI {
|
|
22
|
+
const adapter = mockServiceContext(null).routerEngine.getAPI();
|
|
23
|
+
adapter.getParseSyncRulesOptions = () => ({
|
|
24
|
+
defaultSchema: 'source_db',
|
|
25
|
+
connections: { default: { type: 'mongodb' } }
|
|
26
|
+
});
|
|
27
|
+
adapter.validateSourceCapabilities = async () => [];
|
|
28
|
+
return adapter;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
describe('SyncConfigEngine', () => {
|
|
32
|
+
it('constructs the shared engine before source adapter registration', async () => {
|
|
33
|
+
const configuration = await new CompoundConfigCollector().collectConfig({
|
|
34
|
+
config_base64: Buffer.from(
|
|
35
|
+
JSON.stringify({ storage: { type: 'mongodb' }, sync_config: { content: CONTENT } })
|
|
36
|
+
).toString('base64')
|
|
37
|
+
});
|
|
38
|
+
const context = new ServiceContextContainer({ serviceMode: ServiceContextMode.API, configuration });
|
|
39
|
+
const adapter = source();
|
|
40
|
+
const parseOptions = vi.spyOn(adapter, 'getParseSyncRulesOptions');
|
|
41
|
+
const engine = context.syncConfigEngine;
|
|
42
|
+
// Source modules register after context construction; creating the engine must not resolve the adapter.
|
|
43
|
+
context.routerEngine.registerAPI(adapter);
|
|
44
|
+
expect(context.syncConfigEngine).toBe(engine);
|
|
45
|
+
expect(parseOptions).not.toHaveBeenCalled();
|
|
46
|
+
expect((await engine.getConfigured()).status).toBe(SyncConfigResultStatus.ACCEPTED);
|
|
47
|
+
expect(parseOptions).toHaveBeenCalledOnce();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('shares a single source-aware parse and capability result between concurrent consumers', async () => {
|
|
51
|
+
const adapter = source();
|
|
52
|
+
const capability = vi.spyOn(adapter, 'validateSourceCapabilities').mockImplementation(async (config) => {
|
|
53
|
+
expect(config.getSourceTables()[0].schema).toBe('source_db');
|
|
54
|
+
return [{ level: 'warning', message: 'Advisory' }];
|
|
55
|
+
});
|
|
56
|
+
const parse = vi.spyOn(syncConfigParser, 'parseSyncConfig');
|
|
57
|
+
try {
|
|
58
|
+
const settings = {
|
|
59
|
+
present: true,
|
|
60
|
+
exit_on_error: false,
|
|
61
|
+
content: CONTENT
|
|
62
|
+
};
|
|
63
|
+
const service = new SyncConfigEngine(settings, () => adapter);
|
|
64
|
+
// Startup and diagnostics can race on first use; both must share the same in-flight parse.
|
|
65
|
+
const first = service.getConfigured();
|
|
66
|
+
expect(service.getConfigured()).toBe(first);
|
|
67
|
+
const [replication, diagnostics] = await Promise.all([first, service.getConfigured()]);
|
|
68
|
+
expect(replication).toBe(diagnostics);
|
|
69
|
+
expect(replication.status).toBe(SyncConfigResultStatus.ACCEPTED);
|
|
70
|
+
expect(replication).toMatchObject({
|
|
71
|
+
diagnostics: [expect.objectContaining({ level: 'warning', message: 'Advisory' })]
|
|
72
|
+
});
|
|
73
|
+
expect(capability).toHaveBeenCalledOnce();
|
|
74
|
+
expect(parse).toHaveBeenCalledOnce();
|
|
75
|
+
settings.content = 'streams: [';
|
|
76
|
+
expect(await service.getConfigured()).toBe(replication);
|
|
77
|
+
} finally {
|
|
78
|
+
parse.mockRestore();
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('retains capability errors and locations in the common rejection decision', async () => {
|
|
83
|
+
const adapter = source();
|
|
84
|
+
vi.spyOn(adapter, 'validateSourceCapabilities').mockResolvedValue([
|
|
85
|
+
{ level: 'fatal', message: 'Unsupported executor', location: { start_offset: 0, end_offset: 5 } }
|
|
86
|
+
]);
|
|
87
|
+
const result = await new SyncConfigEngine({ present: false, exit_on_error: true }, () => adapter).validate(CONTENT);
|
|
88
|
+
expect(result).toMatchObject({
|
|
89
|
+
status: SyncConfigResultStatus.REJECTED,
|
|
90
|
+
reason: SyncConfigRejectionReason.CAPABILITY,
|
|
91
|
+
diagnostics: [expect.objectContaining({ level: 'fatal', location: { start_offset: 0, end_offset: 5 } })]
|
|
92
|
+
});
|
|
93
|
+
expect(result).not.toHaveProperty('compiled');
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it.each([
|
|
97
|
+
{ content: 'streams: [', hasParsedConfig: true },
|
|
98
|
+
{ content: '# empty config\n', hasParsedConfig: false }
|
|
99
|
+
])(
|
|
100
|
+
'checks capabilities for malformed input whenever a config was produced (parsed=$hasParsedConfig)',
|
|
101
|
+
async ({ content, hasParsedConfig }) => {
|
|
102
|
+
const adapter = source();
|
|
103
|
+
const capability = vi.spyOn(adapter, 'validateSourceCapabilities');
|
|
104
|
+
const result = await new SyncConfigEngine({ present: false, exit_on_error: false }, () => adapter).validate(
|
|
105
|
+
content
|
|
106
|
+
);
|
|
107
|
+
expect(result).toMatchObject({
|
|
108
|
+
status: SyncConfigResultStatus.REJECTED,
|
|
109
|
+
reason: SyncConfigRejectionReason.PARSE
|
|
110
|
+
});
|
|
111
|
+
expect(result.diagnostics.some((diagnostic) => diagnostic.level === 'fatal')).toBe(true);
|
|
112
|
+
// Partial parses still need capability diagnostics; a parser exception leaves no config to check.
|
|
113
|
+
expect(capability).toHaveBeenCalledTimes(hasParsedConfig ? 1 : 0);
|
|
114
|
+
}
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
it.each([{ capabilityCheckThrows: false }, { capabilityCheckThrows: true }])(
|
|
118
|
+
'collects source-capability failures alongside fatal parser diagnostics (throws=$capabilityCheckThrows)',
|
|
119
|
+
async ({ capabilityCheckThrows }) => {
|
|
120
|
+
const adapter = source();
|
|
121
|
+
const capability = vi.spyOn(adapter, 'validateSourceCapabilities').mockImplementation(async () => {
|
|
122
|
+
if (capabilityCheckThrows) {
|
|
123
|
+
throw new Error('Unsupported executor');
|
|
124
|
+
}
|
|
125
|
+
return [{ level: 'fatal', message: 'Unsupported executor' }];
|
|
126
|
+
});
|
|
127
|
+
const result = await new SyncConfigEngine({ present: false, exit_on_error: false }, () => adapter).validate(
|
|
128
|
+
CONTENT.replace('SELECT * FROM orders', 'SELECT FROM')
|
|
129
|
+
);
|
|
130
|
+
expect(capability).toHaveBeenCalledOnce();
|
|
131
|
+
expect(result).toMatchObject({
|
|
132
|
+
status: SyncConfigResultStatus.REJECTED,
|
|
133
|
+
reason: SyncConfigRejectionReason.PARSE
|
|
134
|
+
});
|
|
135
|
+
// Keep both failures for diagnostics, with parsing as the primary rejection reason.
|
|
136
|
+
const fatal = result.diagnostics.filter((diagnostic) => diagnostic.level === 'fatal');
|
|
137
|
+
expect(fatal.some((diagnostic) => diagnostic.message === 'Unsupported executor')).toBe(true);
|
|
138
|
+
expect(fatal.some((diagnostic) => diagnostic.message !== 'Unsupported executor')).toBe(true);
|
|
139
|
+
expect(result).not.toHaveProperty('compiled');
|
|
140
|
+
}
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
it('validates submissions without changing the API-managed snapshot', async () => {
|
|
144
|
+
const adapter = source();
|
|
145
|
+
const parseOptions = vi.spyOn(adapter, 'getParseSyncRulesOptions');
|
|
146
|
+
const getRouteAPI = vi.fn(() => adapter);
|
|
147
|
+
const service = new SyncConfigEngine({ present: false, exit_on_error: false }, getRouteAPI);
|
|
148
|
+
const configured = await service.getConfigured();
|
|
149
|
+
expect(getRouteAPI).not.toHaveBeenCalled();
|
|
150
|
+
expect(parseOptions).not.toHaveBeenCalled();
|
|
151
|
+
expect(configured.status).toBe(SyncConfigResultStatus.ABSENT);
|
|
152
|
+
// An API submission must not become this runner's configured snapshot.
|
|
153
|
+
expect((await service.validate('streams: [')).status).toBe(SyncConfigResultStatus.REJECTED);
|
|
154
|
+
expect((await service.validate(CONTENT)).status).toBe(SyncConfigResultStatus.ACCEPTED);
|
|
155
|
+
expect(await service.getConfigured()).toBe(configured);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it.each([
|
|
159
|
+
{ inputKind: 'valid', initial: CONTENT },
|
|
160
|
+
{ inputKind: 'invalid query', initial: CONTENT.replace('SELECT * FROM orders', 'SELECT FROM') }
|
|
161
|
+
])('retains the first filesystem result until a new engine is created: $inputKind', async ({ initial }) => {
|
|
162
|
+
const dir = await mkdtemp(join(tmpdir(), 'sync-config-snapshot-'));
|
|
163
|
+
try {
|
|
164
|
+
const path = join(dir, 'sync-config.yaml');
|
|
165
|
+
const settings = { present: true, path, exit_on_error: false };
|
|
166
|
+
const adapter = source();
|
|
167
|
+
await writeFile(path, initial);
|
|
168
|
+
const engine = new SyncConfigEngine(settings, () => adapter);
|
|
169
|
+
const first = await engine.getConfigured();
|
|
170
|
+
expect(first.status).toBe(
|
|
171
|
+
initial === CONTENT ? SyncConfigResultStatus.ACCEPTED : SyncConfigResultStatus.REJECTED
|
|
172
|
+
);
|
|
173
|
+
const replacement = initial === CONTENT ? CONTENT.replace('SELECT * FROM orders', 'SELECT FROM') : CONTENT;
|
|
174
|
+
// File edits take effect after restart, represented here by constructing a new engine.
|
|
175
|
+
await writeFile(path, replacement);
|
|
176
|
+
expect(await engine.getConfigured()).toBe(first);
|
|
177
|
+
expect(await new SyncConfigEngine(settings, () => adapter).getConfigured()).toMatchObject({
|
|
178
|
+
content: replacement,
|
|
179
|
+
status: replacement === CONTENT ? SyncConfigResultStatus.ACCEPTED : SyncConfigResultStatus.REJECTED
|
|
180
|
+
});
|
|
181
|
+
} finally {
|
|
182
|
+
await rm(dir, { recursive: true, force: true });
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it('retains an unavailable file result until a new engine is created', async () => {
|
|
187
|
+
const dir = await mkdtemp(join(tmpdir(), 'sync-config-unavailable-'));
|
|
188
|
+
try {
|
|
189
|
+
const settings = { present: true, path: join(dir, 'sync-config.yaml'), exit_on_error: false };
|
|
190
|
+
const adapter = source();
|
|
191
|
+
const parseOptions = vi.spyOn(adapter, 'getParseSyncRulesOptions');
|
|
192
|
+
const getRouteAPI = vi.fn(() => adapter);
|
|
193
|
+
const engine = new SyncConfigEngine(settings, getRouteAPI);
|
|
194
|
+
const first = await engine.getConfigured();
|
|
195
|
+
expect(first).toMatchObject({ status: SyncConfigResultStatus.UNAVAILABLE, error: expect.any(Error) });
|
|
196
|
+
// A file appearing later must not silently switch the running process to another snapshot.
|
|
197
|
+
await writeFile(settings.path, CONTENT);
|
|
198
|
+
expect(await engine.getConfigured()).toBe(first);
|
|
199
|
+
expect(getRouteAPI).not.toHaveBeenCalled();
|
|
200
|
+
expect(parseOptions).not.toHaveBeenCalled();
|
|
201
|
+
expect((await new SyncConfigEngine(settings, () => adapter).getConfigured()).status).toBe(
|
|
202
|
+
SyncConfigResultStatus.ACCEPTED
|
|
203
|
+
);
|
|
204
|
+
} finally {
|
|
205
|
+
await rm(dir, { recursive: true, force: true });
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
});
|
|
@@ -1,35 +1,43 @@
|
|
|
1
1
|
import { BroadcastIterable, IterableSource } from '@/streams/BroadcastIterable.js';
|
|
2
|
-
import { AsyncIterableX, interval } from 'ix/asynciterable/index.js';
|
|
3
|
-
import { delayEach } from 'ix/asynciterable/operators/delayeach.js';
|
|
4
|
-
import { take } from 'ix/asynciterable/operators/take.js';
|
|
5
|
-
import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
|
|
6
|
-
import { toArray } from 'ix/asynciterable/toarray.js';
|
|
7
2
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
8
3
|
|
|
9
4
|
// The global setTimeout, not the one from timers/promises: vi.useFakeTimers() only replaces the global timers
|
|
10
5
|
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
11
6
|
|
|
7
|
+
async function collect<T>(source: AsyncIterable<T>, options: { delayMs?: number; limit?: number } = {}): Promise<T[]> {
|
|
8
|
+
const results: T[] = [];
|
|
9
|
+
for await (const value of source) {
|
|
10
|
+
if (options.delayMs) await sleep(options.delayMs);
|
|
11
|
+
results.push(value);
|
|
12
|
+
if (results.length === options.limit) break;
|
|
13
|
+
}
|
|
14
|
+
return results;
|
|
15
|
+
}
|
|
16
|
+
|
|
12
17
|
describe('BroadcastIterable', () => {
|
|
13
18
|
afterEach(() => {
|
|
14
19
|
vi.useRealTimers();
|
|
15
20
|
});
|
|
16
21
|
|
|
17
22
|
it('should iterate', async () => {
|
|
18
|
-
const range =
|
|
19
|
-
|
|
23
|
+
const range = async function* () {
|
|
24
|
+
yield* [1, 2, 3];
|
|
25
|
+
};
|
|
26
|
+
const broadcast = new BroadcastIterable(range);
|
|
20
27
|
|
|
21
|
-
const results = await
|
|
28
|
+
const results = await collect(broadcast);
|
|
22
29
|
expect(results).toEqual([1, 2, 3]);
|
|
23
30
|
expect(broadcast.active).toBe(false);
|
|
24
31
|
});
|
|
25
32
|
|
|
26
33
|
it('should skip values if sink is slow', async () => {
|
|
27
|
-
const range =
|
|
28
|
-
|
|
34
|
+
const range = async function* () {
|
|
35
|
+
yield* [1, 2, 3];
|
|
36
|
+
};
|
|
37
|
+
const broadcast = new BroadcastIterable(range);
|
|
29
38
|
|
|
30
39
|
// Delay reading each element
|
|
31
|
-
const
|
|
32
|
-
const results = await toArray(delayed);
|
|
40
|
+
const results = await collect(broadcast, { delayMs: 1 });
|
|
33
41
|
|
|
34
42
|
// We only get a single value, then done.
|
|
35
43
|
expect(results).toEqual([1]);
|
|
@@ -37,32 +45,58 @@ describe('BroadcastIterable', () => {
|
|
|
37
45
|
});
|
|
38
46
|
|
|
39
47
|
it('should abort', async () => {
|
|
40
|
-
const range =
|
|
48
|
+
const range = async function* () {
|
|
49
|
+
yield* [1, 2, 3];
|
|
50
|
+
};
|
|
41
51
|
let recordedSignal: AbortSignal | undefined;
|
|
42
52
|
const broadcast = new BroadcastIterable((signal) => {
|
|
43
53
|
recordedSignal = signal;
|
|
44
|
-
return range;
|
|
54
|
+
return range();
|
|
45
55
|
});
|
|
46
56
|
|
|
47
57
|
const controller = new AbortController();
|
|
48
58
|
const iter = broadcast[Symbol.asyncIterator](controller.signal);
|
|
49
59
|
await iter.next();
|
|
50
60
|
controller.abort();
|
|
51
|
-
await expect(iter.next()).rejects.
|
|
61
|
+
await expect(iter.next()).rejects.toBe(controller.signal.reason);
|
|
52
62
|
expect(recordedSignal!.aborted).toEqual(true);
|
|
63
|
+
expect(recordedSignal!.reason).toBe(controller.signal.reason);
|
|
53
64
|
});
|
|
54
65
|
|
|
66
|
+
it.each([new Error('subscriber cancelled'), 'cancelled', 0, false, null])(
|
|
67
|
+
'preserves subscriber abort reason %s when stopping the shared source',
|
|
68
|
+
async (reason) => {
|
|
69
|
+
let sourceSignal: AbortSignal | undefined;
|
|
70
|
+
const broadcast = new BroadcastIterable(async function* (signal) {
|
|
71
|
+
sourceSignal = signal;
|
|
72
|
+
yield 1;
|
|
73
|
+
if (!signal.aborted) {
|
|
74
|
+
await new Promise<void>((resolve) => signal.addEventListener('abort', () => resolve(), { once: true }));
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
const iterator = broadcast[Symbol.asyncIterator](controller.signal);
|
|
79
|
+
await iterator.next();
|
|
80
|
+
controller.abort(reason);
|
|
81
|
+
await expect(iterator.next()).rejects.toBe(reason);
|
|
82
|
+
expect(sourceSignal!.reason).toBe(reason);
|
|
83
|
+
}
|
|
84
|
+
);
|
|
85
|
+
|
|
55
86
|
it('should handle indefinite sources', async () => {
|
|
56
87
|
vi.useFakeTimers();
|
|
57
|
-
const source: IterableSource<number> = (signal)
|
|
58
|
-
|
|
88
|
+
const source: IterableSource<number> = async function* (signal) {
|
|
89
|
+
for (let i = 0; !signal.aborted; i++) {
|
|
90
|
+
await sleep(1);
|
|
91
|
+
signal.throwIfAborted();
|
|
92
|
+
yield i;
|
|
93
|
+
}
|
|
59
94
|
};
|
|
60
95
|
|
|
61
96
|
const broadcast = new BroadcastIterable(source);
|
|
62
97
|
|
|
63
|
-
const delayed = delayEach(10)(broadcast);
|
|
64
98
|
expect(broadcast.active).toBe(false);
|
|
65
|
-
const resultsPromise =
|
|
99
|
+
const resultsPromise = collect(broadcast, { delayMs: 10, limit: 5 });
|
|
66
100
|
await vi.advanceTimersByTimeAsync(100);
|
|
67
101
|
const results = await resultsPromise;
|
|
68
102
|
|
|
@@ -93,11 +127,9 @@ describe('BroadcastIterable', () => {
|
|
|
93
127
|
|
|
94
128
|
const broadcast = new BroadcastIterable(source);
|
|
95
129
|
|
|
96
|
-
const delayed1 = delayEach(9)(broadcast);
|
|
97
|
-
const delayed2 = delayEach(10)(broadcast);
|
|
98
130
|
expect(broadcast.active).toBe(false);
|
|
99
|
-
const results1Promise =
|
|
100
|
-
const results2Promise =
|
|
131
|
+
const results1Promise = collect(broadcast, { delayMs: 9, limit: 5 });
|
|
132
|
+
const results2Promise = collect(broadcast, { delayMs: 10, limit: 5 });
|
|
101
133
|
await vi.advanceTimersByTimeAsync(100);
|
|
102
134
|
const [results1, results2] = [await results1Promise, await results2Promise];
|
|
103
135
|
|
|
@@ -114,8 +146,7 @@ describe('BroadcastIterable', () => {
|
|
|
114
146
|
expect(results2[4]).toBeLessThan(1045);
|
|
115
147
|
|
|
116
148
|
// This starts a new source
|
|
117
|
-
const
|
|
118
|
-
const results3Promise = toArray(take(5)(delayed3));
|
|
149
|
+
const results3Promise = collect(broadcast, { delayMs: 10, limit: 5 });
|
|
119
150
|
await vi.advanceTimersByTimeAsync(100);
|
|
120
151
|
const results3 = await results3Promise;
|
|
121
152
|
expect(results3.length).toEqual(5);
|
|
@@ -146,11 +177,9 @@ describe('BroadcastIterable', () => {
|
|
|
146
177
|
|
|
147
178
|
const broadcast = new BroadcastIterable(source);
|
|
148
179
|
|
|
149
|
-
const delayed1 = delayEach(9)(broadcast);
|
|
150
|
-
const delayed2 = delayEach(10)(broadcast);
|
|
151
180
|
expect(broadcast.active).toBe(false);
|
|
152
|
-
const results1Promise =
|
|
153
|
-
const results2Promise =
|
|
181
|
+
const results1Promise = collect(broadcast, { delayMs: 9, limit: 5 }) as Promise<number[]>;
|
|
182
|
+
const results2Promise = collect(broadcast, { delayMs: 10, limit: 5 }) as Promise<number[]>;
|
|
154
183
|
|
|
155
184
|
const settled = Promise.allSettled([results1Promise, results2Promise]);
|
|
156
185
|
await vi.advanceTimersByTimeAsync(100);
|
|
@@ -162,8 +191,7 @@ describe('BroadcastIterable', () => {
|
|
|
162
191
|
expect(broadcast.active).toBe(false);
|
|
163
192
|
|
|
164
193
|
// This starts a new source
|
|
165
|
-
const
|
|
166
|
-
const results3Promise = toArray(take(5)(delayed3));
|
|
194
|
+
const results3Promise = collect(broadcast, { delayMs: 10, limit: 5 });
|
|
167
195
|
await vi.advanceTimersByTimeAsync(100);
|
|
168
196
|
const results3 = await results3Promise;
|
|
169
197
|
expect(results3.length).toEqual(5);
|
package/test/src/config.test.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Vitest Unit Tests
|
|
2
2
|
import { CompoundConfigCollector } from '@/index.js';
|
|
3
|
+
import fs from 'node:fs/promises';
|
|
3
4
|
import { describe, expect, it, vi } from 'vitest';
|
|
4
5
|
|
|
5
6
|
describe('Config', () => {
|
|
@@ -251,7 +252,7 @@ describe('Config', () => {
|
|
|
251
252
|
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
252
253
|
});
|
|
253
254
|
|
|
254
|
-
expect(result.sync_rules).
|
|
255
|
+
expect(result.sync_rules).toMatchObject({
|
|
255
256
|
present: true,
|
|
256
257
|
version_label: 'v6',
|
|
257
258
|
exit_on_error: true,
|
|
@@ -281,7 +282,7 @@ describe('Config', () => {
|
|
|
281
282
|
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
282
283
|
});
|
|
283
284
|
|
|
284
|
-
expect(result.sync_rules).
|
|
285
|
+
expect(result.sync_rules).toMatchObject({
|
|
285
286
|
present: true,
|
|
286
287
|
exit_on_error: true,
|
|
287
288
|
content: expect.stringContaining('edition: 2')
|
|
@@ -312,7 +313,7 @@ describe('Config', () => {
|
|
|
312
313
|
sync_config_base64: Buffer.from(yamlSyncConfig, 'utf-8').toString('base64')
|
|
313
314
|
});
|
|
314
315
|
|
|
315
|
-
expect(result.sync_rules).
|
|
316
|
+
expect(result.sync_rules).toMatchObject({
|
|
316
317
|
present: true,
|
|
317
318
|
exit_on_error: true,
|
|
318
319
|
content: expect.stringContaining('edition: 2')
|
|
@@ -345,3 +346,64 @@ describe('Config', () => {
|
|
|
345
346
|
).rejects.toThrow(/Both `sync_config` and `sync_rules` are present/);
|
|
346
347
|
});
|
|
347
348
|
});
|
|
349
|
+
|
|
350
|
+
const LOADED_TEST_CONFIG = /* yaml */ `
|
|
351
|
+
# Input used to test source selection during configuration resolution.
|
|
352
|
+
config:
|
|
353
|
+
edition: 3
|
|
354
|
+
streams:
|
|
355
|
+
todos:
|
|
356
|
+
query: SELECT id FROM todos
|
|
357
|
+
`;
|
|
358
|
+
|
|
359
|
+
describe('resolved sync config input sources', () => {
|
|
360
|
+
it.each([
|
|
361
|
+
{ name: 'omitted', sync: undefined, base64: false, present: false },
|
|
362
|
+
{ name: 'empty section', sync: {}, base64: false, present: false },
|
|
363
|
+
{ name: 'options without content', sync: { exit_on_error: false }, base64: false, present: false },
|
|
364
|
+
{ name: 'inline', sync: { content: LOADED_TEST_CONFIG }, base64: false, present: true },
|
|
365
|
+
{ name: 'Base64', sync: undefined, base64: true, present: true },
|
|
366
|
+
{ name: 'empty section with Base64', sync: {}, base64: true, present: true }
|
|
367
|
+
])('loads $name', async ({ sync, base64, present }) => {
|
|
368
|
+
const result = await new CompoundConfigCollector().collectConfig({
|
|
369
|
+
config_base64: Buffer.from(JSON.stringify({ storage: { type: 'mongodb' }, sync_config: sync })).toString(
|
|
370
|
+
'base64'
|
|
371
|
+
),
|
|
372
|
+
sync_config_base64: base64 ? Buffer.from(LOADED_TEST_CONFIG).toString('base64') : undefined
|
|
373
|
+
});
|
|
374
|
+
expect(result.sync_rules.present).toBe(present);
|
|
375
|
+
expect(result.sync_rules.content).toBe(present ? LOADED_TEST_CONFIG : undefined);
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
it('defers configured file loading to the sync config engine', async () => {
|
|
379
|
+
// Configuration collection records the path; source-aware admission owns reading the file.
|
|
380
|
+
using readFile = vi.spyOn(fs, 'readFile');
|
|
381
|
+
const result = await new CompoundConfigCollector().collectConfig({
|
|
382
|
+
config_base64: Buffer.from(
|
|
383
|
+
JSON.stringify({
|
|
384
|
+
storage: { type: 'mongodb' },
|
|
385
|
+
sync_config: { path: '/nonexistent-powersync-source-test/sync-config.yaml' }
|
|
386
|
+
})
|
|
387
|
+
).toString('base64')
|
|
388
|
+
});
|
|
389
|
+
expect(result.sync_rules.present).toBe(true);
|
|
390
|
+
expect(result.sync_rules.path).toBe('/nonexistent-powersync-source-test/sync-config.yaml');
|
|
391
|
+
expect(readFile).not.toHaveBeenCalled();
|
|
392
|
+
});
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
describe('deferred sync config parsing', () => {
|
|
396
|
+
it.each([{ exitOnError: true }, { exitOnError: false }])(
|
|
397
|
+
'preserves malformed input for source-aware validation (exit_on_error=$exitOnError)',
|
|
398
|
+
async ({ exitOnError }) => {
|
|
399
|
+
// Keep malformed input available for diagnostics regardless of the startup failure policy.
|
|
400
|
+
const content = '# empty sync config\n';
|
|
401
|
+
const config = await new CompoundConfigCollector().collectConfig({
|
|
402
|
+
config_base64: Buffer.from(
|
|
403
|
+
JSON.stringify({ storage: { type: 'mongodb' }, sync_config: { content, exit_on_error: exitOnError } })
|
|
404
|
+
).toString('base64')
|
|
405
|
+
});
|
|
406
|
+
expect(config.sync_rules.content).toBe(content);
|
|
407
|
+
}
|
|
408
|
+
);
|
|
409
|
+
});
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createCoreAPIMetrics,
|
|
3
|
+
initializeCoreAPIMetrics,
|
|
4
|
+
MetricsEngine,
|
|
5
|
+
OpenTelemetryMetricsFactory,
|
|
6
|
+
recordSyncConnection,
|
|
7
|
+
SyncCloseReason,
|
|
8
|
+
SyncTransport
|
|
9
|
+
} from '@/index.js';
|
|
10
|
+
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
|
11
|
+
import { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
12
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
13
|
+
import { createServer } from 'node:http';
|
|
14
|
+
import { AddressInfo } from 'node:net';
|
|
15
|
+
import { expect, it } from 'vitest';
|
|
16
|
+
|
|
17
|
+
it('scrapes only observed connection series and omits error_code for ok closes', async () => {
|
|
18
|
+
const exporter = new PrometheusExporter({ preventServerStart: true });
|
|
19
|
+
const provider = new MeterProvider({ readers: [exporter] });
|
|
20
|
+
const engine = new MetricsEngine({
|
|
21
|
+
factory: new OpenTelemetryMetricsFactory(provider.getMeter('connection-metrics-test')),
|
|
22
|
+
disable_telemetry_sharing: true
|
|
23
|
+
});
|
|
24
|
+
createCoreAPIMetrics(engine);
|
|
25
|
+
initializeCoreAPIMetrics(engine);
|
|
26
|
+
const server = createServer((request, response) => exporter.getMetricsRequestHandler(request, response));
|
|
27
|
+
try {
|
|
28
|
+
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
|
29
|
+
const address = server.address() as AddressInfo;
|
|
30
|
+
const scrape = async () => {
|
|
31
|
+
const response = await fetch(`http://127.0.0.1:${address.port}/metrics`, { signal: AbortSignal.timeout(2_000) });
|
|
32
|
+
expect(response.status).toBe(200);
|
|
33
|
+
return (await response.text()).split('\n').filter((line) => line.startsWith('powersync_sync_connections_total{'));
|
|
34
|
+
};
|
|
35
|
+
const errorSeries = (lines: string[]) =>
|
|
36
|
+
lines.find((line) =>
|
|
37
|
+
['outcome="error"', 'close_reason="stream_error"', 'error_code="PSYNC_S2403"', 'transport="http_stream"'].every(
|
|
38
|
+
(label) => line.includes(label)
|
|
39
|
+
)
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
expect(await scrape()).toEqual([]);
|
|
43
|
+
|
|
44
|
+
recordSyncConnection(engine, {
|
|
45
|
+
transport: SyncTransport.HttpStream,
|
|
46
|
+
closeReason: SyncCloseReason.StreamError,
|
|
47
|
+
error: new ServiceError(ErrorCode.PSYNC_S2403, 'Storage query timed out')
|
|
48
|
+
});
|
|
49
|
+
const afterFailure = await scrape();
|
|
50
|
+
expect(afterFailure).toHaveLength(1);
|
|
51
|
+
expect(errorSeries(afterFailure)).toMatch(/ 1$/);
|
|
52
|
+
|
|
53
|
+
for (const [index, error] of [
|
|
54
|
+
undefined,
|
|
55
|
+
new ServiceError(ErrorCode.PSYNC_S2403, 'Ignored on an ok close')
|
|
56
|
+
].entries()) {
|
|
57
|
+
recordSyncConnection(engine, {
|
|
58
|
+
transport: SyncTransport.HttpStream,
|
|
59
|
+
closeReason: SyncCloseReason.ClientClosed,
|
|
60
|
+
error
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const afterClose = await scrape();
|
|
64
|
+
expect(afterClose).toHaveLength(2);
|
|
65
|
+
expect(errorSeries(afterClose)).toMatch(/ 1$/);
|
|
66
|
+
const okSeries = afterClose.find((line) => line.includes('outcome="ok"'));
|
|
67
|
+
expect(okSeries).toContain('close_reason="client_closed"');
|
|
68
|
+
expect(okSeries).toContain('transport="http_stream"');
|
|
69
|
+
expect(okSeries).not.toContain('error_code=');
|
|
70
|
+
expect(okSeries).toMatch(new RegExp(` ${index + 1}$`));
|
|
71
|
+
}
|
|
72
|
+
} finally {
|
|
73
|
+
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
74
|
+
await provider.shutdown();
|
|
75
|
+
}
|
|
76
|
+
});
|