@powersync/service-core 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223080959
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -8
- package/dist/api/RouteAPI.d.ts +2 -2
- package/dist/api/diagnostics.js +14 -6
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/JwtPayload.d.ts +7 -8
- package/dist/auth/JwtPayload.js +19 -1
- package/dist/auth/JwtPayload.js.map +1 -1
- package/dist/auth/KeyStore.js +2 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/auth.d.ts +0 -1
- package/dist/routes/auth.js +2 -4
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +84 -0
- package/dist/routes/configure-fastify.js +0 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +171 -0
- package/dist/routes/endpoints/admin.js +36 -21
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +3 -3
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +4 -10
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +10 -13
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +3 -8
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/router.d.ts +0 -1
- package/dist/routes/router.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +29 -15
- package/dist/storage/BucketStorageFactory.js +58 -1
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +28 -4
- package/dist/storage/PersistedSyncRulesContent.js +56 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReportStorage.d.ts +1 -8
- package/dist/storage/StorageVersionConfig.d.ts +20 -0
- package/dist/storage/StorageVersionConfig.js +20 -0
- package/dist/storage/StorageVersionConfig.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +8 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +4 -6
- package/dist/sync/BucketChecksumState.js +4 -9
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.d.ts +0 -8
- package/dist/sync/sync.js +9 -19
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +0 -22
- package/dist/sync/util.js +0 -24
- package/dist/sync/util.js.map +1 -1
- package/dist/util/config.js +4 -1
- package/dist/util/config.js.map +1 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +2 -2
- package/src/api/diagnostics.ts +16 -7
- package/src/auth/JwtPayload.ts +16 -8
- package/src/auth/KeyStore.ts +1 -1
- package/src/replication/AbstractReplicator.ts +3 -5
- package/src/routes/auth.ts +2 -4
- package/src/routes/configure-fastify.ts +0 -1
- package/src/routes/endpoints/admin.ts +45 -26
- package/src/routes/endpoints/checkpointing.ts +5 -3
- package/src/routes/endpoints/socket-route.ts +4 -11
- package/src/routes/endpoints/sync-rules.ts +18 -17
- package/src/routes/endpoints/sync-stream.ts +4 -8
- package/src/routes/router.ts +0 -2
- package/src/storage/BucketStorageFactory.ts +67 -19
- package/src/storage/PersistedSyncRulesContent.ts +82 -5
- package/src/storage/ReportStorage.ts +3 -9
- package/src/storage/StorageVersionConfig.ts +30 -0
- package/src/storage/SyncRulesBucketStorage.ts +9 -0
- package/src/storage/storage-index.ts +1 -0
- package/src/sync/BucketChecksumState.ts +10 -13
- package/src/sync/sync.ts +15 -42
- package/src/sync/util.ts +0 -25
- package/src/util/config.ts +7 -2
- package/test/src/auth.test.ts +76 -20
- package/test/src/routes/admin.test.ts +48 -0
- package/test/src/routes/mocks.ts +22 -1
- package/test/src/routes/stream.test.ts +10 -9
- package/test/src/sync/BucketChecksumState.test.ts +92 -84
- package/test/tsconfig.json +3 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
|
-
import { SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
|
|
2
|
+
import { SourceSchema, SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
|
|
3
3
|
import { internal_routes } from '@powersync/service-types';
|
|
4
4
|
|
|
5
|
+
import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
|
|
5
6
|
import * as api from '../../api/api-index.js';
|
|
6
7
|
import * as storage from '../../storage/storage-index.js';
|
|
7
8
|
import { authApi } from '../auth.js';
|
|
8
9
|
import { routeDefinition } from '../router.js';
|
|
9
10
|
|
|
11
|
+
/**
|
|
12
|
+
* @deprecated This will be removed in a future release
|
|
13
|
+
*/
|
|
10
14
|
export const executeSql = routeDefinition({
|
|
11
15
|
path: '/api/admin/v1/execute-sql',
|
|
12
16
|
method: router.HTTPMethod.POST,
|
|
@@ -127,12 +131,13 @@ export const reprocess = routeDefinition({
|
|
|
127
131
|
});
|
|
128
132
|
}
|
|
129
133
|
|
|
130
|
-
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
134
|
+
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
135
|
+
storage.updateSyncRulesFromYaml(active.sync_rules.config.content, {
|
|
136
|
+
// These sync rules already passed validation. But if the rules are not valid anymore due
|
|
137
|
+
// to a service change, we do want to report the error here.
|
|
138
|
+
validate: true
|
|
139
|
+
})
|
|
140
|
+
);
|
|
136
141
|
|
|
137
142
|
const baseConfig = await apiHandler.getSourceConfig();
|
|
138
143
|
|
|
@@ -149,6 +154,35 @@ export const reprocess = routeDefinition({
|
|
|
149
154
|
}
|
|
150
155
|
});
|
|
151
156
|
|
|
157
|
+
class FakeSyncRulesContentForValidation extends storage.PersistedSyncRulesContent {
|
|
158
|
+
constructor(
|
|
159
|
+
private readonly apiHandler: api.RouteAPI,
|
|
160
|
+
private readonly schema: SourceSchema,
|
|
161
|
+
data: storage.PersistedSyncRulesContentData
|
|
162
|
+
) {
|
|
163
|
+
super(data);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
current_lock: storage.ReplicationLock | null = null;
|
|
167
|
+
|
|
168
|
+
async lock(): Promise<storage.ReplicationLock> {
|
|
169
|
+
throw new Error('Lock not implemented');
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
|
|
173
|
+
return {
|
|
174
|
+
...this,
|
|
175
|
+
sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, {
|
|
176
|
+
...this.apiHandler.getParseSyncRulesOptions(),
|
|
177
|
+
schema: this.schema
|
|
178
|
+
}),
|
|
179
|
+
hydratedSyncRules() {
|
|
180
|
+
return this.sync_rules.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
152
186
|
export const validate = routeDefinition({
|
|
153
187
|
path: '/api/admin/v1/validate',
|
|
154
188
|
method: router.HTTPMethod.POST,
|
|
@@ -164,30 +198,15 @@ export const validate = routeDefinition({
|
|
|
164
198
|
const schemaData = await api.getConnectionsSchema(apiHandler);
|
|
165
199
|
const schema = new StaticSchema(schemaData.connections);
|
|
166
200
|
|
|
167
|
-
const sync_rules
|
|
201
|
+
const sync_rules = new FakeSyncRulesContentForValidation(apiHandler, schema, {
|
|
168
202
|
// Dummy values
|
|
169
203
|
id: 0,
|
|
170
204
|
slot_name: '',
|
|
171
205
|
active: false,
|
|
172
206
|
last_checkpoint_lsn: '',
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
...this,
|
|
177
|
-
sync_rules: SqlSyncRules.fromYaml(content, {
|
|
178
|
-
...apiHandler.getParseSyncRulesOptions(),
|
|
179
|
-
schema
|
|
180
|
-
}),
|
|
181
|
-
hydratedSyncRules() {
|
|
182
|
-
return this.sync_rules.hydrate();
|
|
183
|
-
}
|
|
184
|
-
};
|
|
185
|
-
},
|
|
186
|
-
sync_rules_content: content,
|
|
187
|
-
async lock() {
|
|
188
|
-
throw new Error('Lock not implemented');
|
|
189
|
-
}
|
|
190
|
-
};
|
|
207
|
+
storageVersion: storage.LEGACY_STORAGE_VERSION,
|
|
208
|
+
sync_rules_content: content
|
|
209
|
+
});
|
|
191
210
|
|
|
192
211
|
const connectionStatus = await apiHandler.getConnectionStatus();
|
|
193
212
|
if (!connectionStatus) {
|
|
@@ -52,18 +52,20 @@ export const writeCheckpoint2 = routeDefinition({
|
|
|
52
52
|
authorize: authUser,
|
|
53
53
|
validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
|
|
54
54
|
handler: async (payload) => {
|
|
55
|
-
const {
|
|
55
|
+
const { token_payload, service_context } = payload.context;
|
|
56
56
|
|
|
57
57
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
58
58
|
|
|
59
59
|
const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
|
|
60
|
-
userId:
|
|
60
|
+
userId: token_payload!.userIdString,
|
|
61
61
|
clientId: payload.params.client_id,
|
|
62
62
|
api: apiHandler,
|
|
63
63
|
storage: service_context.storageEngine.activeBucketStorage
|
|
64
64
|
});
|
|
65
65
|
|
|
66
|
-
logger.info(
|
|
66
|
+
logger.info(
|
|
67
|
+
`Write checkpoint for ${token_payload!.userIdString}/${payload.params.client_id}: ${writeCheckpoint} | ${replicationHead}`
|
|
68
|
+
);
|
|
67
69
|
|
|
68
70
|
return {
|
|
69
71
|
write_checkpoint: String(writeCheckpoint)
|
|
@@ -18,19 +18,18 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
18
18
|
|
|
19
19
|
logger.defaultMeta = {
|
|
20
20
|
...logger.defaultMeta,
|
|
21
|
-
user_id: context.token_payload
|
|
21
|
+
user_id: context.token_payload!.userIdJson,
|
|
22
22
|
client_id: params.client_id,
|
|
23
23
|
user_agent: context.user_agent
|
|
24
24
|
};
|
|
25
25
|
|
|
26
26
|
const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
|
|
27
27
|
client_id: params.client_id ?? '',
|
|
28
|
-
user_id: context.
|
|
28
|
+
user_id: context.token_payload!.userIdString,
|
|
29
29
|
user_agent: context.user_agent,
|
|
30
30
|
// At this point the token_payload is guaranteed to be present
|
|
31
31
|
jwt_exp: new Date(context.token_payload!.exp * 1000),
|
|
32
|
-
connected_at: new Date(streamStart)
|
|
33
|
-
app_metadata: params.app_metadata
|
|
32
|
+
connected_at: new Date(streamStart)
|
|
34
33
|
};
|
|
35
34
|
|
|
36
35
|
// Best effort guess on why the stream was closed.
|
|
@@ -122,12 +121,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
122
121
|
tracker,
|
|
123
122
|
signal,
|
|
124
123
|
logger,
|
|
125
|
-
isEncodingAsBson: true
|
|
126
|
-
event: {
|
|
127
|
-
engine: service_context.eventsEngine,
|
|
128
|
-
user_id: sdkData.user_id,
|
|
129
|
-
client_id: sdkData.client_id
|
|
130
|
-
}
|
|
124
|
+
isEncodingAsBson: true
|
|
131
125
|
})) {
|
|
132
126
|
if (signal.aborted) {
|
|
133
127
|
break;
|
|
@@ -138,7 +132,6 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
138
132
|
|
|
139
133
|
{
|
|
140
134
|
const serialized = sync.syncLineToBson(data);
|
|
141
|
-
|
|
142
135
|
responder.onNext({ data: serialized }, false);
|
|
143
136
|
requestedN--;
|
|
144
137
|
tracker.addPlaintextDataSynced(serialized.length);
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
|
-
import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
2
|
+
import { SqlSyncRules, SyncConfigWithErrors, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
3
3
|
import type { FastifyPluginAsync } from 'fastify';
|
|
4
4
|
import * as t from 'ts-codec';
|
|
5
5
|
|
|
6
6
|
import { RouteAPI } from '../../api/RouteAPI.js';
|
|
7
7
|
import { authApi } from '../auth.js';
|
|
8
8
|
import { routeDefinition } from '../router.js';
|
|
9
|
+
import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
|
|
9
10
|
|
|
10
11
|
const DeploySyncRulesRequest = t.object({
|
|
11
12
|
content: t.string
|
|
@@ -51,10 +52,11 @@ export const deploySyncRules = routeDefinition({
|
|
|
51
52
|
});
|
|
52
53
|
}
|
|
53
54
|
const content = payload.params.content;
|
|
55
|
+
let syncConfig: SyncConfigWithErrors;
|
|
54
56
|
|
|
55
57
|
try {
|
|
56
58
|
const apiHandler = service_context.routerEngine.getAPI();
|
|
57
|
-
SqlSyncRules.fromYaml(payload.params.content, {
|
|
59
|
+
syncConfig = SqlSyncRules.fromYaml(payload.params.content, {
|
|
58
60
|
...apiHandler.getParseSyncRulesOptions(),
|
|
59
61
|
// We don't do any schema-level validation at this point
|
|
60
62
|
schema: undefined
|
|
@@ -68,11 +70,9 @@ export const deploySyncRules = routeDefinition({
|
|
|
68
70
|
});
|
|
69
71
|
}
|
|
70
72
|
|
|
71
|
-
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
validate: false
|
|
75
|
-
});
|
|
73
|
+
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(
|
|
74
|
+
updateSyncRulesFromConfig(syncConfig.config)
|
|
75
|
+
);
|
|
76
76
|
|
|
77
77
|
return {
|
|
78
78
|
slot_name: sync_rules.slot_name
|
|
@@ -168,12 +168,13 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
168
168
|
});
|
|
169
169
|
}
|
|
170
170
|
|
|
171
|
-
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
171
|
+
const new_rules = await activeBucketStorage.updateSyncRules(
|
|
172
|
+
updateSyncRulesFromYaml(sync_rules.sync_rules.config.content, {
|
|
173
|
+
// These sync rules already passed validation. But if the rules are not valid anymore due
|
|
174
|
+
// to a service change, we do want to report the error here.
|
|
175
|
+
validate: true
|
|
176
|
+
})
|
|
177
|
+
);
|
|
177
178
|
return {
|
|
178
179
|
slot_name: new_rules.slot_name
|
|
179
180
|
};
|
|
@@ -197,14 +198,14 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
|
|
|
197
198
|
// No schema-based validation at this point
|
|
198
199
|
schema: undefined
|
|
199
200
|
});
|
|
200
|
-
const source_table_patterns = rules.getSourceTables();
|
|
201
|
-
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
201
|
+
const source_table_patterns = rules.config.getSourceTables();
|
|
202
|
+
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);
|
|
202
203
|
|
|
203
204
|
return {
|
|
204
205
|
valid: true,
|
|
205
|
-
bucket_definitions: rules.debugRepresentation(),
|
|
206
|
+
bucket_definitions: rules.config.debugRepresentation(),
|
|
206
207
|
source_tables: resolved_tables,
|
|
207
|
-
data_tables: rules.debugGetOutputTables()
|
|
208
|
+
data_tables: rules.config.debugGetOutputTables()
|
|
208
209
|
};
|
|
209
210
|
} catch (e) {
|
|
210
211
|
if (e instanceof SyncRulesErrors) {
|
|
@@ -42,12 +42,12 @@ export const syncStreamed = routeDefinition({
|
|
|
42
42
|
...logger.defaultMeta,
|
|
43
43
|
user_agent: userAgent,
|
|
44
44
|
client_id: clientId,
|
|
45
|
-
user_id: payload.context.
|
|
45
|
+
user_id: payload.context.token_payload!.userIdJson,
|
|
46
46
|
bson: useBson
|
|
47
47
|
};
|
|
48
48
|
const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
|
|
49
49
|
client_id: clientId ?? '',
|
|
50
|
-
user_id: payload.context.
|
|
50
|
+
user_id: payload.context.token_payload!.userIdString,
|
|
51
51
|
user_agent: userAgent as string,
|
|
52
52
|
// At this point the token_payload is guaranteed to be present
|
|
53
53
|
jwt_exp: new Date(token_payload!.exp * 1000),
|
|
@@ -98,12 +98,7 @@ export const syncStreamed = routeDefinition({
|
|
|
98
98
|
tracker,
|
|
99
99
|
signal: controller.signal,
|
|
100
100
|
logger,
|
|
101
|
-
isEncodingAsBson: useBson
|
|
102
|
-
event: {
|
|
103
|
-
engine: service_context.eventsEngine,
|
|
104
|
-
user_id: sdkData.user_id,
|
|
105
|
-
client_id: sdkData.client_id
|
|
106
|
-
}
|
|
101
|
+
isEncodingAsBson: useBson
|
|
107
102
|
});
|
|
108
103
|
|
|
109
104
|
const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
|
|
@@ -142,6 +137,7 @@ export const syncStreamed = routeDefinition({
|
|
|
142
137
|
logger.error('Streaming sync request failed', error);
|
|
143
138
|
}
|
|
144
139
|
});
|
|
140
|
+
|
|
145
141
|
return new router.RouterResponse({
|
|
146
142
|
status: 200,
|
|
147
143
|
headers: {
|
package/src/routes/router.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
|
|
3
3
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
4
4
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
5
5
|
import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
|
|
6
6
|
import { ReportStorage } from './ReportStorage.js';
|
|
7
|
+
import { SqlSyncRules, SyncConfig } from '@powersync/service-sync-rules';
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Represents a configured storage provider.
|
|
@@ -13,23 +14,41 @@ import { ReportStorage } from './ReportStorage.js';
|
|
|
13
14
|
*
|
|
14
15
|
* Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
|
|
15
16
|
*/
|
|
16
|
-
export
|
|
17
|
+
export abstract class BucketStorageFactory
|
|
18
|
+
extends BaseObserver<BucketStorageFactoryListener>
|
|
19
|
+
implements AsyncDisposable
|
|
20
|
+
{
|
|
17
21
|
/**
|
|
18
22
|
* Update sync rules from configuration, if changed.
|
|
19
23
|
*/
|
|
20
|
-
configureSyncRules(
|
|
24
|
+
async configureSyncRules(
|
|
21
25
|
options: UpdateSyncRulesOptions
|
|
22
|
-
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }
|
|
26
|
+
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }> {
|
|
27
|
+
const next = await this.getNextSyncRulesContent();
|
|
28
|
+
const active = await this.getActiveSyncRulesContent();
|
|
29
|
+
|
|
30
|
+
if (next?.sync_rules_content == options.config.yaml) {
|
|
31
|
+
logger.info('Sync rules from configuration unchanged');
|
|
32
|
+
return { updated: false };
|
|
33
|
+
} else if (next == null && active?.sync_rules_content == options.config.yaml) {
|
|
34
|
+
logger.info('Sync rules from configuration unchanged');
|
|
35
|
+
return { updated: false };
|
|
36
|
+
} else {
|
|
37
|
+
logger.info('Sync rules updated from configuration');
|
|
38
|
+
const persisted_sync_rules = await this.updateSyncRules(options);
|
|
39
|
+
return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
|
|
40
|
+
}
|
|
41
|
+
}
|
|
23
42
|
|
|
24
43
|
/**
|
|
25
44
|
* Get a storage instance to query sync data for specific sync rules.
|
|
26
45
|
*/
|
|
27
|
-
getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
46
|
+
abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
28
47
|
|
|
29
48
|
/**
|
|
30
49
|
* Deploy new sync rules.
|
|
31
50
|
*/
|
|
32
|
-
updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
51
|
+
abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
33
52
|
|
|
34
53
|
/**
|
|
35
54
|
* Indicate that a slot was removed, and we should re-sync by creating
|
|
@@ -41,57 +60,65 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
|
|
|
41
60
|
*
|
|
42
61
|
* Replication should be restarted after this.
|
|
43
62
|
*/
|
|
44
|
-
restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
63
|
+
abstract restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
45
64
|
|
|
46
65
|
/**
|
|
47
66
|
* Get the sync rules used for querying.
|
|
48
67
|
*/
|
|
49
|
-
getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null
|
|
68
|
+
async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
69
|
+
const content = await this.getActiveSyncRulesContent();
|
|
70
|
+
return content?.parsed(options) ?? null;
|
|
71
|
+
}
|
|
50
72
|
|
|
51
73
|
/**
|
|
52
74
|
* Get the sync rules used for querying.
|
|
53
75
|
*/
|
|
54
|
-
getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
76
|
+
abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
55
77
|
|
|
56
78
|
/**
|
|
57
79
|
* Get the sync rules that will be active next once done with initial replicatino.
|
|
58
80
|
*/
|
|
59
|
-
getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null
|
|
81
|
+
async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
82
|
+
const content = await this.getNextSyncRulesContent();
|
|
83
|
+
return content?.parsed(options) ?? null;
|
|
84
|
+
}
|
|
60
85
|
|
|
61
86
|
/**
|
|
62
87
|
* Get the sync rules that will be active next once done with initial replicatino.
|
|
63
88
|
*/
|
|
64
|
-
getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
89
|
+
abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
65
90
|
|
|
66
91
|
/**
|
|
67
92
|
* Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
|
|
68
93
|
*/
|
|
69
|
-
getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
94
|
+
abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
70
95
|
|
|
71
96
|
/**
|
|
72
97
|
* Get all sync rules stopped but not terminated yet.
|
|
73
98
|
*/
|
|
74
|
-
getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
99
|
+
abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
75
100
|
|
|
76
101
|
/**
|
|
77
102
|
* Get the active storage instance.
|
|
78
103
|
*/
|
|
79
|
-
getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
104
|
+
abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
80
105
|
|
|
81
106
|
/**
|
|
82
107
|
* Get storage size of active sync rules.
|
|
83
108
|
*/
|
|
84
|
-
getStorageMetrics(): Promise<StorageMetrics>;
|
|
109
|
+
abstract getStorageMetrics(): Promise<StorageMetrics>;
|
|
85
110
|
|
|
86
111
|
/**
|
|
87
112
|
* Get the unique identifier for this instance of Powersync
|
|
88
113
|
*/
|
|
89
|
-
getPowerSyncInstanceId(): Promise<string>;
|
|
114
|
+
abstract getPowerSyncInstanceId(): Promise<string>;
|
|
90
115
|
|
|
91
116
|
/**
|
|
92
117
|
* Get a unique identifier for the system used for storage.
|
|
93
118
|
*/
|
|
94
|
-
getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
119
|
+
abstract getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
120
|
+
|
|
121
|
+
abstract [Symbol.asyncDispose](): PromiseLike<void>;
|
|
95
122
|
}
|
|
96
123
|
|
|
97
124
|
export interface BucketStorageFactoryListener {
|
|
@@ -119,9 +146,30 @@ export interface StorageMetrics {
|
|
|
119
146
|
}
|
|
120
147
|
|
|
121
148
|
export interface UpdateSyncRulesOptions {
|
|
122
|
-
|
|
149
|
+
config: {
|
|
150
|
+
yaml: string;
|
|
151
|
+
// TODO: Add serialized sync plan if available
|
|
152
|
+
};
|
|
123
153
|
lock?: boolean;
|
|
124
|
-
|
|
154
|
+
storageVersion?: number;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function updateSyncRulesFromYaml(
|
|
158
|
+
content: string,
|
|
159
|
+
options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
|
|
160
|
+
): UpdateSyncRulesOptions {
|
|
161
|
+
const { config } = SqlSyncRules.fromYaml(content, {
|
|
162
|
+
// No schema-based validation at this point
|
|
163
|
+
schema: undefined,
|
|
164
|
+
defaultSchema: 'not_applicable', // Not needed for validation
|
|
165
|
+
throwOnError: options?.validate ?? false
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
return updateSyncRulesFromConfig(config, options);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function updateSyncRulesFromConfig(parsed: SyncConfig, options?: Omit<UpdateSyncRulesOptions, 'config'>) {
|
|
172
|
+
return { config: { yaml: parsed.content }, ...options };
|
|
125
173
|
}
|
|
126
174
|
|
|
127
175
|
export interface GetIntanceOptions {
|
|
@@ -1,11 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
CompatibilityOption,
|
|
3
|
+
DEFAULT_HYDRATION_STATE,
|
|
4
|
+
HydratedSyncRules,
|
|
5
|
+
HydrationState,
|
|
6
|
+
SqlSyncRules,
|
|
7
|
+
SyncConfigWithErrors,
|
|
8
|
+
versionedHydrationState
|
|
9
|
+
} from '@powersync/service-sync-rules';
|
|
2
10
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
11
|
+
import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
|
|
12
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
13
|
+
import { UpdateSyncRulesOptions } from './BucketStorageFactory.js';
|
|
3
14
|
|
|
4
15
|
export interface ParseSyncRulesOptions {
|
|
5
16
|
defaultSchema: string;
|
|
6
17
|
}
|
|
7
18
|
|
|
8
|
-
export interface
|
|
19
|
+
export interface PersistedSyncRulesContentData {
|
|
9
20
|
readonly id: number;
|
|
10
21
|
readonly sync_rules_content: string;
|
|
11
22
|
readonly slot_name: string;
|
|
@@ -13,6 +24,7 @@ export interface PersistedSyncRulesContent {
|
|
|
13
24
|
* True if this is the "active" copy of the sync rules.
|
|
14
25
|
*/
|
|
15
26
|
readonly active: boolean;
|
|
27
|
+
readonly storageVersion: number;
|
|
16
28
|
|
|
17
29
|
readonly last_checkpoint_lsn: string | null;
|
|
18
30
|
|
|
@@ -20,15 +32,80 @@ export interface PersistedSyncRulesContent {
|
|
|
20
32
|
readonly last_fatal_error_ts?: Date | null;
|
|
21
33
|
readonly last_keepalive_ts?: Date | null;
|
|
22
34
|
readonly last_checkpoint_ts?: Date | null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export abstract class PersistedSyncRulesContent implements PersistedSyncRulesContentData {
|
|
38
|
+
readonly id!: number;
|
|
39
|
+
readonly sync_rules_content!: string;
|
|
40
|
+
readonly slot_name!: string;
|
|
41
|
+
readonly active!: boolean;
|
|
42
|
+
readonly storageVersion!: number;
|
|
43
|
+
|
|
44
|
+
readonly last_checkpoint_lsn!: string | null;
|
|
45
|
+
|
|
46
|
+
readonly last_fatal_error?: string | null;
|
|
47
|
+
readonly last_fatal_error_ts?: Date | null;
|
|
48
|
+
readonly last_keepalive_ts?: Date | null;
|
|
49
|
+
readonly last_checkpoint_ts?: Date | null;
|
|
50
|
+
|
|
51
|
+
abstract readonly current_lock: ReplicationLock | null;
|
|
52
|
+
|
|
53
|
+
constructor(data: PersistedSyncRulesContentData) {
|
|
54
|
+
Object.assign(this, data);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Load the storage config.
|
|
59
|
+
*
|
|
60
|
+
* This may throw if the persisted storage version is not supported.
|
|
61
|
+
*/
|
|
62
|
+
getStorageConfig(): StorageVersionConfig {
|
|
63
|
+
const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
|
|
64
|
+
if (storageConfig == null) {
|
|
65
|
+
throw new ServiceError(
|
|
66
|
+
ErrorCode.PSYNC_S1005,
|
|
67
|
+
`Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
return storageConfig;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
parsed(options: ParseSyncRulesOptions): PersistedSyncRules {
|
|
74
|
+
let hydrationState: HydrationState;
|
|
75
|
+
const syncRules = SqlSyncRules.fromYaml(this.sync_rules_content, options);
|
|
76
|
+
const storageConfig = this.getStorageConfig();
|
|
77
|
+
if (
|
|
78
|
+
storageConfig.versionedBuckets ||
|
|
79
|
+
syncRules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)
|
|
80
|
+
) {
|
|
81
|
+
hydrationState = versionedHydrationState(this.id);
|
|
82
|
+
} else {
|
|
83
|
+
hydrationState = DEFAULT_HYDRATION_STATE;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
id: this.id,
|
|
88
|
+
slot_name: this.slot_name,
|
|
89
|
+
sync_rules: syncRules,
|
|
90
|
+
hydratedSyncRules: () => {
|
|
91
|
+
return syncRules.config.hydrate({ hydrationState });
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
}
|
|
23
95
|
|
|
24
|
-
|
|
96
|
+
asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
|
|
97
|
+
return {
|
|
98
|
+
config: { yaml: this.sync_rules_content },
|
|
99
|
+
...options
|
|
100
|
+
};
|
|
101
|
+
}
|
|
25
102
|
|
|
26
|
-
lock(): Promise<ReplicationLock>;
|
|
103
|
+
abstract lock(): Promise<ReplicationLock>;
|
|
27
104
|
}
|
|
28
105
|
|
|
29
106
|
export interface PersistedSyncRules {
|
|
30
107
|
readonly id: number;
|
|
31
|
-
readonly sync_rules:
|
|
108
|
+
readonly sync_rules: SyncConfigWithErrors;
|
|
32
109
|
readonly slot_name: string;
|
|
33
110
|
|
|
34
111
|
hydratedSyncRules(): HydratedSyncRules;
|
|
@@ -28,7 +28,9 @@ export interface ReportStorage extends AsyncDisposable {
|
|
|
28
28
|
* Usually this is call on the start of the new day, week or month. It will return all unique completed connections
|
|
29
29
|
* as well as uniques currently connected clients.
|
|
30
30
|
*/
|
|
31
|
-
getClientConnectionReports(
|
|
31
|
+
getClientConnectionReports(
|
|
32
|
+
data: event_types.ClientConnectionReportRequest
|
|
33
|
+
): Promise<event_types.ClientConnectionReportResponse>;
|
|
32
34
|
/**
|
|
33
35
|
* Get a paginated list of client connection events
|
|
34
36
|
* This will return a paginated list of connections for a client/ user ID or all if neither is provided, within a date range if provided
|
|
@@ -41,12 +43,4 @@ export interface ReportStorage extends AsyncDisposable {
|
|
|
41
43
|
* This is used to clean up old connection data that is no longer needed.
|
|
42
44
|
*/
|
|
43
45
|
deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void>;
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Report a sync analytics.
|
|
47
|
-
*/
|
|
48
|
-
reportSyncAnalyticsEvent(data: event_types.SyncAnalyticsEventData): Promise<void>;
|
|
49
|
-
getSyncCheckpoint(data: event_types.SyncCheckpointRequest): Promise<event_types.PaginatedResponse<any>>;
|
|
50
|
-
getSyncBucketStats(data: event_types.SyncBucketStatsRequest): Promise<any>;
|
|
51
|
-
getLastSyncReport(data: event_types.LastSyncRequest): Promise<any>;
|
|
52
46
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export interface StorageVersionConfig {
|
|
2
|
+
/**
|
|
3
|
+
* Whether versioned bucket names are automatically enabled.
|
|
4
|
+
*
|
|
5
|
+
* If this is false, bucket names may still be versioned depending on the sync config.
|
|
6
|
+
*/
|
|
7
|
+
versionedBuckets: boolean;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Oldest supported storage version.
|
|
12
|
+
*/
|
|
13
|
+
export const LEGACY_STORAGE_VERSION = 1;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Default storage version for newly persisted sync rules.
|
|
17
|
+
*/
|
|
18
|
+
export const CURRENT_STORAGE_VERSION = 2;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Shared storage-version behavior across storage implementations.
|
|
22
|
+
*/
|
|
23
|
+
export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undefined> = {
|
|
24
|
+
[LEGACY_STORAGE_VERSION]: {
|
|
25
|
+
versionedBuckets: false
|
|
26
|
+
},
|
|
27
|
+
[CURRENT_STORAGE_VERSION]: {
|
|
28
|
+
versionedBuckets: true
|
|
29
|
+
}
|
|
30
|
+
};
|
|
@@ -218,10 +218,19 @@ export interface CompactOptions {
|
|
|
218
218
|
moveBatchQueryLimit?: number;
|
|
219
219
|
|
|
220
220
|
/**
|
|
221
|
+
* Minimum number new operations in a bucket to trigger compaction of that bucket.
|
|
222
|
+
*
|
|
221
223
|
* Minimum of 1, default of 10.
|
|
222
224
|
*/
|
|
223
225
|
minBucketChanges?: number;
|
|
224
226
|
|
|
227
|
+
/**
|
|
228
|
+
* Minimum ratio of new operations to existing operations in a bucket to trigger compaction of that bucket.
|
|
229
|
+
*
|
|
230
|
+
* Number between 0 and 1, default of 0.1.
|
|
231
|
+
*/
|
|
232
|
+
minChangeRatio?: number;
|
|
233
|
+
|
|
225
234
|
/**
|
|
226
235
|
* Internal/testing use: Cache size for compacting parameters.
|
|
227
236
|
*/
|