@powersync/service-core 1.19.1 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/dist/api/diagnostics.js +11 -4
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +13 -2
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/config-command.js +2 -2
- package/dist/entry/commands/config-command.js.map +1 -1
- package/dist/replication/AbstractReplicator.js +2 -5
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +84 -0
- package/dist/routes/configure-fastify.js +0 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +168 -0
- package/dist/routes/endpoints/admin.js +33 -20
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +6 -9
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +43 -15
- package/dist/storage/BucketStorageFactory.js +70 -1
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +28 -2
- package/dist/storage/PersistedSyncRulesContent.js +79 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +20 -0
- package/dist/storage/StorageVersionConfig.js +20 -0
- package/dist/storage/StorageVersionConfig.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +2 -1
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +6 -2
- package/dist/sync/BucketChecksumState.js +85 -10
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/util/config/collectors/config-collector.js +13 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +4 -4
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +2 -2
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +3 -3
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -1
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/env.d.ts +1 -0
- package/dist/util/env.js +5 -0
- package/dist/util/env.js.map +1 -1
- package/package.json +5 -5
- package/src/api/diagnostics.ts +12 -4
- package/src/entry/commands/compact-action.ts +15 -2
- package/src/entry/commands/config-command.ts +3 -3
- package/src/replication/AbstractReplicator.ts +3 -5
- package/src/routes/configure-fastify.ts +0 -1
- package/src/routes/endpoints/admin.ts +42 -25
- package/src/routes/endpoints/sync-rules.ts +14 -13
- package/src/storage/BucketStorageFactory.ts +110 -19
- package/src/storage/PersistedSyncRulesContent.ts +114 -4
- package/src/storage/StorageVersionConfig.ts +30 -0
- package/src/storage/SyncRulesBucketStorage.ts +2 -1
- package/src/storage/storage-index.ts +1 -0
- package/src/sync/BucketChecksumState.ts +129 -16
- package/src/util/config/collectors/config-collector.ts +16 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +5 -5
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +4 -4
- package/src/util/config/types.ts +1 -2
- package/src/util/env.ts +5 -0
- package/test/src/config.test.ts +115 -0
- package/test/src/routes/admin.test.ts +48 -0
- package/test/src/routes/mocks.ts +22 -1
- package/test/src/routes/stream.test.ts +3 -2
- package/test/src/sync/BucketChecksumState.test.ts +285 -78
- package/test/tsconfig.json +3 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,9 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
|
|
3
3
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
4
4
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
5
5
|
import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
|
|
6
6
|
import { ReportStorage } from './ReportStorage.js';
|
|
7
|
+
import {
|
|
8
|
+
PrecompiledSyncConfig,
|
|
9
|
+
SerializedCompatibilityContext,
|
|
10
|
+
serializeSyncPlan,
|
|
11
|
+
SqlSyncRules,
|
|
12
|
+
SyncConfig
|
|
13
|
+
} from '@powersync/service-sync-rules';
|
|
7
14
|
|
|
8
15
|
/**
|
|
9
16
|
* Represents a configured storage provider.
|
|
@@ -13,23 +20,41 @@ import { ReportStorage } from './ReportStorage.js';
|
|
|
13
20
|
*
|
|
14
21
|
* Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
|
|
15
22
|
*/
|
|
16
|
-
export
|
|
23
|
+
export abstract class BucketStorageFactory
|
|
24
|
+
extends BaseObserver<BucketStorageFactoryListener>
|
|
25
|
+
implements AsyncDisposable
|
|
26
|
+
{
|
|
17
27
|
/**
|
|
18
28
|
* Update sync rules from configuration, if changed.
|
|
19
29
|
*/
|
|
20
|
-
configureSyncRules(
|
|
30
|
+
async configureSyncRules(
|
|
21
31
|
options: UpdateSyncRulesOptions
|
|
22
|
-
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }
|
|
32
|
+
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }> {
|
|
33
|
+
const next = await this.getNextSyncRulesContent();
|
|
34
|
+
const active = await this.getActiveSyncRulesContent();
|
|
35
|
+
|
|
36
|
+
if (next?.sync_rules_content == options.config.yaml) {
|
|
37
|
+
logger.info('Sync rules from configuration unchanged');
|
|
38
|
+
return { updated: false };
|
|
39
|
+
} else if (next == null && active?.sync_rules_content == options.config.yaml) {
|
|
40
|
+
logger.info('Sync rules from configuration unchanged');
|
|
41
|
+
return { updated: false };
|
|
42
|
+
} else {
|
|
43
|
+
logger.info('Sync rules updated from configuration');
|
|
44
|
+
const persisted_sync_rules = await this.updateSyncRules(options);
|
|
45
|
+
return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
23
48
|
|
|
24
49
|
/**
|
|
25
50
|
* Get a storage instance to query sync data for specific sync rules.
|
|
26
51
|
*/
|
|
27
|
-
getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
52
|
+
abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
|
|
28
53
|
|
|
29
54
|
/**
|
|
30
55
|
* Deploy new sync rules.
|
|
31
56
|
*/
|
|
32
|
-
updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
57
|
+
abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
33
58
|
|
|
34
59
|
/**
|
|
35
60
|
* Indicate that a slot was removed, and we should re-sync by creating
|
|
@@ -41,57 +66,65 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
|
|
|
41
66
|
*
|
|
42
67
|
* Replication should be restarted after this.
|
|
43
68
|
*/
|
|
44
|
-
restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
69
|
+
abstract restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
45
70
|
|
|
46
71
|
/**
|
|
47
72
|
* Get the sync rules used for querying.
|
|
48
73
|
*/
|
|
49
|
-
getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null
|
|
74
|
+
async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
75
|
+
const content = await this.getActiveSyncRulesContent();
|
|
76
|
+
return content?.parsed(options) ?? null;
|
|
77
|
+
}
|
|
50
78
|
|
|
51
79
|
/**
|
|
52
80
|
* Get the sync rules used for querying.
|
|
53
81
|
*/
|
|
54
|
-
getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
82
|
+
abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
55
83
|
|
|
56
84
|
/**
|
|
57
85
|
* Get the sync rules that will be active next once done with initial replicatino.
|
|
58
86
|
*/
|
|
59
|
-
getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null
|
|
87
|
+
async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
88
|
+
const content = await this.getNextSyncRulesContent();
|
|
89
|
+
return content?.parsed(options) ?? null;
|
|
90
|
+
}
|
|
60
91
|
|
|
61
92
|
/**
|
|
62
93
|
* Get the sync rules that will be active next once done with initial replicatino.
|
|
63
94
|
*/
|
|
64
|
-
getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
95
|
+
abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
65
96
|
|
|
66
97
|
/**
|
|
67
98
|
* Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
|
|
68
99
|
*/
|
|
69
|
-
getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
100
|
+
abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
70
101
|
|
|
71
102
|
/**
|
|
72
103
|
* Get all sync rules stopped but not terminated yet.
|
|
73
104
|
*/
|
|
74
|
-
getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
105
|
+
abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
75
106
|
|
|
76
107
|
/**
|
|
77
108
|
* Get the active storage instance.
|
|
78
109
|
*/
|
|
79
|
-
getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
110
|
+
abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
80
111
|
|
|
81
112
|
/**
|
|
82
113
|
* Get storage size of active sync rules.
|
|
83
114
|
*/
|
|
84
|
-
getStorageMetrics(): Promise<StorageMetrics>;
|
|
115
|
+
abstract getStorageMetrics(): Promise<StorageMetrics>;
|
|
85
116
|
|
|
86
117
|
/**
|
|
87
118
|
* Get the unique identifier for this instance of Powersync
|
|
88
119
|
*/
|
|
89
|
-
getPowerSyncInstanceId(): Promise<string>;
|
|
120
|
+
abstract getPowerSyncInstanceId(): Promise<string>;
|
|
90
121
|
|
|
91
122
|
/**
|
|
92
123
|
* Get a unique identifier for the system used for storage.
|
|
93
124
|
*/
|
|
94
|
-
getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
125
|
+
abstract getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
126
|
+
|
|
127
|
+
abstract [Symbol.asyncDispose](): PromiseLike<void>;
|
|
95
128
|
}
|
|
96
129
|
|
|
97
130
|
export interface BucketStorageFactoryListener {
|
|
@@ -119,9 +152,67 @@ export interface StorageMetrics {
|
|
|
119
152
|
}
|
|
120
153
|
|
|
121
154
|
export interface UpdateSyncRulesOptions {
|
|
122
|
-
|
|
155
|
+
config: {
|
|
156
|
+
yaml: string;
|
|
157
|
+
/**
|
|
158
|
+
* The serialized sync plan for the sync configuration, or `null` for configurations not using the sync stream
|
|
159
|
+
* compiler.
|
|
160
|
+
*/
|
|
161
|
+
plan: SerializedSyncPlan | null;
|
|
162
|
+
};
|
|
123
163
|
lock?: boolean;
|
|
124
|
-
|
|
164
|
+
storageVersion?: number;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export interface SerializedSyncPlan {
|
|
168
|
+
/**
|
|
169
|
+
* The serialized plan, from {@link serializeSyncPlan}.
|
|
170
|
+
*/
|
|
171
|
+
plan: unknown;
|
|
172
|
+
compatibility: SerializedCompatibilityContext;
|
|
173
|
+
/**
|
|
174
|
+
* Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
|
|
175
|
+
* yet.
|
|
176
|
+
*
|
|
177
|
+
* We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore
|
|
178
|
+
* them.
|
|
179
|
+
*/
|
|
180
|
+
eventDescriptors: Record<string, string[]>;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function updateSyncRulesFromYaml(
|
|
184
|
+
content: string,
|
|
185
|
+
options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
|
|
186
|
+
): UpdateSyncRulesOptions {
|
|
187
|
+
const { config } = SqlSyncRules.fromYaml(content, {
|
|
188
|
+
// No schema-based validation at this point
|
|
189
|
+
schema: undefined,
|
|
190
|
+
defaultSchema: 'not_applicable', // Not needed for validation
|
|
191
|
+
throwOnError: options?.validate ?? false
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
return updateSyncRulesFromConfig(config, options);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function updateSyncRulesFromConfig(
|
|
198
|
+
parsed: SyncConfig,
|
|
199
|
+
options?: Omit<UpdateSyncRulesOptions, 'config'>
|
|
200
|
+
): UpdateSyncRulesOptions {
|
|
201
|
+
let plan: SerializedSyncPlan | null = null;
|
|
202
|
+
if (parsed instanceof PrecompiledSyncConfig) {
|
|
203
|
+
const eventDescriptors: Record<string, string[]> = {};
|
|
204
|
+
for (const event of parsed.eventDescriptors) {
|
|
205
|
+
eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
plan = {
|
|
209
|
+
compatibility: parsed.compatibility.serialize(),
|
|
210
|
+
plan: serializeSyncPlan(parsed.plan),
|
|
211
|
+
eventDescriptors
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
return { config: { yaml: parsed.content, plan }, ...options };
|
|
125
216
|
}
|
|
126
217
|
|
|
127
218
|
export interface GetIntanceOptions {
|
|
@@ -1,18 +1,36 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
CompatibilityContext,
|
|
3
|
+
CompatibilityOption,
|
|
4
|
+
DEFAULT_HYDRATION_STATE,
|
|
5
|
+
deserializeSyncPlan,
|
|
6
|
+
HydratedSyncRules,
|
|
7
|
+
HydrationState,
|
|
8
|
+
javaScriptExpressionEngine,
|
|
9
|
+
PrecompiledSyncConfig,
|
|
10
|
+
SqlEventDescriptor,
|
|
11
|
+
SqlSyncRules,
|
|
12
|
+
SyncConfigWithErrors,
|
|
13
|
+
versionedHydrationState
|
|
14
|
+
} from '@powersync/service-sync-rules';
|
|
2
15
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
16
|
+
import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
|
|
17
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
18
|
+
import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
|
|
3
19
|
|
|
4
20
|
export interface ParseSyncRulesOptions {
|
|
5
21
|
defaultSchema: string;
|
|
6
22
|
}
|
|
7
23
|
|
|
8
|
-
export interface
|
|
24
|
+
export interface PersistedSyncRulesContentData {
|
|
9
25
|
readonly id: number;
|
|
10
26
|
readonly sync_rules_content: string;
|
|
27
|
+
readonly compiled_plan: SerializedSyncPlan | null;
|
|
11
28
|
readonly slot_name: string;
|
|
12
29
|
/**
|
|
13
30
|
* True if this is the "active" copy of the sync rules.
|
|
14
31
|
*/
|
|
15
32
|
readonly active: boolean;
|
|
33
|
+
readonly storageVersion: number;
|
|
16
34
|
|
|
17
35
|
readonly last_checkpoint_lsn: string | null;
|
|
18
36
|
|
|
@@ -20,10 +38,102 @@ export interface PersistedSyncRulesContent {
|
|
|
20
38
|
readonly last_fatal_error_ts?: Date | null;
|
|
21
39
|
readonly last_keepalive_ts?: Date | null;
|
|
22
40
|
readonly last_checkpoint_ts?: Date | null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export abstract class PersistedSyncRulesContent implements PersistedSyncRulesContentData {
|
|
44
|
+
readonly id!: number;
|
|
45
|
+
readonly sync_rules_content!: string;
|
|
46
|
+
readonly compiled_plan!: SerializedSyncPlan | null;
|
|
47
|
+
readonly slot_name!: string;
|
|
48
|
+
readonly active!: boolean;
|
|
49
|
+
readonly storageVersion!: number;
|
|
50
|
+
|
|
51
|
+
readonly last_checkpoint_lsn!: string | null;
|
|
52
|
+
|
|
53
|
+
readonly last_fatal_error?: string | null;
|
|
54
|
+
readonly last_fatal_error_ts?: Date | null;
|
|
55
|
+
readonly last_keepalive_ts?: Date | null;
|
|
56
|
+
readonly last_checkpoint_ts?: Date | null;
|
|
57
|
+
|
|
58
|
+
abstract readonly current_lock: ReplicationLock | null;
|
|
59
|
+
|
|
60
|
+
constructor(data: PersistedSyncRulesContentData) {
|
|
61
|
+
Object.assign(this, data);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Load the storage config.
|
|
66
|
+
*
|
|
67
|
+
* This may throw if the persisted storage version is not supported.
|
|
68
|
+
*/
|
|
69
|
+
getStorageConfig(): StorageVersionConfig {
|
|
70
|
+
const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
|
|
71
|
+
if (storageConfig == null) {
|
|
72
|
+
throw new ServiceError(
|
|
73
|
+
ErrorCode.PSYNC_S1005,
|
|
74
|
+
`Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
return storageConfig;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
parsed(options: ParseSyncRulesOptions): PersistedSyncRules {
|
|
81
|
+
let hydrationState: HydrationState;
|
|
82
|
+
|
|
83
|
+
// Do we have a compiled sync plan? If so, restore from there instead of parsing everything again.
|
|
84
|
+
let config: SyncConfigWithErrors;
|
|
85
|
+
if (this.compiled_plan != null) {
|
|
86
|
+
const plan = deserializeSyncPlan(this.compiled_plan.plan);
|
|
87
|
+
const compatibility = CompatibilityContext.deserialize(this.compiled_plan.compatibility);
|
|
88
|
+
const eventDefinitions: SqlEventDescriptor[] = [];
|
|
89
|
+
for (const [name, queries] of Object.entries(this.compiled_plan.eventDescriptors)) {
|
|
90
|
+
const descriptor = new SqlEventDescriptor(name, compatibility);
|
|
91
|
+
for (const query of queries) {
|
|
92
|
+
descriptor.addSourceQuery(query, options);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
eventDefinitions.push(descriptor);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
|
|
99
|
+
defaultSchema: options.defaultSchema,
|
|
100
|
+
engine: javaScriptExpressionEngine(compatibility),
|
|
101
|
+
sourceText: this.sync_rules_content
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
config = { config: precompiled, errors: [] };
|
|
105
|
+
} else {
|
|
106
|
+
config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const storageConfig = this.getStorageConfig();
|
|
110
|
+
if (
|
|
111
|
+
storageConfig.versionedBuckets ||
|
|
112
|
+
config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)
|
|
113
|
+
) {
|
|
114
|
+
hydrationState = versionedHydrationState(this.id);
|
|
115
|
+
} else {
|
|
116
|
+
hydrationState = DEFAULT_HYDRATION_STATE;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
id: this.id,
|
|
121
|
+
slot_name: this.slot_name,
|
|
122
|
+
sync_rules: config,
|
|
123
|
+
hydratedSyncRules: () => {
|
|
124
|
+
return config.config.hydrate({ hydrationState });
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
23
128
|
|
|
24
|
-
|
|
129
|
+
asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
|
|
130
|
+
return {
|
|
131
|
+
config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
|
|
132
|
+
...options
|
|
133
|
+
};
|
|
134
|
+
}
|
|
25
135
|
|
|
26
|
-
lock(): Promise<ReplicationLock>;
|
|
136
|
+
abstract lock(): Promise<ReplicationLock>;
|
|
27
137
|
}
|
|
28
138
|
|
|
29
139
|
export interface PersistedSyncRules {
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export interface StorageVersionConfig {
|
|
2
|
+
/**
|
|
3
|
+
* Whether versioned bucket names are automatically enabled.
|
|
4
|
+
*
|
|
5
|
+
* If this is false, bucket names may still be versioned depending on the sync config.
|
|
6
|
+
*/
|
|
7
|
+
versionedBuckets: boolean;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Oldest supported storage version.
|
|
12
|
+
*/
|
|
13
|
+
export const LEGACY_STORAGE_VERSION = 1;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Default storage version for newly persisted sync rules.
|
|
17
|
+
*/
|
|
18
|
+
export const CURRENT_STORAGE_VERSION = 2;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Shared storage-version behavior across storage implementations.
|
|
22
|
+
*/
|
|
23
|
+
export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undefined> = {
|
|
24
|
+
[LEGACY_STORAGE_VERSION]: {
|
|
25
|
+
versionedBuckets: false
|
|
26
|
+
},
|
|
27
|
+
[CURRENT_STORAGE_VERSION]: {
|
|
28
|
+
versionedBuckets: true
|
|
29
|
+
}
|
|
30
|
+
};
|
|
@@ -202,7 +202,8 @@ export interface CompactOptions {
|
|
|
202
202
|
*
|
|
203
203
|
* If not specified, compacts all buckets.
|
|
204
204
|
*
|
|
205
|
-
* These
|
|
205
|
+
* These must be full bucket names (e.g., "global[]", "mybucket[\"user1\"]").
|
|
206
|
+
* Bucket definition names (e.g., "global") are not supported.
|
|
206
207
|
*/
|
|
207
208
|
compactBuckets?: string[];
|
|
208
209
|
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BucketDescription,
|
|
3
|
+
BucketParameterQuerier,
|
|
3
4
|
BucketPriority,
|
|
4
5
|
BucketSource,
|
|
5
6
|
HydratedSyncRules,
|
|
7
|
+
QuerierError,
|
|
6
8
|
RequestedStream,
|
|
7
9
|
RequestParameters,
|
|
8
10
|
ResolvedBucket
|
|
@@ -18,7 +20,6 @@ import {
|
|
|
18
20
|
ServiceAssertionError,
|
|
19
21
|
ServiceError
|
|
20
22
|
} from '@powersync/lib-services-framework';
|
|
21
|
-
import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
|
|
22
23
|
import { JwtPayload } from '../auth/JwtPayload.js';
|
|
23
24
|
import { SyncContext } from './SyncContext.js';
|
|
24
25
|
import { getIntersection, hasIntersection } from './util.js';
|
|
@@ -117,7 +118,7 @@ export class BucketChecksumState {
|
|
|
117
118
|
const storage = this.bucketStorage;
|
|
118
119
|
|
|
119
120
|
const update = await this.parameterState.getCheckpointUpdate(next);
|
|
120
|
-
const { buckets: allBuckets, updatedBuckets } = update;
|
|
121
|
+
const { buckets: allBuckets, updatedBuckets, parameterQueryResultsByDefinition } = update;
|
|
121
122
|
|
|
122
123
|
/** Set of all buckets in this checkpoint. */
|
|
123
124
|
const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
|
|
@@ -213,18 +214,27 @@ export class BucketChecksumState {
|
|
|
213
214
|
});
|
|
214
215
|
|
|
215
216
|
deferredLog = () => {
|
|
217
|
+
const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
|
|
216
218
|
let message = `Updated checkpoint: ${base.checkpoint} | `;
|
|
217
219
|
message += `write: ${writeCheckpoint} | `;
|
|
218
220
|
message += `buckets: ${allBuckets.length} | `;
|
|
221
|
+
if (totalParamResults !== undefined) {
|
|
222
|
+
message += `param_results: ${totalParamResults} | `;
|
|
223
|
+
}
|
|
219
224
|
message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
|
|
220
225
|
message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
226
|
+
logCheckpoint(
|
|
227
|
+
this.logger,
|
|
228
|
+
message,
|
|
229
|
+
{
|
|
230
|
+
checkpoint: base.checkpoint,
|
|
231
|
+
user_id: userIdForLogs,
|
|
232
|
+
buckets: allBuckets.length,
|
|
233
|
+
updated: diff.updatedBuckets.length,
|
|
234
|
+
removed: diff.removedBuckets.length
|
|
235
|
+
},
|
|
236
|
+
totalParamResults
|
|
237
|
+
);
|
|
228
238
|
};
|
|
229
239
|
|
|
230
240
|
checkpointLine = {
|
|
@@ -237,9 +247,23 @@ export class BucketChecksumState {
|
|
|
237
247
|
} satisfies util.StreamingSyncCheckpointDiff;
|
|
238
248
|
} else {
|
|
239
249
|
deferredLog = () => {
|
|
250
|
+
const totalParamResults = computeTotalParamResults(parameterQueryResultsByDefinition);
|
|
240
251
|
let message = `New checkpoint: ${base.checkpoint} | write: ${writeCheckpoint} | `;
|
|
241
|
-
message += `buckets: ${allBuckets.length}
|
|
242
|
-
|
|
252
|
+
message += `buckets: ${allBuckets.length}`;
|
|
253
|
+
if (totalParamResults !== undefined) {
|
|
254
|
+
message += ` | param_results: ${totalParamResults}`;
|
|
255
|
+
}
|
|
256
|
+
message += ` ${limitedBuckets(allBuckets, 20)}`;
|
|
257
|
+
logCheckpoint(
|
|
258
|
+
this.logger,
|
|
259
|
+
message,
|
|
260
|
+
{
|
|
261
|
+
checkpoint: base.checkpoint,
|
|
262
|
+
user_id: userIdForLogs,
|
|
263
|
+
buckets: allBuckets.length
|
|
264
|
+
},
|
|
265
|
+
totalParamResults
|
|
266
|
+
);
|
|
243
267
|
};
|
|
244
268
|
bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
|
|
245
269
|
|
|
@@ -370,6 +394,12 @@ export interface CheckpointUpdate {
|
|
|
370
394
|
* If null, assume that any bucket in `buckets` may have been updated.
|
|
371
395
|
*/
|
|
372
396
|
updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Number of parameter query results per sync stream definition (before deduplication).
|
|
400
|
+
* Map from definition name to count.
|
|
401
|
+
*/
|
|
402
|
+
parameterQueryResultsByDefinition?: Map<string, number>;
|
|
373
403
|
}
|
|
374
404
|
|
|
375
405
|
export class BucketParameterState {
|
|
@@ -502,11 +532,21 @@ export class BucketParameterState {
|
|
|
502
532
|
ErrorCode.PSYNC_S2305,
|
|
503
533
|
`Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`
|
|
504
534
|
);
|
|
505
|
-
|
|
535
|
+
|
|
536
|
+
let errorMessage = error.message;
|
|
537
|
+
const logData: any = {
|
|
506
538
|
checkpoint: checkpoint,
|
|
507
539
|
user_id: this.syncParams.userId,
|
|
508
|
-
|
|
509
|
-
}
|
|
540
|
+
parameter_query_results: update.buckets.length
|
|
541
|
+
};
|
|
542
|
+
|
|
543
|
+
if (update.parameterQueryResultsByDefinition && update.parameterQueryResultsByDefinition.size > 0) {
|
|
544
|
+
const breakdown = formatParameterQueryBreakdown(update.parameterQueryResultsByDefinition);
|
|
545
|
+
errorMessage += breakdown.message;
|
|
546
|
+
logData.parameter_query_results_by_definition = breakdown.countsByDefinition;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
this.logger.error(errorMessage, logData);
|
|
510
550
|
|
|
511
551
|
throw error;
|
|
512
552
|
}
|
|
@@ -562,6 +602,7 @@ export class BucketParameterState {
|
|
|
562
602
|
}
|
|
563
603
|
|
|
564
604
|
let dynamicBuckets: ResolvedBucket[];
|
|
605
|
+
let parameterQueryResultsByDefinition: Map<string, number> | undefined;
|
|
565
606
|
if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
|
|
566
607
|
const recordedLookups = new Set<string>();
|
|
567
608
|
|
|
@@ -574,6 +615,14 @@ export class BucketParameterState {
|
|
|
574
615
|
return checkpoint.base.getParameterSets(lookups);
|
|
575
616
|
}
|
|
576
617
|
});
|
|
618
|
+
|
|
619
|
+
// Count parameter query results per definition (before deduplication)
|
|
620
|
+
parameterQueryResultsByDefinition = new Map<string, number>();
|
|
621
|
+
for (const bucket of dynamicBuckets) {
|
|
622
|
+
const count = parameterQueryResultsByDefinition.get(bucket.definition) ?? 0;
|
|
623
|
+
parameterQueryResultsByDefinition.set(bucket.definition, count + 1);
|
|
624
|
+
}
|
|
625
|
+
|
|
577
626
|
this.cachedDynamicBuckets = dynamicBuckets;
|
|
578
627
|
this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
|
|
579
628
|
this.lookupsFromPreviousCheckpoint = recordedLookups;
|
|
@@ -596,12 +645,14 @@ export class BucketParameterState {
|
|
|
596
645
|
return {
|
|
597
646
|
buckets: allBuckets,
|
|
598
647
|
// We cannot track individual bucket updates for dynamic lookups yet
|
|
599
|
-
updatedBuckets: INVALIDATE_ALL_BUCKETS
|
|
648
|
+
updatedBuckets: INVALIDATE_ALL_BUCKETS,
|
|
649
|
+
parameterQueryResultsByDefinition
|
|
600
650
|
};
|
|
601
651
|
} else {
|
|
602
652
|
return {
|
|
603
653
|
buckets: allBuckets,
|
|
604
|
-
updatedBuckets: updatedBuckets
|
|
654
|
+
updatedBuckets: updatedBuckets,
|
|
655
|
+
parameterQueryResultsByDefinition
|
|
605
656
|
};
|
|
606
657
|
}
|
|
607
658
|
}
|
|
@@ -635,6 +686,68 @@ export interface CheckpointLine {
|
|
|
635
686
|
// Use a more specific type to simplify testing
|
|
636
687
|
export type BucketChecksumStateStorage = Pick<storage.SyncRulesBucketStorage, 'getChecksums'>;
|
|
637
688
|
|
|
689
|
+
/**
|
|
690
|
+
* Compute the total number of parameter query results across all definitions.
|
|
691
|
+
*/
|
|
692
|
+
function computeTotalParamResults(
|
|
693
|
+
parameterQueryResultsByDefinition: Map<string, number> | undefined
|
|
694
|
+
): number | undefined {
|
|
695
|
+
if (!parameterQueryResultsByDefinition) {
|
|
696
|
+
return undefined;
|
|
697
|
+
}
|
|
698
|
+
return Array.from(parameterQueryResultsByDefinition.values()).reduce((sum, count) => sum + count, 0);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Log a checkpoint message, enriching it with parameter query result counts if available.
|
|
703
|
+
*
|
|
704
|
+
* @param logger The logger instance to use
|
|
705
|
+
* @param message The base message string (param_results will NOT be appended — caller includes it if needed)
|
|
706
|
+
* @param logData The base log data object
|
|
707
|
+
* @param totalParamResults The total parameter query results count, or undefined if not applicable
|
|
708
|
+
*/
|
|
709
|
+
function logCheckpoint(
|
|
710
|
+
logger: Logger,
|
|
711
|
+
message: string,
|
|
712
|
+
logData: Record<string, any>,
|
|
713
|
+
totalParamResults: number | undefined
|
|
714
|
+
): void {
|
|
715
|
+
if (totalParamResults !== undefined) {
|
|
716
|
+
logData.parameter_query_results = totalParamResults;
|
|
717
|
+
}
|
|
718
|
+
logger.info(message, logData);
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* Format a breakdown of parameter query results by sync rule definition.
|
|
723
|
+
*
|
|
724
|
+
* Sorts definitions by count (descending), includes the top 10, and returns both the
|
|
725
|
+
* formatted message string and the counts record suitable for structured log data.
|
|
726
|
+
*/
|
|
727
|
+
function formatParameterQueryBreakdown(parameterQueryResultsByDefinition: Map<string, number>): {
|
|
728
|
+
message: string;
|
|
729
|
+
countsByDefinition: Record<string, number>;
|
|
730
|
+
} {
|
|
731
|
+
// Sort definitions by count (descending) and take top 10
|
|
732
|
+
const allSorted = Array.from(parameterQueryResultsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
|
|
733
|
+
const sortedDefinitions = allSorted.slice(0, 10);
|
|
734
|
+
|
|
735
|
+
let message = '\nParameter query results by definition:';
|
|
736
|
+
const countsByDefinition: Record<string, number> = {};
|
|
737
|
+
for (const [definition, count] of sortedDefinitions) {
|
|
738
|
+
message += `\n ${definition}: ${count}`;
|
|
739
|
+
countsByDefinition[definition] = count;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
if (allSorted.length > 10) {
|
|
743
|
+
const remainingResults = allSorted.slice(10).reduce((sum, [, count]) => sum + count, 0);
|
|
744
|
+
const remainingDefinitions = allSorted.length - 10;
|
|
745
|
+
message += `\n ... and ${remainingResults} more results from ${remainingDefinitions} definitions`;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
return { message, countsByDefinition };
|
|
749
|
+
}
|
|
750
|
+
|
|
638
751
|
function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) {
|
|
639
752
|
buckets = buckets.map((b) => {
|
|
640
753
|
if (typeof b != 'string') {
|
|
@@ -40,6 +40,16 @@ export abstract class ConfigCollector {
|
|
|
40
40
|
*/
|
|
41
41
|
const decoded = this.decode(serialized);
|
|
42
42
|
this.validate(decoded);
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* For internal convenience, we duplicate sync_rules and sync_config. Making them interchangeable.
|
|
46
|
+
* Note, we only do this after validation (which only allows one option to be present)
|
|
47
|
+
*/
|
|
48
|
+
if (decoded.sync_config) {
|
|
49
|
+
decoded.sync_rules = decoded.sync_config;
|
|
50
|
+
} else if (decoded.sync_rules) {
|
|
51
|
+
decoded.sync_config = decoded.sync_rules;
|
|
52
|
+
}
|
|
43
53
|
return decoded;
|
|
44
54
|
}
|
|
45
55
|
|
|
@@ -52,6 +62,12 @@ export abstract class ConfigCollector {
|
|
|
52
62
|
if (!valid.valid) {
|
|
53
63
|
throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
|
|
54
64
|
}
|
|
65
|
+
|
|
66
|
+
if (config.sync_config && config.sync_rules) {
|
|
67
|
+
throw new Error(
|
|
68
|
+
'Both `sync_config` and `sync_rules` are present in the service configuration. Please consolidate into one sync_config.'
|
|
69
|
+
);
|
|
70
|
+
}
|
|
55
71
|
}
|
|
56
72
|
|
|
57
73
|
decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig {
|