@powersync/service-core 1.21.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/dist/api/RouteAPI.d.ts +2 -2
- package/dist/api/diagnostics.d.ts +12 -1
- package/dist/api/diagnostics.js +34 -22
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js +3 -2
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/entry/commands/compact-action.js +1 -1
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +6 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +2 -2
- package/dist/replication/AbstractReplicator.js +24 -23
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/RelationCache.d.ts +9 -2
- package/dist/replication/RelationCache.js +21 -2
- package/dist/replication/RelationCache.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +3 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +12 -12
- package/dist/routes/configure-fastify.js +3 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +24 -24
- package/dist/routes/endpoints/admin.js +49 -35
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +22 -13
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +1 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +2 -0
- package/dist/routes/route-register.js +65 -3
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +8 -8
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +29 -6
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +26 -27
- package/dist/storage/BucketStorageFactory.js +5 -19
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
- package/dist/storage/ParsedSyncConfigSet.js +2 -0
- package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/PersistedReplicationStream.d.ts +32 -0
- package/dist/storage/PersistedReplicationStream.js +26 -0
- package/dist/storage/PersistedReplicationStream.js.map +1 -0
- package/dist/storage/PersistedSyncConfigContent.d.ts +60 -0
- package/dist/storage/PersistedSyncConfigContent.js +123 -0
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
- package/dist/storage/PersistedSyncConfigStatus.js +2 -0
- package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
- package/dist/storage/SourceEntity.d.ts +8 -1
- package/dist/storage/SourceTable.d.ts +37 -10
- package/dist/storage/SourceTable.js +50 -14
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +80 -24
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +117 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js +277 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +62 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
- package/dist/storage/storage-index.d.ts +8 -1
- package/dist/storage/storage-index.js +8 -1
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +4 -4
- package/dist/sync/BucketChecksumState.js +1 -1
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -2
- package/dist/sync/sync.js +5 -0
- package/dist/sync/sync.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +17 -1
- package/dist/tracing/PerformanceTracer.js +3 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -1
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -1
- package/dist/util/lsn.d.ts +4 -0
- package/dist/util/lsn.js +10 -0
- package/dist/util/lsn.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +5 -0
- package/dist/util/utils.js +7 -0
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -7
- package/src/api/RouteAPI.ts +2 -2
- package/src/api/diagnostics.ts +45 -28
- package/src/auth/RemoteJWKSCollector.ts +3 -1
- package/src/entry/commands/compact-action.ts +1 -1
- package/src/modules/AbstractModule.ts +6 -2
- package/src/replication/AbstractReplicator.ts +27 -26
- package/src/replication/RelationCache.ts +23 -4
- package/src/replication/ReplicationModule.ts +2 -3
- package/src/routes/configure-fastify.ts +8 -1
- package/src/routes/endpoints/admin.ts +75 -49
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +23 -13
- package/src/routes/endpoints/sync-stream.ts +1 -1
- package/src/routes/route-register.ts +73 -4
- package/src/runner/teardown.ts +8 -8
- package/src/storage/BucketStorageBatch.ts +32 -7
- package/src/storage/BucketStorageFactory.ts +36 -40
- package/src/storage/ParsedSyncConfigSet.ts +25 -0
- package/src/storage/PersistedReplicationStream.ts +55 -0
- package/src/storage/PersistedSyncConfigContent.ts +188 -0
- package/src/storage/PersistedSyncConfigStatus.ts +14 -0
- package/src/storage/SourceEntity.ts +9 -1
- package/src/storage/SourceTable.ts +72 -21
- package/src/storage/SyncRulesBucketStorage.ts +87 -26
- package/src/storage/implementation/BucketDefinitionMapping.ts +436 -0
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +109 -0
- package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
- package/src/storage/storage-index.ts +9 -1
- package/src/sync/BucketChecksumState.ts +5 -5
- package/src/sync/sync.ts +8 -3
- package/src/tracing/PerformanceTracer.ts +24 -1
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/types.ts +1 -1
- package/src/util/lsn.ts +9 -0
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +8 -0
- package/test/src/auth.test.ts +11 -0
- package/test/src/diagnostics.test.ts +64 -33
- package/test/src/module-loader.test.ts +33 -1
- package/test/src/routes/admin.test.ts +180 -18
- package/test/src/routes/error-handler.integration.test.ts +275 -0
- package/test/src/routes/mocks.ts +6 -2
- package/test/src/routes/stream.test.ts +15 -4
- package/test/src/storage/SourceTable.test.ts +89 -0
- package/test/src/sync/BucketChecksumState.test.ts +25 -17
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
- package/dist/storage/PersistedSyncRulesContent.js +0 -100
- package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
- package/src/storage/PersistedSyncRulesContent.ts +0 -174
|
@@ -2,13 +2,15 @@ import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
|
2
2
|
import {
|
|
3
3
|
PrecompiledSyncConfig,
|
|
4
4
|
SerializedCompatibilityContext,
|
|
5
|
+
SerializedSyncPlanV1,
|
|
5
6
|
serializeSyncPlan,
|
|
6
7
|
SqlSyncRules,
|
|
7
8
|
SyncConfigWithErrors
|
|
8
9
|
} from '@powersync/service-sync-rules';
|
|
9
10
|
import { ReplicationError } from '@powersync/service-types';
|
|
10
11
|
import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
|
|
11
|
-
import {
|
|
12
|
+
import { PersistedReplicationStream } from './PersistedReplicationStream.js';
|
|
13
|
+
import { PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
|
|
12
14
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
13
15
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
14
16
|
import { ReportStorage } from './ReportStorage.js';
|
|
@@ -31,14 +33,14 @@ export abstract class BucketStorageFactory
|
|
|
31
33
|
*/
|
|
32
34
|
async configureSyncRules(
|
|
33
35
|
options: UpdateSyncRulesOptions
|
|
34
|
-
): Promise<{ updated: boolean; persisted_sync_rules?:
|
|
35
|
-
const
|
|
36
|
-
const active = await this.
|
|
36
|
+
): Promise<{ updated: boolean; persisted_sync_rules?: PersistedReplicationStream; lock?: ReplicationLock }> {
|
|
37
|
+
const deploying = await this.getDeployingSyncConfig();
|
|
38
|
+
const active = await this.getActiveSyncConfig();
|
|
37
39
|
|
|
38
|
-
if (
|
|
40
|
+
if (deploying?.content.sync_rules_content == options.config.yaml) {
|
|
39
41
|
logger.info('Sync config unchanged');
|
|
40
42
|
return { updated: false };
|
|
41
|
-
} else if (
|
|
43
|
+
} else if (deploying == null && active?.content.sync_rules_content == options.config.yaml) {
|
|
42
44
|
logger.info('Sync config unchanged');
|
|
43
45
|
return { updated: false };
|
|
44
46
|
} else {
|
|
@@ -51,12 +53,15 @@ export abstract class BucketStorageFactory
|
|
|
51
53
|
/**
|
|
52
54
|
* Get a storage instance to query sync data for specific sync config.
|
|
53
55
|
*/
|
|
54
|
-
abstract getInstance(
|
|
56
|
+
abstract getInstance(
|
|
57
|
+
replicationStream: PersistedReplicationStream,
|
|
58
|
+
options?: GetIntanceOptions
|
|
59
|
+
): SyncRulesBucketStorage;
|
|
55
60
|
|
|
56
61
|
/**
|
|
57
62
|
* Deploy new sync config.
|
|
58
63
|
*/
|
|
59
|
-
abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<
|
|
64
|
+
abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedReplicationStream>;
|
|
60
65
|
|
|
61
66
|
/**
|
|
62
67
|
* Indicate that a slot was removed, and we should re-sync by creating
|
|
@@ -68,48 +73,27 @@ export abstract class BucketStorageFactory
|
|
|
68
73
|
*
|
|
69
74
|
* Replication should be restarted after this.
|
|
70
75
|
*/
|
|
71
|
-
abstract restartReplication(
|
|
76
|
+
abstract restartReplication(replicationStreamId: number): Promise<void>;
|
|
72
77
|
|
|
73
78
|
/**
|
|
74
|
-
* Get the sync config used for querying.
|
|
79
|
+
* Get the sync config and storage used for querying.
|
|
75
80
|
*/
|
|
76
|
-
|
|
77
|
-
const content = await this.getActiveSyncRulesContent();
|
|
78
|
-
return content?.parsed(options) ?? null;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Get the sync config used for querying.
|
|
83
|
-
*/
|
|
84
|
-
abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* Get the sync config that will be active next once done with initial replicatino.
|
|
88
|
-
*/
|
|
89
|
-
async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
|
|
90
|
-
const content = await this.getNextSyncRulesContent();
|
|
91
|
-
return content?.parsed(options) ?? null;
|
|
92
|
-
}
|
|
81
|
+
abstract getActiveSyncConfig(): Promise<ResolvedSyncConfig | null>;
|
|
93
82
|
|
|
94
83
|
/**
|
|
95
|
-
* Get the sync config
|
|
84
|
+
* Get the sync config and storage that is still deploying.
|
|
96
85
|
*/
|
|
97
|
-
abstract
|
|
86
|
+
abstract getDeployingSyncConfig(): Promise<ResolvedSyncConfig | null>;
|
|
98
87
|
|
|
99
88
|
/**
|
|
100
|
-
* Get all
|
|
89
|
+
* Get all replication streams currently replicating.
|
|
101
90
|
*/
|
|
102
|
-
abstract
|
|
91
|
+
abstract getReplicatingReplicationStreams(): Promise<PersistedReplicationStream[]>;
|
|
103
92
|
|
|
104
93
|
/**
|
|
105
|
-
* Get all
|
|
94
|
+
* Get all replication streams stopped but not terminated yet.
|
|
106
95
|
*/
|
|
107
|
-
abstract
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Get the active storage instance.
|
|
111
|
-
*/
|
|
112
|
-
abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
96
|
+
abstract getStoppedReplicationStreams(): Promise<PersistedReplicationStream[]>;
|
|
113
97
|
|
|
114
98
|
/**
|
|
115
99
|
* Get storage size of active replication stream.
|
|
@@ -134,6 +118,12 @@ export interface BucketStorageFactoryListener {
|
|
|
134
118
|
replicationEvent: (event: ReplicationEventPayload) => void;
|
|
135
119
|
}
|
|
136
120
|
|
|
121
|
+
export interface ResolvedSyncConfig {
|
|
122
|
+
content: PersistedSyncConfigContent;
|
|
123
|
+
replicationStream: PersistedReplicationStream;
|
|
124
|
+
storage: SyncRulesBucketStorage;
|
|
125
|
+
}
|
|
126
|
+
|
|
137
127
|
export interface StorageMetrics {
|
|
138
128
|
/**
|
|
139
129
|
* Size of operations (bucket_data)
|
|
@@ -170,13 +160,18 @@ export interface UpdateSyncRulesOptions {
|
|
|
170
160
|
};
|
|
171
161
|
lock?: boolean;
|
|
172
162
|
storageVersion?: number;
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Only relevant if the result is used. This does not affect the persisted config.
|
|
166
|
+
*/
|
|
167
|
+
defaultSchema?: string;
|
|
173
168
|
}
|
|
174
169
|
|
|
175
170
|
export interface SerializedSyncPlan {
|
|
176
171
|
/**
|
|
177
172
|
* The serialized plan, from {@link serializeSyncPlan}.
|
|
178
173
|
*/
|
|
179
|
-
plan:
|
|
174
|
+
plan: SerializedSyncPlanV1;
|
|
180
175
|
compatibility: SerializedCompatibilityContext;
|
|
181
176
|
/**
|
|
182
177
|
* Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
|
|
@@ -196,7 +191,7 @@ export function updateSyncRulesFromYaml(
|
|
|
196
191
|
const config = SqlSyncRules.fromYaml(content, {
|
|
197
192
|
// No schema-based validation at this point
|
|
198
193
|
schema: undefined,
|
|
199
|
-
defaultSchema: 'not_applicable', // Not needed for validation
|
|
194
|
+
defaultSchema: options?.defaultSchema ?? 'not_applicable', // Not needed for validation
|
|
200
195
|
throwOnError: options?.validate ?? false
|
|
201
196
|
});
|
|
202
197
|
|
|
@@ -273,4 +268,5 @@ export interface TestStorageConfig {
|
|
|
273
268
|
factory: TestStorageFactory;
|
|
274
269
|
tableIdStrings: boolean;
|
|
275
270
|
storageVersion?: number;
|
|
271
|
+
compressedBucketStorage?: boolean;
|
|
276
272
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { HydratedSyncConfig, HydrationState, SyncConfigWithErrors } from '@powersync/service-sync-rules';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A parsed sync config set is the identity boundary for parsed sync config state.
|
|
5
|
+
*
|
|
6
|
+
* All values on a single instance - the parsed configs, hydration state and hydrated
|
|
7
|
+
* sync config - are derived from the same parse, and may be associated with each other
|
|
8
|
+
* by object identity. Values from two different instances must never be mixed, even if
|
|
9
|
+
* they were parsed from the same persisted content.
|
|
10
|
+
*/
|
|
11
|
+
export interface ParsedSyncConfigSet {
|
|
12
|
+
readonly replicationStreamId: number;
|
|
13
|
+
readonly replicationStreamName: string;
|
|
14
|
+
readonly syncConfigs: SyncConfigWithErrors[];
|
|
15
|
+
/**
|
|
16
|
+
* For testing only.
|
|
17
|
+
*/
|
|
18
|
+
readonly hydrationState: HydrationState;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The hydrated sync config for this parse. This is a stable property: repeated reads
|
|
22
|
+
* return the same instance.
|
|
23
|
+
*/
|
|
24
|
+
readonly hydratedSyncConfig: HydratedSyncConfig;
|
|
25
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { Logger } from 'winston';
|
|
3
|
+
import { SyncRuleState } from './BucketStorage.js';
|
|
4
|
+
import type { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
|
|
5
|
+
import type { ParseSyncConfigOptions, PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
|
|
6
|
+
import { ReplicationLock } from './ReplicationLock.js';
|
|
7
|
+
import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
|
|
8
|
+
|
|
9
|
+
export abstract class PersistedReplicationStream implements PersistedReplicationStreamData {
|
|
10
|
+
readonly replicationStreamId: number;
|
|
11
|
+
readonly replicationJobId: string;
|
|
12
|
+
readonly replicationStreamName: string;
|
|
13
|
+
readonly state: SyncRuleState;
|
|
14
|
+
readonly storageVersion: number;
|
|
15
|
+
readonly logger: Logger;
|
|
16
|
+
|
|
17
|
+
abstract readonly syncConfigContent: readonly PersistedSyncConfigContent[];
|
|
18
|
+
abstract readonly current_lock: ReplicationLock | null;
|
|
19
|
+
|
|
20
|
+
constructor(data: PersistedReplicationStreamData) {
|
|
21
|
+
this.replicationStreamId = data.replicationStreamId;
|
|
22
|
+
this.replicationJobId = data.replicationJobId ?? String(data.replicationStreamId);
|
|
23
|
+
this.replicationStreamName = data.replicationStreamName;
|
|
24
|
+
this.state = data.state;
|
|
25
|
+
this.storageVersion = data.storageVersion;
|
|
26
|
+
this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
getStorageConfig(): StorageVersionConfig {
|
|
30
|
+
const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
|
|
31
|
+
if (storageConfig == null) {
|
|
32
|
+
throw new ServiceError(
|
|
33
|
+
ErrorCode.PSYNC_S1005,
|
|
34
|
+
`Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
return storageConfig;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
abstract lock(): Promise<ReplicationLock>;
|
|
41
|
+
|
|
42
|
+
abstract parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
|
|
43
|
+
}
|
|
44
|
+
export interface PersistedReplicationStreamData {
|
|
45
|
+
readonly replicationStreamId: number;
|
|
46
|
+
readonly replicationStreamName: string;
|
|
47
|
+
readonly state: SyncRuleState;
|
|
48
|
+
readonly storageVersion: number;
|
|
49
|
+
/**
|
|
50
|
+
* Uniquely identifies a job, as a combination of replication stream and processing sync configs.
|
|
51
|
+
*
|
|
52
|
+
* Used in-memory only.
|
|
53
|
+
*/
|
|
54
|
+
readonly replicationJobId?: string;
|
|
55
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import {
|
|
3
|
+
CompatibilityContext,
|
|
4
|
+
CompatibilityOption,
|
|
5
|
+
DEFAULT_HYDRATION_STATE,
|
|
6
|
+
deserializeSyncPlan,
|
|
7
|
+
ErrorLocation,
|
|
8
|
+
HydratedSyncConfig,
|
|
9
|
+
HydrationState,
|
|
10
|
+
nodeSqlite,
|
|
11
|
+
PrecompiledSyncConfig,
|
|
12
|
+
SqlEventDescriptor,
|
|
13
|
+
SqlSyncRules,
|
|
14
|
+
SyncConfigWithErrors,
|
|
15
|
+
versionedHydrationState,
|
|
16
|
+
YamlError
|
|
17
|
+
} from '@powersync/service-sync-rules';
|
|
18
|
+
import * as sqlite from 'node:sqlite';
|
|
19
|
+
import { Logger } from 'winston';
|
|
20
|
+
import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
|
|
21
|
+
import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
|
|
22
|
+
import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
|
|
23
|
+
import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
|
|
24
|
+
|
|
25
|
+
export interface ParsePersistedSyncConfigContentOptions {
|
|
26
|
+
content: string;
|
|
27
|
+
compiledPlan: SerializedSyncPlan | null;
|
|
28
|
+
storageVersion: number;
|
|
29
|
+
parseOptions: ParseSyncConfigOptions;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfigContentOptions): SyncConfigWithErrors {
|
|
33
|
+
const { content, compiledPlan, storageVersion, parseOptions } = options;
|
|
34
|
+
|
|
35
|
+
if (compiledPlan == null) {
|
|
36
|
+
// Fallback: Only parse from YAML if no compiled plan is available.
|
|
37
|
+
return SqlSyncRules.fromYaml(content, parseOptions);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const plan = deserializeSyncPlan(compiledPlan.plan);
|
|
41
|
+
const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility);
|
|
42
|
+
const eventDefinitions: SqlEventDescriptor[] = [];
|
|
43
|
+
for (const [name, queries] of Object.entries(compiledPlan.eventDescriptors)) {
|
|
44
|
+
const descriptor = new SqlEventDescriptor(name, compatibility);
|
|
45
|
+
for (const query of queries) {
|
|
46
|
+
descriptor.addSourceQuery(query, parseOptions);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
eventDefinitions.push(descriptor);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
|
|
53
|
+
defaultSchema: parseOptions.defaultSchema,
|
|
54
|
+
sourceText: content
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// Note: If the original content did not define a storage version, this will still set the storage version.
|
|
58
|
+
// This means asUpdateOptions will not change the storage version, even if the default changes.
|
|
59
|
+
precompiled.storageVersion = storageVersion;
|
|
60
|
+
|
|
61
|
+
const errors: YamlError[] = [];
|
|
62
|
+
if (compiledPlan.errors) {
|
|
63
|
+
for (const error of compiledPlan.errors) {
|
|
64
|
+
const location: ErrorLocation | undefined = error.location && {
|
|
65
|
+
start: error.location.start_offset,
|
|
66
|
+
end: error.location.end_offset
|
|
67
|
+
};
|
|
68
|
+
const asYamlError = new YamlError(new Error(error.message), location);
|
|
69
|
+
asYamlError.type = error.level;
|
|
70
|
+
|
|
71
|
+
errors.push(asYamlError);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return { config: precompiled, errors };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Immutable sync config content for one sync config inside a replication stream.
|
|
80
|
+
*
|
|
81
|
+
* This represents the persisted config content. Fetch per-config status with
|
|
82
|
+
* {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
|
|
83
|
+
* lifecycle concerns such as locking and termination.
|
|
84
|
+
*/
|
|
85
|
+
|
|
86
|
+
export abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData {
|
|
87
|
+
readonly replicationStreamId: number;
|
|
88
|
+
readonly sync_rules_content: string;
|
|
89
|
+
readonly compiled_plan: SerializedSyncPlan | null;
|
|
90
|
+
readonly replicationStreamName: string;
|
|
91
|
+
readonly storageVersion: number;
|
|
92
|
+
readonly logger: Logger;
|
|
93
|
+
readonly syncConfigId: PersistedSyncConfigId | null;
|
|
94
|
+
|
|
95
|
+
constructor(data: PersistedSyncConfigContentData) {
|
|
96
|
+
this.replicationStreamId = data.replicationStreamId;
|
|
97
|
+
this.sync_rules_content = data.sync_rules_content;
|
|
98
|
+
this.compiled_plan = data.compiled_plan;
|
|
99
|
+
this.replicationStreamName = data.replicationStreamName;
|
|
100
|
+
this.storageVersion = data.storageVersion;
|
|
101
|
+
this.syncConfigId = data.syncConfigId ?? null;
|
|
102
|
+
this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Load the storage config.
|
|
107
|
+
*
|
|
108
|
+
* This may throw if the persisted storage version is not supported.
|
|
109
|
+
*/
|
|
110
|
+
getStorageConfig(): StorageVersionConfig {
|
|
111
|
+
const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
|
|
112
|
+
if (storageConfig == null) {
|
|
113
|
+
throw new ServiceError(
|
|
114
|
+
ErrorCode.PSYNC_S1005,
|
|
115
|
+
`Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
return storageConfig;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Parse only this config's content into a single {@link SyncConfigWithErrors}.
|
|
123
|
+
*
|
|
124
|
+
* This does not depend on any other configs in the same replication stream.
|
|
125
|
+
*/
|
|
126
|
+
protected parseSingleConfig(options: ParseSyncConfigOptions): SyncConfigWithErrors {
|
|
127
|
+
return parsePersistedSyncConfigContent({
|
|
128
|
+
content: this.sync_rules_content,
|
|
129
|
+
compiledPlan: this.compiled_plan,
|
|
130
|
+
storageVersion: this.storageVersion,
|
|
131
|
+
parseOptions: options
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet {
|
|
136
|
+
let hydrationState: HydrationState;
|
|
137
|
+
const config = this.parseSingleConfig(options);
|
|
138
|
+
|
|
139
|
+
const storageConfig = this.getStorageConfig();
|
|
140
|
+
if (
|
|
141
|
+
storageConfig.versionedBuckets ||
|
|
142
|
+
config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)
|
|
143
|
+
) {
|
|
144
|
+
hydrationState = versionedHydrationState(this.replicationStreamId);
|
|
145
|
+
} else {
|
|
146
|
+
hydrationState = DEFAULT_HYDRATION_STATE;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
let hydrated: HydratedSyncConfig | undefined;
|
|
150
|
+
return {
|
|
151
|
+
replicationStreamId: this.replicationStreamId,
|
|
152
|
+
replicationStreamName: this.replicationStreamName,
|
|
153
|
+
syncConfigs: [config],
|
|
154
|
+
hydrationState,
|
|
155
|
+
get hydratedSyncConfig(): HydratedSyncConfig {
|
|
156
|
+
hydrated ??= config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
|
|
157
|
+
return hydrated;
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
|
|
163
|
+
// defaultSchema is not relevant for the parsed version here
|
|
164
|
+
const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' });
|
|
165
|
+
return {
|
|
166
|
+
config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed },
|
|
167
|
+
...options
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Fetch the current persisted state for this exact sync config.
|
|
173
|
+
*/
|
|
174
|
+
abstract getSyncConfigStatus(): Promise<PersistedSyncConfigStatus | null>;
|
|
175
|
+
}
|
|
176
|
+
export interface PersistedSyncConfigContentData {
|
|
177
|
+
readonly replicationStreamId: number;
|
|
178
|
+
readonly sync_rules_content: string;
|
|
179
|
+
readonly compiled_plan: SerializedSyncPlan | null;
|
|
180
|
+
readonly replicationStreamName: string;
|
|
181
|
+
readonly storageVersion: number;
|
|
182
|
+
|
|
183
|
+
readonly syncConfigId?: PersistedSyncConfigId | null;
|
|
184
|
+
}
|
|
185
|
+
export type PersistedSyncConfigId = string;
|
|
186
|
+
export interface ParseSyncConfigOptions {
|
|
187
|
+
defaultSchema: string;
|
|
188
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { SyncRuleState } from './BucketStorage.js';
|
|
2
|
+
import { PersistedSyncConfigId } from './PersistedSyncConfigContent.js';
|
|
3
|
+
|
|
4
|
+
export interface PersistedSyncConfigStatus {
|
|
5
|
+
readonly id: PersistedSyncConfigId;
|
|
6
|
+
readonly replicationStreamId: number;
|
|
7
|
+
readonly state: SyncRuleState;
|
|
8
|
+
readonly snapshot_done?: boolean;
|
|
9
|
+
readonly last_checkpoint_lsn: string | null;
|
|
10
|
+
readonly last_fatal_error?: string | null;
|
|
11
|
+
readonly last_fatal_error_ts?: Date | null;
|
|
12
|
+
readonly last_keepalive_ts?: Date | null;
|
|
13
|
+
readonly last_checkpoint_ts?: Date | null;
|
|
14
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { SourceTableRef } from '@powersync/service-sync-rules';
|
|
2
|
+
|
|
1
3
|
export interface ColumnDescriptor {
|
|
2
4
|
name: string;
|
|
3
5
|
/**
|
|
@@ -10,7 +12,7 @@ export interface ColumnDescriptor {
|
|
|
10
12
|
typeId?: number;
|
|
11
13
|
}
|
|
12
14
|
|
|
13
|
-
export interface SourceEntityDescriptor {
|
|
15
|
+
export interface SourceEntityDescriptor extends SourceTableRef {
|
|
14
16
|
/**
|
|
15
17
|
* The internal id of the source entity structure in the database.
|
|
16
18
|
* If undefined, the schema and name are used as the identifier.
|
|
@@ -23,4 +25,10 @@ export interface SourceEntityDescriptor {
|
|
|
23
25
|
* The columns that are used to uniquely identify a record in the source entity.
|
|
24
26
|
*/
|
|
25
27
|
replicaIdColumns: ColumnDescriptor[];
|
|
28
|
+
/**
|
|
29
|
+
* Whether the source always sends complete row data with each operation (e.g. Postgres REPLICA
|
|
30
|
+
* IDENTITY FULL). When true, no current_data copy is needed. Undefined means the source does not
|
|
31
|
+
* report this, in which case we default to keeping a copy.
|
|
32
|
+
*/
|
|
33
|
+
sendsCompleteRows?: boolean;
|
|
26
34
|
}
|
|
@@ -1,7 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
BucketDataSource,
|
|
3
|
+
BucketDefinitionId,
|
|
4
|
+
DEFAULT_TAG,
|
|
5
|
+
ParameterIndexId,
|
|
6
|
+
ParameterIndexLookupCreator,
|
|
7
|
+
SourceTableRef
|
|
8
|
+
} from '@powersync/service-sync-rules';
|
|
2
9
|
import { bson } from '../index.js';
|
|
3
10
|
import * as util from '../util/util-index.js';
|
|
4
|
-
import { ColumnDescriptor
|
|
11
|
+
import { ColumnDescriptor } from './SourceEntity.js';
|
|
5
12
|
|
|
6
13
|
/**
|
|
7
14
|
* Format of the id depends on the bucket storage module. It should be consistent within the module.
|
|
@@ -10,12 +17,14 @@ export type SourceTableId = string | bson.ObjectId;
|
|
|
10
17
|
|
|
11
18
|
export interface SourceTableOptions {
|
|
12
19
|
id: SourceTableId;
|
|
13
|
-
|
|
20
|
+
ref: SourceTableRef;
|
|
14
21
|
objectId: number | string | undefined;
|
|
15
|
-
schema: string;
|
|
16
|
-
name: string;
|
|
17
22
|
replicaIdColumns: ColumnDescriptor[];
|
|
18
23
|
snapshotComplete: boolean;
|
|
24
|
+
bucketDataSources: BucketDataSource[];
|
|
25
|
+
parameterLookupSources: ParameterIndexLookupCreator[];
|
|
26
|
+
bucketDataSourceIds?: Set<BucketDefinitionId>;
|
|
27
|
+
parameterLookupSourceIds?: Set<ParameterIndexId>;
|
|
19
28
|
}
|
|
20
29
|
|
|
21
30
|
export interface TableSnapshotStatus {
|
|
@@ -24,7 +33,13 @@ export interface TableSnapshotStatus {
|
|
|
24
33
|
lastKey: Uint8Array | null;
|
|
25
34
|
}
|
|
26
35
|
|
|
27
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Represents a resolved source table.
|
|
38
|
+
*
|
|
39
|
+
* There could be multiple of these for the same SourceTableRef.
|
|
40
|
+
* For that reason, we do not implement the SourceTableRef interface, to ensure that the two are not used interchangably.
|
|
41
|
+
*/
|
|
42
|
+
export class SourceTable {
|
|
28
43
|
static readonly DEFAULT_TAG = DEFAULT_TAG;
|
|
29
44
|
|
|
30
45
|
/**
|
|
@@ -46,14 +61,29 @@ export class SourceTable implements SourceEntityDescriptor {
|
|
|
46
61
|
public syncParameters = true;
|
|
47
62
|
|
|
48
63
|
/**
|
|
49
|
-
* True if
|
|
64
|
+
* True if this table should fire events for row changes.
|
|
50
65
|
*
|
|
51
|
-
* This value is resolved externally, and cached here.
|
|
66
|
+
* This value is resolved externally, and cached here. When multiple SourceTables exist
|
|
67
|
+
* for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
|
|
68
|
+
* as the event carrier, so that a row change saved once per table fires each event once.
|
|
52
69
|
*
|
|
53
70
|
* Defaults to true for tests.
|
|
54
71
|
*/
|
|
55
72
|
public syncEvent = true;
|
|
56
73
|
|
|
74
|
+
/**
|
|
75
|
+
* True if raw data should be stored in current_data collection.
|
|
76
|
+
*
|
|
77
|
+
* This is needed when the source sends partial row data (e.g. TOAST values).
|
|
78
|
+
* When REPLICA IDENTITY FULL is configured, complete rows are always sent,
|
|
79
|
+
* so we don't need to store raw data.
|
|
80
|
+
*
|
|
81
|
+
* This value is resolved externally based on table configuration.
|
|
82
|
+
*
|
|
83
|
+
* Defaults to true for tests (conservative approach).
|
|
84
|
+
*/
|
|
85
|
+
public storeCurrentData = true;
|
|
86
|
+
|
|
57
87
|
/**
|
|
58
88
|
* Always undefined if snapshotComplete = true.
|
|
59
89
|
*
|
|
@@ -71,31 +101,47 @@ export class SourceTable implements SourceEntityDescriptor {
|
|
|
71
101
|
return this.options.id;
|
|
72
102
|
}
|
|
73
103
|
|
|
74
|
-
get connectionTag() {
|
|
75
|
-
return this.options.connectionTag;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
104
|
get objectId() {
|
|
79
105
|
return this.options.objectId;
|
|
80
106
|
}
|
|
81
107
|
|
|
82
108
|
get schema() {
|
|
83
|
-
return this.options.schema;
|
|
109
|
+
return this.options.ref.schema;
|
|
84
110
|
}
|
|
85
111
|
get name() {
|
|
86
|
-
return this.options.name;
|
|
112
|
+
return this.options.ref.name;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
get ref() {
|
|
116
|
+
return this.options.ref;
|
|
87
117
|
}
|
|
88
118
|
|
|
89
119
|
get replicaIdColumns() {
|
|
90
120
|
return this.options.replicaIdColumns;
|
|
91
121
|
}
|
|
92
122
|
|
|
123
|
+
get bucketDataSources() {
|
|
124
|
+
return this.options.bucketDataSources;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
get parameterLookupSources() {
|
|
128
|
+
return this.options.parameterLookupSources;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
get bucketDataSourceIds() {
|
|
132
|
+
return this.options.bucketDataSourceIds;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
get parameterLookupSourceIds() {
|
|
136
|
+
return this.options.parameterLookupSourceIds;
|
|
137
|
+
}
|
|
138
|
+
|
|
93
139
|
/**
|
|
94
|
-
*
|
|
95
|
-
*
|
|
140
|
+
* Sanitized name of the entity in the format of "{schema}.{entity name}".
|
|
141
|
+
* Suitable for safe use in Postgres queries.
|
|
96
142
|
*/
|
|
97
143
|
get qualifiedName() {
|
|
98
|
-
return
|
|
144
|
+
return util.qualifiedName(this.ref);
|
|
99
145
|
}
|
|
100
146
|
|
|
101
147
|
get syncAny() {
|
|
@@ -108,15 +154,20 @@ export class SourceTable implements SourceEntityDescriptor {
|
|
|
108
154
|
clone() {
|
|
109
155
|
const copy = new SourceTable({
|
|
110
156
|
id: this.id,
|
|
111
|
-
|
|
157
|
+
ref: this.options.ref,
|
|
112
158
|
objectId: this.objectId,
|
|
113
|
-
schema: this.schema,
|
|
114
|
-
name: this.name,
|
|
115
159
|
replicaIdColumns: this.replicaIdColumns,
|
|
116
|
-
snapshotComplete: this.snapshotComplete
|
|
160
|
+
snapshotComplete: this.snapshotComplete,
|
|
161
|
+
bucketDataSources: this.bucketDataSources,
|
|
162
|
+
parameterLookupSources: this.parameterLookupSources,
|
|
163
|
+
bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
|
|
164
|
+
parameterLookupSourceIds:
|
|
165
|
+
this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
|
|
117
166
|
});
|
|
118
167
|
copy.syncData = this.syncData;
|
|
119
168
|
copy.syncParameters = this.syncParameters;
|
|
169
|
+
copy.syncEvent = this.syncEvent;
|
|
170
|
+
copy.storeCurrentData = this.storeCurrentData;
|
|
120
171
|
copy.snapshotStatus = this.snapshotStatus;
|
|
121
172
|
return copy;
|
|
122
173
|
}
|