@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +152 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +36 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +30 -22
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/metrics/metrics-interfaces.d.ts +5 -1
  9. package/dist/metrics/metrics-interfaces.js.map +1 -1
  10. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
  11. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
  12. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
  13. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
  14. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
  15. package/dist/metrics/open-telemetry/util.js +7 -3
  16. package/dist/metrics/open-telemetry/util.js.map +1 -1
  17. package/dist/modules/AbstractModule.d.ts +6 -2
  18. package/dist/modules/AbstractModule.js.map +1 -1
  19. package/dist/replication/AbstractReplicator.d.ts +31 -7
  20. package/dist/replication/AbstractReplicator.js +168 -74
  21. package/dist/replication/AbstractReplicator.js.map +1 -1
  22. package/dist/replication/ReplicationModule.d.ts +3 -3
  23. package/dist/replication/ReplicationModule.js.map +1 -1
  24. package/dist/routes/RouterEngine.d.ts +1 -0
  25. package/dist/routes/RouterEngine.js +6 -1
  26. package/dist/routes/RouterEngine.js.map +1 -1
  27. package/dist/routes/configure-fastify.d.ts +104 -56
  28. package/dist/routes/configure-fastify.js +15 -2
  29. package/dist/routes/configure-fastify.js.map +1 -1
  30. package/dist/routes/endpoints/admin.d.ts +42 -24
  31. package/dist/routes/endpoints/admin.js +49 -36
  32. package/dist/routes/endpoints/admin.js.map +1 -1
  33. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  34. package/dist/routes/endpoints/checkpointing.js +65 -7
  35. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  36. package/dist/routes/endpoints/socket-route.js +3 -2
  37. package/dist/routes/endpoints/socket-route.js.map +1 -1
  38. package/dist/routes/endpoints/sync-rules.js +25 -14
  39. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  40. package/dist/routes/endpoints/sync-stream.js +1 -1
  41. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  42. package/dist/runner/teardown.js +8 -8
  43. package/dist/runner/teardown.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +64 -17
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/BucketStorageFactory.d.ts +50 -31
  47. package/dist/storage/BucketStorageFactory.js +10 -20
  48. package/dist/storage/BucketStorageFactory.js.map +1 -1
  49. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  50. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  51. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  52. package/dist/storage/ChecksumCache.d.ts +11 -6
  53. package/dist/storage/ChecksumCache.js +5 -4
  54. package/dist/storage/ChecksumCache.js.map +1 -1
  55. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  56. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  57. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  58. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  59. package/dist/storage/PersistedReplicationStream.js +26 -0
  60. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  61. package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
  62. package/dist/storage/PersistedSyncConfigContent.js +138 -0
  63. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  64. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  65. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  66. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  67. package/dist/storage/ReplicationEventPayload.d.ts +6 -1
  68. package/dist/storage/SourceEntity.d.ts +13 -2
  69. package/dist/storage/SourceTable.d.ts +53 -4
  70. package/dist/storage/SourceTable.js +62 -7
  71. package/dist/storage/SourceTable.js.map +1 -1
  72. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  73. package/dist/storage/SourceTableReconciler.js +95 -0
  74. package/dist/storage/SourceTableReconciler.js.map +1 -0
  75. package/dist/storage/StorageVersionConfig.d.ts +4 -0
  76. package/dist/storage/StorageVersionConfig.js +10 -1
  77. package/dist/storage/StorageVersionConfig.js.map +1 -1
  78. package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
  79. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  80. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  81. package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
  82. package/dist/storage/WriteCheckpointAPI.js +33 -0
  83. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  84. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
  85. package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
  86. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  87. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  88. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  89. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  90. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  91. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
  92. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  93. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  94. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  95. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  96. package/dist/storage/storage-index.d.ts +10 -1
  97. package/dist/storage/storage-index.js +10 -1
  98. package/dist/storage/storage-index.js.map +1 -1
  99. package/dist/storage/storage-metrics.js +120 -4
  100. package/dist/storage/storage-metrics.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +12 -2
  102. package/dist/sync/BucketChecksumState.js +69 -21
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.d.ts +7 -0
  105. package/dist/sync/RequestTracker.js +12 -0
  106. package/dist/sync/RequestTracker.js.map +1 -1
  107. package/dist/sync/sync.js +184 -43
  108. package/dist/sync/sync.js.map +1 -1
  109. package/dist/sync/util.d.ts +5 -0
  110. package/dist/sync/util.js +19 -1
  111. package/dist/sync/util.js.map +1 -1
  112. package/dist/system/ServiceContext.d.ts +2 -0
  113. package/dist/system/ServiceContext.js +2 -0
  114. package/dist/system/ServiceContext.js.map +1 -1
  115. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  116. package/dist/tracing/PerformanceTracer.js +11 -0
  117. package/dist/tracing/PerformanceTracer.js.map +1 -1
  118. package/dist/util/checkpointing.d.ts +4 -8
  119. package/dist/util/checkpointing.js +1 -16
  120. package/dist/util/checkpointing.js.map +1 -1
  121. package/dist/util/config/compound-config-collector.js +23 -2
  122. package/dist/util/config/compound-config-collector.js.map +1 -1
  123. package/dist/util/config/defaults.d.ts +2 -0
  124. package/dist/util/config/defaults.js +2 -0
  125. package/dist/util/config/defaults.js.map +1 -1
  126. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  127. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  128. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  130. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  131. package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
  132. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  133. package/dist/util/config/types.d.ts +4 -1
  134. package/dist/util/lsn.d.ts +4 -0
  135. package/dist/util/lsn.js +10 -0
  136. package/dist/util/lsn.js.map +1 -1
  137. package/dist/util/protocol-types.d.ts +4 -4
  138. package/dist/util/protocol-types.js +3 -2
  139. package/dist/util/protocol-types.js.map +1 -1
  140. package/dist/util/util-index.d.ts +1 -0
  141. package/dist/util/util-index.js +1 -0
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +3 -2
  144. package/dist/util/utils.js +17 -2
  145. package/dist/util/utils.js.map +1 -1
  146. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  147. package/dist/util/write-checkpoint-batcher.js +100 -0
  148. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  149. package/package.json +7 -7
  150. package/src/api/RouteAPI.ts +43 -14
  151. package/src/api/diagnostics.ts +45 -26
  152. package/src/entry/commands/compact-action.ts +37 -23
  153. package/src/metrics/metrics-interfaces.ts +6 -1
  154. package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
  155. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
  156. package/src/metrics/open-telemetry/util.ts +7 -3
  157. package/src/modules/AbstractModule.ts +6 -2
  158. package/src/replication/AbstractReplicator.ts +212 -78
  159. package/src/replication/ReplicationModule.ts +2 -3
  160. package/src/routes/RouterEngine.ts +7 -1
  161. package/src/routes/configure-fastify.ts +25 -2
  162. package/src/routes/endpoints/admin.ts +75 -51
  163. package/src/routes/endpoints/checkpointing.ts +79 -8
  164. package/src/routes/endpoints/socket-route.ts +3 -2
  165. package/src/routes/endpoints/sync-rules.ts +26 -14
  166. package/src/routes/endpoints/sync-stream.ts +1 -1
  167. package/src/runner/teardown.ts +8 -8
  168. package/src/storage/BucketStorageBatch.ts +65 -18
  169. package/src/storage/BucketStorageFactory.ts +73 -46
  170. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  171. package/src/storage/ChecksumCache.ts +27 -14
  172. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  173. package/src/storage/PersistedReplicationStream.ts +55 -0
  174. package/src/storage/PersistedSyncConfigContent.ts +207 -0
  175. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  176. package/src/storage/ReplicationEventPayload.ts +6 -1
  177. package/src/storage/SourceEntity.ts +12 -2
  178. package/src/storage/SourceTable.ts +96 -8
  179. package/src/storage/SourceTableReconciler.ts +188 -0
  180. package/src/storage/StorageVersionConfig.ts +11 -1
  181. package/src/storage/SyncRulesBucketStorage.ts +189 -27
  182. package/src/storage/WriteCheckpointAPI.ts +140 -6
  183. package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
  184. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  185. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
  186. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  187. package/src/storage/storage-index.ts +11 -1
  188. package/src/storage/storage-metrics.ts +154 -8
  189. package/src/sync/BucketChecksumState.ts +105 -21
  190. package/src/sync/RequestTracker.ts +20 -0
  191. package/src/sync/sync.ts +214 -50
  192. package/src/sync/util.ts +33 -8
  193. package/src/system/ServiceContext.ts +7 -0
  194. package/src/tracing/PerformanceTracer.ts +41 -1
  195. package/src/util/checkpointing.ts +4 -22
  196. package/src/util/config/compound-config-collector.ts +31 -1
  197. package/src/util/config/defaults.ts +2 -0
  198. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  199. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  200. package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
  201. package/src/util/config/types.ts +4 -1
  202. package/src/util/lsn.ts +9 -0
  203. package/src/util/protocol-types.ts +3 -2
  204. package/src/util/util-index.ts +1 -0
  205. package/src/util/utils.ts +16 -2
  206. package/src/util/write-checkpoint-batcher.ts +142 -0
  207. package/test/src/AbstractReplicator.test.ts +177 -0
  208. package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
  209. package/test/src/PersistedSyncConfigContent.test.ts +152 -0
  210. package/test/src/checksum_cache.test.ts +22 -0
  211. package/test/src/config.test.ts +142 -0
  212. package/test/src/diagnostics.test.ts +87 -34
  213. package/test/src/module-loader.test.ts +33 -1
  214. package/test/src/routes/admin.test.ts +188 -18
  215. package/test/src/routes/checkpointing.test.ts +54 -0
  216. package/test/src/routes/mocks.ts +6 -2
  217. package/test/src/routes/stream.test.ts +123 -2
  218. package/test/src/source-table-reconciler.test.ts +244 -0
  219. package/test/src/storage/SourceTable.test.ts +92 -0
  220. package/test/src/storage-metrics.test.ts +148 -0
  221. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  222. package/test/src/sync/util.test.ts +28 -1
  223. package/test/src/util/checkpointing.test.ts +249 -0
  224. package/test/src/util/utils.test.ts +23 -0
  225. package/tsconfig.json +1 -4
  226. package/tsconfig.tsbuildinfo +1 -1
  227. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  228. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  229. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  230. package/src/storage/PersistedSyncRulesContent.ts +0 -174
@@ -0,0 +1,207 @@
1
+ import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
+ import {
3
+ CompatibilityContext,
4
+ CompatibilityOption,
5
+ compileEventDefinitions,
6
+ DEFAULT_HYDRATION_STATE,
7
+ deserializeSyncPlan,
8
+ ErrorLocation,
9
+ HydratedSyncConfig,
10
+ HydrationState,
11
+ nodeSqlite,
12
+ PrecompiledSyncConfig,
13
+ SqlSyncRules,
14
+ SyncConfigWithErrors,
15
+ versionedHydrationState,
16
+ YamlError
17
+ } from '@powersync/service-sync-rules';
18
+ import * as sqlite from 'node:sqlite';
19
+ import { Logger } from 'winston';
20
+ import { SyncRuleState } from './BucketStorage.js';
21
+ import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
22
+ import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
23
+ import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
24
+ import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
25
+
26
+ export interface ParsePersistedSyncConfigContentOptions {
27
+ content: string;
28
+ compiledPlan: SerializedSyncPlan | null;
29
+ storageVersion: number;
30
+ parseOptions: ParseSyncConfigOptions;
31
+ }
32
+
33
+ export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfigContentOptions): SyncConfigWithErrors {
34
+ const { content, compiledPlan, storageVersion, parseOptions } = options;
35
+
36
+ if (compiledPlan == null) {
37
+ // Fallback: Only parse from YAML if no compiled plan is available.
38
+ return SqlSyncRules.fromYaml(content, parseOptions);
39
+ }
40
+
41
+ const plan = deserializeSyncPlan(compiledPlan.plan);
42
+ const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility);
43
+ const errors: YamlError[] = [];
44
+ // Compiled events are additive to plan versions 1 and 2. New readers prefer them when present; when an older plan
45
+ // does not contain them, normalize the dual-written raw SQL at this loading boundary. This keeps legacy event
46
+ // evaluators out of PrecompiledSyncConfig while older binaries can continue reading the same persisted config.
47
+ if (compiledPlan.plan.events == null) {
48
+ const normalized = compileEventDefinitions(compiledPlan.eventDescriptors, {
49
+ ...parseOptions,
50
+ // The legacy evaluator ignored event payload filters. Preserve that behavior for plans deployed before compiled
51
+ // events existed; a redeploy compiles and validates those filters before the replacement config is activated.
52
+ compileEventPayloadFilters: false
53
+ });
54
+ const fatalErrors = normalized.errors.filter((error) => error.type == 'fatal');
55
+ if (fatalErrors.length != 0) {
56
+ throw new Error(
57
+ `Failed to compile persisted replication events: ${fatalErrors.map((error) => error.message).join(', ')}`
58
+ );
59
+ }
60
+ plan.events = normalized.events;
61
+ errors.push(...normalized.errors.map((error) => new YamlError(error)));
62
+ }
63
+
64
+ const precompiled = new PrecompiledSyncConfig(plan, compatibility, {
65
+ defaultSchema: parseOptions.defaultSchema,
66
+ sourceText: content
67
+ });
68
+
69
+ // Note: If the original content did not define a storage version, this will still set the storage version.
70
+ // This means asUpdateOptions will not change the storage version, even if the default changes.
71
+ precompiled.storageVersion = storageVersion;
72
+
73
+ if (compiledPlan.errors) {
74
+ for (const error of compiledPlan.errors) {
75
+ const location: ErrorLocation | undefined = error.location && {
76
+ start: error.location.start_offset,
77
+ end: error.location.end_offset
78
+ };
79
+ const asYamlError = new YamlError(new Error(error.message), location);
80
+ asYamlError.type = error.level;
81
+
82
+ errors.push(asYamlError);
83
+ }
84
+ }
85
+
86
+ return { config: precompiled, errors };
87
+ }
88
+
89
+ /**
90
+ * Immutable sync config content for one sync config inside a replication stream.
91
+ *
92
+ * This represents the persisted config content. Fetch per-config status with
93
+ * {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
94
+ * lifecycle concerns such as locking and termination.
95
+ */
96
+
97
+ export abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData {
98
+ readonly replicationStreamId: number;
99
+ readonly sync_rules_content: string;
100
+ readonly compiled_plan: SerializedSyncPlan | null;
101
+ readonly replicationStreamName: string;
102
+ readonly storageVersion: number;
103
+ readonly logger: Logger;
104
+ readonly syncConfigId: PersistedSyncConfigId | null;
105
+ readonly syncConfigState: SyncRuleState;
106
+ readonly version_label: string | undefined;
107
+
108
+ constructor(data: PersistedSyncConfigContentData) {
109
+ this.replicationStreamId = data.replicationStreamId;
110
+ this.sync_rules_content = data.sync_rules_content;
111
+ this.compiled_plan = data.compiled_plan;
112
+ this.replicationStreamName = data.replicationStreamName;
113
+ this.storageVersion = data.storageVersion;
114
+ this.syncConfigId = data.syncConfigId ?? null;
115
+ this.syncConfigState = data.syncConfigState;
116
+ this.version_label = data.version_label;
117
+ const versionPrefix = this.version_label == null ? '' : `[${this.version_label}]`;
118
+ this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}]${versionPrefix} ` });
119
+ }
120
+
121
+ /**
122
+ * Load the storage config.
123
+ *
124
+ * This may throw if the persisted storage version is not supported.
125
+ */
126
+ getStorageConfig(): StorageVersionConfig {
127
+ const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
128
+ if (storageConfig == null) {
129
+ throw new ServiceError(
130
+ ErrorCode.PSYNC_S1005,
131
+ `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`
132
+ );
133
+ }
134
+ return storageConfig;
135
+ }
136
+
137
+ /**
138
+ * Parse only this config's content into a single {@link SyncConfigWithErrors}.
139
+ *
140
+ * This does not depend on any other configs in the same replication stream.
141
+ */
142
+ protected parseSingleConfig(options: ParseSyncConfigOptions): SyncConfigWithErrors {
143
+ return parsePersistedSyncConfigContent({
144
+ content: this.sync_rules_content,
145
+ compiledPlan: this.compiled_plan,
146
+ storageVersion: this.storageVersion,
147
+ parseOptions: options
148
+ });
149
+ }
150
+
151
+ parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet {
152
+ let hydrationState: HydrationState;
153
+ const config = this.parseSingleConfig(options);
154
+
155
+ const storageConfig = this.getStorageConfig();
156
+ if (
157
+ storageConfig.versionedBuckets ||
158
+ config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)
159
+ ) {
160
+ hydrationState = versionedHydrationState(this.replicationStreamId);
161
+ } else {
162
+ hydrationState = DEFAULT_HYDRATION_STATE;
163
+ }
164
+
165
+ let hydrated: HydratedSyncConfig | undefined;
166
+ return {
167
+ replicationStreamId: this.replicationStreamId,
168
+ replicationStreamName: this.replicationStreamName,
169
+ syncConfigs: [config],
170
+ hydrationState,
171
+ get hydratedSyncConfig(): HydratedSyncConfig {
172
+ hydrated ??= config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
173
+ return hydrated;
174
+ }
175
+ };
176
+ }
177
+
178
+ asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
179
+ // defaultSchema is not relevant for the parsed version here
180
+ const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' });
181
+ return {
182
+ config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed },
183
+ version_label: this.version_label,
184
+ ...options
185
+ };
186
+ }
187
+
188
+ /**
189
+ * Fetch the current persisted state for this exact sync config.
190
+ */
191
+ abstract getSyncConfigStatus(): Promise<PersistedSyncConfigStatus | null>;
192
+ }
193
+ export interface PersistedSyncConfigContentData {
194
+ readonly replicationStreamId: number;
195
+ readonly sync_rules_content: string;
196
+ readonly compiled_plan: SerializedSyncPlan | null;
197
+ readonly replicationStreamName: string;
198
+ readonly storageVersion: number;
199
+
200
+ readonly syncConfigId?: PersistedSyncConfigId | null;
201
+ readonly syncConfigState: SyncRuleState;
202
+ readonly version_label?: string;
203
+ }
204
+ export type PersistedSyncConfigId = string;
205
+ export interface ParseSyncConfigOptions {
206
+ defaultSchema: string;
207
+ }
@@ -0,0 +1,14 @@
1
+ import { SyncRuleState } from './BucketStorage.js';
2
+ import { PersistedSyncConfigId } from './PersistedSyncConfigContent.js';
3
+
4
+ export interface PersistedSyncConfigStatus {
5
+ readonly id: PersistedSyncConfigId;
6
+ readonly replicationStreamId: number;
7
+ readonly state: SyncRuleState;
8
+ readonly snapshot_done?: boolean;
9
+ readonly last_checkpoint_lsn: string | null;
10
+ readonly last_fatal_error?: string | null;
11
+ readonly last_fatal_error_ts?: Date | null;
12
+ readonly last_keepalive_ts?: Date | null;
13
+ readonly last_checkpoint_ts?: Date | null;
14
+ }
@@ -11,6 +11,11 @@ export type EventData = {
11
11
  export type ReplicationEventPayload = {
12
12
  batch: BucketStorageBatch;
13
13
  data: EventData;
14
- event: sync_rules.SqlEventDescriptor;
14
+ event: sync_rules.HydratedEventDescriptor;
15
+ /**
16
+ * Storage-assigned id for this event definition. Present when storage uses persisted event mappings, allowing event
17
+ * handlers to route custom checkpoints without deriving an id from compiled event content.
18
+ */
19
+ event_id?: sync_rules.EventDefinitionId;
15
20
  table: SourceTable;
16
21
  };
@@ -1,5 +1,11 @@
1
1
  import { SourceTableRef } from '@powersync/service-sync-rules';
2
2
 
3
+ /**
4
+ * Source-specific JSON metadata. Storage does not interpret it. Source-table APIs use null to
5
+ * represent the absence of metadata.
6
+ */
7
+ export type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
8
+
3
9
  export interface ColumnDescriptor {
4
10
  name: string;
5
11
  /**
@@ -19,10 +25,14 @@ export interface SourceEntityDescriptor extends SourceTableRef {
19
25
  * If specified, this is specifically used to detect renames.
20
26
  */
21
27
  objectId: number | string | undefined;
22
- schema: string;
23
- name: string;
24
28
  /**
25
29
  * The columns that are used to uniquely identify a record in the source entity.
26
30
  */
27
31
  replicaIdColumns: ColumnDescriptor[];
32
+ /**
33
+ * Whether the source always sends complete row data with each operation (e.g. Postgres REPLICA
34
+ * IDENTITY FULL). When true, no current_data copy is needed. Undefined means the source does not
35
+ * report this, in which case we default to keeping a copy.
36
+ */
37
+ sendsCompleteRows?: boolean;
28
38
  }
@@ -1,18 +1,31 @@
1
1
  import {
2
2
  BucketDataSource,
3
+ BucketDefinitionId,
3
4
  DEFAULT_TAG,
5
+ EventDefinitionId,
6
+ ParameterIndexId,
4
7
  ParameterIndexLookupCreator,
5
8
  SourceTableRef
6
9
  } from '@powersync/service-sync-rules';
7
10
  import { bson } from '../index.js';
8
11
  import * as util from '../util/util-index.js';
9
- import { ColumnDescriptor } from './SourceEntity.js';
12
+ import { ColumnDescriptor, JsonValue } from './SourceEntity.js';
10
13
 
11
14
  /**
12
15
  * Format of the id depends on the bucket storage module. It should be consistent within the module.
13
16
  */
14
17
  export type SourceTableId = string | bson.ObjectId;
15
18
 
19
+ /**
20
+ * Compare source-table ids without coercing between storage-specific types.
21
+ */
22
+ export function sourceTableIdEquals(left: SourceTableId, right: SourceTableId): boolean {
23
+ if (typeof left === 'string' || typeof right === 'string') {
24
+ return typeof left === 'string' && typeof right === 'string' && left === right;
25
+ }
26
+ return left.equals(right);
27
+ }
28
+
16
29
  export interface SourceTableOptions {
17
30
  id: SourceTableId;
18
31
  ref: SourceTableRef;
@@ -21,6 +34,19 @@ export interface SourceTableOptions {
21
34
  snapshotComplete: boolean;
22
35
  bucketDataSources: BucketDataSource[];
23
36
  parameterLookupSources: ParameterIndexLookupCreator[];
37
+ bucketDataSourceIds?: Set<BucketDefinitionId>;
38
+ parameterLookupSourceIds?: Set<ParameterIndexId>;
39
+ /**
40
+ * Compiled event definitions assigned to this persisted source-table record.
41
+ *
42
+ * Undefined is the legacy/non-incremental representation where event selection is
43
+ * based on the table ref. V3 incremental storage always supplies this set.
44
+ */
45
+ eventDefinitionIds?: Set<EventDefinitionId>;
46
+ /**
47
+ * Source-specific metadata. Null when no metadata has been recorded.
48
+ */
49
+ sourceMetadata?: JsonValue;
24
50
  }
25
51
 
26
52
  export interface TableSnapshotStatus {
@@ -57,14 +83,29 @@ export class SourceTable {
57
83
  public syncParameters = true;
58
84
 
59
85
  /**
60
- * True if the table is used in sync config for events.
86
+ * True if this table should evaluate event definitions for row changes.
61
87
  *
62
- * This value is resolved externally, and cached here.
88
+ * This value is resolved externally, and cached here. V3 storage assigns disjoint
89
+ * event-definition ids to SourceTables for the same physical table. Multiple records
90
+ * may evaluate different events, but each event id is evaluated through at most one record.
63
91
  *
64
92
  * Defaults to true for tests.
65
93
  */
66
94
  public syncEvent = true;
67
95
 
96
+ /**
97
+ * True if raw data should be stored in current_data collection.
98
+ *
99
+ * This is needed when the source sends partial row data (e.g. TOAST values).
100
+ * When REPLICA IDENTITY FULL is configured, complete rows are always sent,
101
+ * so we don't need to store raw data.
102
+ *
103
+ * This value is resolved externally based on table configuration.
104
+ *
105
+ * Defaults to true for tests (conservative approach).
106
+ */
107
+ public storeCurrentData = true;
108
+
68
109
  /**
69
110
  * Always undefined if snapshotComplete = true.
70
111
  *
@@ -109,6 +150,22 @@ export class SourceTable {
109
150
  return this.options.parameterLookupSources;
110
151
  }
111
152
 
153
+ get bucketDataSourceIds() {
154
+ return this.options.bucketDataSourceIds;
155
+ }
156
+
157
+ get parameterLookupSourceIds() {
158
+ return this.options.parameterLookupSourceIds;
159
+ }
160
+
161
+ get eventDefinitionIds() {
162
+ return this.options.eventDefinitionIds;
163
+ }
164
+
165
+ get sourceMetadata() {
166
+ return this.options.sourceMetadata ?? null;
167
+ }
168
+
112
169
  /**
113
170
  * Sanitized name of the entity in the format of "{schema}.{entity name}".
114
171
  * Suitable for safe use in Postgres queries.
@@ -125,19 +182,42 @@ export class SourceTable {
125
182
  * In-memory clone of the table status.
126
183
  */
127
184
  clone() {
185
+ return this.copyWithSourceMetadata(this.sourceMetadata);
186
+ }
187
+
188
+ /**
189
+ * Copy this table with different source metadata, preserving its resolved state.
190
+ */
191
+ withSourceMetadata(sourceMetadata: JsonValue) {
192
+ return this.copyWithSourceMetadata(sourceMetadata);
193
+ }
194
+
195
+ private copyWithSourceMetadata(sourceMetadata: JsonValue) {
128
196
  const copy = new SourceTable({
129
197
  id: this.id,
130
- ref: this.options.ref,
198
+ ref: { ...this.options.ref },
131
199
  objectId: this.objectId,
132
- replicaIdColumns: this.replicaIdColumns,
200
+ replicaIdColumns: this.replicaIdColumns.map((column) => ({ ...column })),
133
201
  snapshotComplete: this.snapshotComplete,
134
- bucketDataSources: this.bucketDataSources,
135
- parameterLookupSources: this.parameterLookupSources
202
+ bucketDataSources: [...this.bucketDataSources],
203
+ parameterLookupSources: [...this.parameterLookupSources],
204
+ bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
205
+ parameterLookupSourceIds:
206
+ this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds),
207
+ eventDefinitionIds: this.eventDefinitionIds == null ? undefined : new Set(this.eventDefinitionIds),
208
+ sourceMetadata: structuredClone(sourceMetadata)
136
209
  });
137
210
  copy.syncData = this.syncData;
138
211
  copy.syncParameters = this.syncParameters;
139
212
  copy.syncEvent = this.syncEvent;
140
- copy.snapshotStatus = this.snapshotStatus;
213
+ copy.storeCurrentData = this.storeCurrentData;
214
+ copy.snapshotStatus =
215
+ this.snapshotStatus == null
216
+ ? undefined
217
+ : {
218
+ ...this.snapshotStatus,
219
+ lastKey: this.snapshotStatus.lastKey?.slice() ?? null
220
+ };
141
221
  return copy;
142
222
  }
143
223
 
@@ -152,3 +232,11 @@ export class SourceTable {
152
232
  }
153
233
  }
154
234
  }
235
+
236
+ /**
237
+ * A cloned SourceTable exposed to reconciliation with public fields typed as read-only.
238
+ * `options` is omitted so callers cannot mutate the underlying option bag without an explicit cast.
239
+ */
240
+ export type SourceTableCandidate = Omit<Readonly<SourceTable>, 'options' | 'withSourceMetadata'> & {
241
+ withSourceMetadata(sourceMetadata: JsonValue): SourceTableCandidate;
242
+ };
@@ -0,0 +1,188 @@
1
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
2
+ import { isDeepStrictEqual } from 'node:util';
3
+ import { JsonValue, SourceEntityDescriptor } from './SourceEntity.js';
4
+ import { SourceTable, SourceTableCandidate, SourceTableId, sourceTableIdEquals } from './SourceTable.js';
5
+
6
+ /**
7
+ * A source connector's classification of overlapping persisted tables.
8
+ */
9
+ export interface SourceTableCandidateResolution {
10
+ /**
11
+ * Records storage can reuse. Copies may include updated source metadata.
12
+ */
13
+ compatibleTables: ReadonlyArray<SourceTableCandidate>;
14
+
15
+ /**
16
+ * Records that cannot be reused. Every candidate must appear in exactly one result list.
17
+ */
18
+ incompatibleTables: ReadonlyArray<SourceTableCandidate>;
19
+
20
+ /**
21
+ * Values for records storage creates during this resolution.
22
+ */
23
+ newTableValues: SourceTableCreateValues;
24
+ }
25
+
26
+ export interface SourceTableCreateValues {
27
+ /**
28
+ * Source metadata for new records. Null means no metadata.
29
+ */
30
+ sourceMetadata: JsonValue;
31
+ }
32
+
33
+ /**
34
+ * Input to a source-owned reconciliation callback. The callback may run inside a storage
35
+ * transaction, so it must not mutate storage or perform slow external work.
36
+ */
37
+ export interface SourceTableCandidateReconcilerInput {
38
+ /**
39
+ * Source entity being resolved.
40
+ */
41
+ source: SourceEntityDescriptor;
42
+
43
+ /**
44
+ * Persisted tables overlapping by name or object id.
45
+ */
46
+ candidates: ReadonlyArray<SourceTableCandidate>;
47
+ }
48
+
49
+ export type SourceTableCandidateReconciler = (
50
+ input: SourceTableCandidateReconcilerInput
51
+ ) => SourceTableCandidateResolution | Promise<SourceTableCandidateResolution>;
52
+
53
+ /**
54
+ * Compare replica-id columns in order.
55
+ */
56
+ export function sameReplicaIdColumns(
57
+ left: SourceTableCandidate['replicaIdColumns'],
58
+ right: SourceEntityDescriptor
59
+ ): boolean {
60
+ const target = right.replicaIdColumns;
61
+ return (
62
+ left.length == target.length &&
63
+ left.every(
64
+ (column, index) =>
65
+ column.name == target[index].name && column.type == target[index].type && column.typeId == target[index].typeId
66
+ )
67
+ );
68
+ }
69
+
70
+ /**
71
+ * Compare the shared source-table identity fields.
72
+ */
73
+ export function sourceIdentityCompatible(source: SourceEntityDescriptor, candidate: SourceTableCandidate): boolean {
74
+ return (
75
+ candidate.schema == source.schema &&
76
+ candidate.name == source.name &&
77
+ (source.objectId == null || candidate.objectId == source.objectId) &&
78
+ sameReplicaIdColumns(candidate.replicaIdColumns, source)
79
+ );
80
+ }
81
+
82
+ /**
83
+ * Default identity-based reconciliation for connectors without source-specific metadata.
84
+ */
85
+ export const defaultSourceTableReconciler: SourceTableCandidateReconciler = ({ source, candidates }) => {
86
+ const compatibleTables: SourceTableCandidate[] = [];
87
+ const incompatibleTables: SourceTableCandidate[] = [];
88
+ for (const candidate of candidates) {
89
+ if (sourceIdentityCompatible(source, candidate)) {
90
+ compatibleTables.push(candidate);
91
+ } else {
92
+ incompatibleTables.push(candidate);
93
+ }
94
+ }
95
+ return { compatibleTables, incompatibleTables, newTableValues: { sourceMetadata: null } };
96
+ };
97
+
98
+ /**
99
+ * Check that every candidate was classified exactly once.
100
+ */
101
+ export function validateSourceTableCandidateResolution(
102
+ candidates: ReadonlyArray<SourceTableCandidate>,
103
+ resolution: SourceTableCandidateResolution
104
+ ): void {
105
+ const classifiedTables = [...resolution.compatibleTables, ...resolution.incompatibleTables];
106
+
107
+ for (const candidate of candidates) {
108
+ const classifications = classifiedTables.filter((table) => sourceTableIdEquals(table.id, candidate.id));
109
+ if (classifications.length !== 1) {
110
+ throw new ServiceAssertionError(
111
+ `Source table candidate ${candidate.id.toString()} must be classified exactly once, got ${classifications.length}`
112
+ );
113
+ }
114
+ }
115
+
116
+ for (const table of classifiedTables) {
117
+ if (!candidates.some((candidate) => sourceTableIdEquals(candidate.id, table.id))) {
118
+ throw new ServiceAssertionError(`Source table reconciliation returned unknown candidate ${table.id.toString()}`);
119
+ }
120
+ }
121
+ }
122
+
123
+ /**
124
+ * A source-metadata update to persist.
125
+ */
126
+ export interface SourceTableMetadataUpdate {
127
+ id: SourceTableId;
128
+ sourceMetadata: JsonValue;
129
+ }
130
+
131
+ /**
132
+ * Rebuild a resolution from storage-owned tables, applying only reconciler-owned metadata to
133
+ * compatible tables. All other mutable table state comes from storage.
134
+ *
135
+ * Reconciler candidates are typed as read-only, but TypeScript types provide no runtime protection:
136
+ * callback code can cast a cloned candidate and mutate it. Rematerializing by id ensures those
137
+ * mutations are not trusted even when the compile-time boundary is bypassed.
138
+ */
139
+ export function materializeSourceTableResolution(
140
+ tables: ReadonlyArray<SourceTable>,
141
+ resolution: SourceTableCandidateResolution
142
+ ): MaterializedSourceTableResolution {
143
+ const findTable = (candidate: SourceTableCandidate): SourceTable => {
144
+ const table = tables.find((table) => sourceTableIdEquals(table.id, candidate.id));
145
+ if (table == null) {
146
+ throw new ServiceAssertionError(`Source table candidate ${candidate.id.toString()} was not persisted`);
147
+ }
148
+ return table;
149
+ };
150
+ return {
151
+ compatibleTables: resolution.compatibleTables.map((candidate) =>
152
+ findTable(candidate).withSourceMetadata(candidate.sourceMetadata)
153
+ ),
154
+ incompatibleTables: resolution.incompatibleTables.map(findTable),
155
+ newTableValues: resolution.newTableValues
156
+ };
157
+ }
158
+
159
+ export interface MaterializedSourceTableResolution {
160
+ compatibleTables: SourceTable[];
161
+ incompatibleTables: SourceTable[];
162
+ newTableValues: SourceTableCreateValues;
163
+ }
164
+
165
+ /**
166
+ * Return source-metadata changes from compatible candidates, comparing metadata by value against
167
+ * the original storage-owned tables. The reconciler may mutate its isolated candidate clones, so
168
+ * those clones cannot be used as the persisted baseline.
169
+ */
170
+ export function diffSourceTableUpdates(
171
+ persistedTables: ReadonlyArray<SourceTable>,
172
+ resolution: SourceTableCandidateResolution
173
+ ): SourceTableMetadataUpdate[] {
174
+ const updates: SourceTableMetadataUpdate[] = [];
175
+ for (const resolvedTable of resolution.compatibleTables) {
176
+ const persistedTable = persistedTables.find((table) => sourceTableIdEquals(table.id, resolvedTable.id));
177
+ if (persistedTable == null) {
178
+ throw new ServiceAssertionError(
179
+ `Source table reconciliation returned unknown candidate ${resolvedTable.id.toString()}`
180
+ );
181
+ }
182
+ if (isDeepStrictEqual(persistedTable.sourceMetadata, resolvedTable.sourceMetadata)) {
183
+ continue;
184
+ }
185
+ updates.push({ id: resolvedTable.id, sourceMetadata: resolvedTable.sourceMetadata });
186
+ }
187
+ return updates;
188
+ }
@@ -39,6 +39,11 @@ export const STORAGE_VERSION_2 = 2;
39
39
  */
40
40
  export const STORAGE_VERSION_3 = 3;
41
41
 
42
+ /**
43
+ * Same as version 3, but marked as stable.
44
+ */
45
+ export const STORAGE_VERSION_4 = 4;
46
+
42
47
  /**
43
48
  * Oldest supported storage version.
44
49
  */
@@ -54,7 +59,7 @@ export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
54
59
  *
55
60
  * This includes unstable versions.
56
61
  */
57
- export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
62
+ export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3, STORAGE_VERSION_4];
58
63
 
59
64
  /**
60
65
  * Shared storage-version behavior across storage implementations.
@@ -74,5 +79,10 @@ export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undef
74
79
  version: STORAGE_VERSION_3,
75
80
  versionedBuckets: true,
76
81
  softDeleteCurrentData: true
82
+ },
83
+ [STORAGE_VERSION_4]: {
84
+ version: STORAGE_VERSION_4,
85
+ versionedBuckets: true,
86
+ softDeleteCurrentData: true
77
87
  }
78
88
  };