@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +152 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +36 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +30 -22
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/metrics/metrics-interfaces.d.ts +5 -1
  9. package/dist/metrics/metrics-interfaces.js.map +1 -1
  10. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
  11. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
  12. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
  13. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
  14. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
  15. package/dist/metrics/open-telemetry/util.js +7 -3
  16. package/dist/metrics/open-telemetry/util.js.map +1 -1
  17. package/dist/modules/AbstractModule.d.ts +6 -2
  18. package/dist/modules/AbstractModule.js.map +1 -1
  19. package/dist/replication/AbstractReplicator.d.ts +31 -7
  20. package/dist/replication/AbstractReplicator.js +168 -74
  21. package/dist/replication/AbstractReplicator.js.map +1 -1
  22. package/dist/replication/ReplicationModule.d.ts +3 -3
  23. package/dist/replication/ReplicationModule.js.map +1 -1
  24. package/dist/routes/RouterEngine.d.ts +1 -0
  25. package/dist/routes/RouterEngine.js +6 -1
  26. package/dist/routes/RouterEngine.js.map +1 -1
  27. package/dist/routes/configure-fastify.d.ts +104 -56
  28. package/dist/routes/configure-fastify.js +15 -2
  29. package/dist/routes/configure-fastify.js.map +1 -1
  30. package/dist/routes/endpoints/admin.d.ts +42 -24
  31. package/dist/routes/endpoints/admin.js +49 -36
  32. package/dist/routes/endpoints/admin.js.map +1 -1
  33. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  34. package/dist/routes/endpoints/checkpointing.js +65 -7
  35. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  36. package/dist/routes/endpoints/socket-route.js +3 -2
  37. package/dist/routes/endpoints/socket-route.js.map +1 -1
  38. package/dist/routes/endpoints/sync-rules.js +25 -14
  39. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  40. package/dist/routes/endpoints/sync-stream.js +1 -1
  41. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  42. package/dist/runner/teardown.js +8 -8
  43. package/dist/runner/teardown.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +64 -17
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/BucketStorageFactory.d.ts +50 -31
  47. package/dist/storage/BucketStorageFactory.js +10 -20
  48. package/dist/storage/BucketStorageFactory.js.map +1 -1
  49. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  50. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  51. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  52. package/dist/storage/ChecksumCache.d.ts +11 -6
  53. package/dist/storage/ChecksumCache.js +5 -4
  54. package/dist/storage/ChecksumCache.js.map +1 -1
  55. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  56. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  57. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  58. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  59. package/dist/storage/PersistedReplicationStream.js +26 -0
  60. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  61. package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
  62. package/dist/storage/PersistedSyncConfigContent.js +138 -0
  63. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  64. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  65. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  66. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  67. package/dist/storage/ReplicationEventPayload.d.ts +6 -1
  68. package/dist/storage/SourceEntity.d.ts +13 -2
  69. package/dist/storage/SourceTable.d.ts +53 -4
  70. package/dist/storage/SourceTable.js +62 -7
  71. package/dist/storage/SourceTable.js.map +1 -1
  72. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  73. package/dist/storage/SourceTableReconciler.js +95 -0
  74. package/dist/storage/SourceTableReconciler.js.map +1 -0
  75. package/dist/storage/StorageVersionConfig.d.ts +4 -0
  76. package/dist/storage/StorageVersionConfig.js +10 -1
  77. package/dist/storage/StorageVersionConfig.js.map +1 -1
  78. package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
  79. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  80. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  81. package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
  82. package/dist/storage/WriteCheckpointAPI.js +33 -0
  83. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  84. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
  85. package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
  86. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  87. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  88. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  89. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  90. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  91. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
  92. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  93. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  94. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  95. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  96. package/dist/storage/storage-index.d.ts +10 -1
  97. package/dist/storage/storage-index.js +10 -1
  98. package/dist/storage/storage-index.js.map +1 -1
  99. package/dist/storage/storage-metrics.js +120 -4
  100. package/dist/storage/storage-metrics.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +12 -2
  102. package/dist/sync/BucketChecksumState.js +69 -21
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.d.ts +7 -0
  105. package/dist/sync/RequestTracker.js +12 -0
  106. package/dist/sync/RequestTracker.js.map +1 -1
  107. package/dist/sync/sync.js +184 -43
  108. package/dist/sync/sync.js.map +1 -1
  109. package/dist/sync/util.d.ts +5 -0
  110. package/dist/sync/util.js +19 -1
  111. package/dist/sync/util.js.map +1 -1
  112. package/dist/system/ServiceContext.d.ts +2 -0
  113. package/dist/system/ServiceContext.js +2 -0
  114. package/dist/system/ServiceContext.js.map +1 -1
  115. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  116. package/dist/tracing/PerformanceTracer.js +11 -0
  117. package/dist/tracing/PerformanceTracer.js.map +1 -1
  118. package/dist/util/checkpointing.d.ts +4 -8
  119. package/dist/util/checkpointing.js +1 -16
  120. package/dist/util/checkpointing.js.map +1 -1
  121. package/dist/util/config/compound-config-collector.js +23 -2
  122. package/dist/util/config/compound-config-collector.js.map +1 -1
  123. package/dist/util/config/defaults.d.ts +2 -0
  124. package/dist/util/config/defaults.js +2 -0
  125. package/dist/util/config/defaults.js.map +1 -1
  126. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  127. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  128. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  130. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  131. package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
  132. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  133. package/dist/util/config/types.d.ts +4 -1
  134. package/dist/util/lsn.d.ts +4 -0
  135. package/dist/util/lsn.js +10 -0
  136. package/dist/util/lsn.js.map +1 -1
  137. package/dist/util/protocol-types.d.ts +4 -4
  138. package/dist/util/protocol-types.js +3 -2
  139. package/dist/util/protocol-types.js.map +1 -1
  140. package/dist/util/util-index.d.ts +1 -0
  141. package/dist/util/util-index.js +1 -0
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +3 -2
  144. package/dist/util/utils.js +17 -2
  145. package/dist/util/utils.js.map +1 -1
  146. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  147. package/dist/util/write-checkpoint-batcher.js +100 -0
  148. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  149. package/package.json +7 -7
  150. package/src/api/RouteAPI.ts +43 -14
  151. package/src/api/diagnostics.ts +45 -26
  152. package/src/entry/commands/compact-action.ts +37 -23
  153. package/src/metrics/metrics-interfaces.ts +6 -1
  154. package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
  155. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
  156. package/src/metrics/open-telemetry/util.ts +7 -3
  157. package/src/modules/AbstractModule.ts +6 -2
  158. package/src/replication/AbstractReplicator.ts +212 -78
  159. package/src/replication/ReplicationModule.ts +2 -3
  160. package/src/routes/RouterEngine.ts +7 -1
  161. package/src/routes/configure-fastify.ts +25 -2
  162. package/src/routes/endpoints/admin.ts +75 -51
  163. package/src/routes/endpoints/checkpointing.ts +79 -8
  164. package/src/routes/endpoints/socket-route.ts +3 -2
  165. package/src/routes/endpoints/sync-rules.ts +26 -14
  166. package/src/routes/endpoints/sync-stream.ts +1 -1
  167. package/src/runner/teardown.ts +8 -8
  168. package/src/storage/BucketStorageBatch.ts +65 -18
  169. package/src/storage/BucketStorageFactory.ts +73 -46
  170. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  171. package/src/storage/ChecksumCache.ts +27 -14
  172. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  173. package/src/storage/PersistedReplicationStream.ts +55 -0
  174. package/src/storage/PersistedSyncConfigContent.ts +207 -0
  175. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  176. package/src/storage/ReplicationEventPayload.ts +6 -1
  177. package/src/storage/SourceEntity.ts +12 -2
  178. package/src/storage/SourceTable.ts +96 -8
  179. package/src/storage/SourceTableReconciler.ts +188 -0
  180. package/src/storage/StorageVersionConfig.ts +11 -1
  181. package/src/storage/SyncRulesBucketStorage.ts +189 -27
  182. package/src/storage/WriteCheckpointAPI.ts +140 -6
  183. package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
  184. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  185. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
  186. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  187. package/src/storage/storage-index.ts +11 -1
  188. package/src/storage/storage-metrics.ts +154 -8
  189. package/src/sync/BucketChecksumState.ts +105 -21
  190. package/src/sync/RequestTracker.ts +20 -0
  191. package/src/sync/sync.ts +214 -50
  192. package/src/sync/util.ts +33 -8
  193. package/src/system/ServiceContext.ts +7 -0
  194. package/src/tracing/PerformanceTracer.ts +41 -1
  195. package/src/util/checkpointing.ts +4 -22
  196. package/src/util/config/compound-config-collector.ts +31 -1
  197. package/src/util/config/defaults.ts +2 -0
  198. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  199. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  200. package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
  201. package/src/util/config/types.ts +4 -1
  202. package/src/util/lsn.ts +9 -0
  203. package/src/util/protocol-types.ts +3 -2
  204. package/src/util/util-index.ts +1 -0
  205. package/src/util/utils.ts +16 -2
  206. package/src/util/write-checkpoint-batcher.ts +142 -0
  207. package/test/src/AbstractReplicator.test.ts +177 -0
  208. package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
  209. package/test/src/PersistedSyncConfigContent.test.ts +152 -0
  210. package/test/src/checksum_cache.test.ts +22 -0
  211. package/test/src/config.test.ts +142 -0
  212. package/test/src/diagnostics.test.ts +87 -34
  213. package/test/src/module-loader.test.ts +33 -1
  214. package/test/src/routes/admin.test.ts +188 -18
  215. package/test/src/routes/checkpointing.test.ts +54 -0
  216. package/test/src/routes/mocks.ts +6 -2
  217. package/test/src/routes/stream.test.ts +123 -2
  218. package/test/src/source-table-reconciler.test.ts +244 -0
  219. package/test/src/storage/SourceTable.test.ts +92 -0
  220. package/test/src/storage-metrics.test.ts +148 -0
  221. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  222. package/test/src/sync/util.test.ts +28 -1
  223. package/test/src/util/checkpointing.test.ts +249 -0
  224. package/test/src/util/utils.test.ts +23 -0
  225. package/tsconfig.json +1 -4
  226. package/tsconfig.tsbuildinfo +1 -1
  227. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  228. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  229. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  230. package/src/storage/PersistedSyncRulesContent.ts +0 -174
@@ -1,27 +1,94 @@
1
- import { BasicRouterRequest, Context, JwtPayload } from '@/index.js';
1
+ import { BasicRouterRequest, Context, JwtPayload, ParsedSyncConfigSet, storage } from '@/index.js';
2
2
  import { logger } from '@powersync/lib-services-framework';
3
- import { describe, expect, it } from 'vitest';
4
- import { validate } from '../../../src/routes/endpoints/admin.js';
3
+ import { SqlSyncRules } from '@powersync/service-sync-rules';
4
+ import { describe, expect, it, vi } from 'vitest';
5
+ import { diagnostics, reprocess, validate } from '../../../src/routes/endpoints/admin.js';
5
6
  import { mockServiceContext } from './mocks.js';
6
7
 
7
8
  describe('admin routes', () => {
9
+ const request: BasicRouterRequest = {
10
+ headers: {},
11
+ hostname: '',
12
+ protocol: 'http'
13
+ };
14
+
15
+ function makeContext(activeBucketStorage?: any): Context {
16
+ const service_context = mockServiceContext(null);
17
+ if (activeBucketStorage != null) {
18
+ (service_context.storageEngine as any).activeBucketStorage = activeBucketStorage;
19
+ }
20
+
21
+ return {
22
+ logger: logger,
23
+ service_context,
24
+ token_payload: new JwtPayload({
25
+ sub: '',
26
+ exp: 0,
27
+ iat: 0
28
+ })
29
+ };
30
+ }
31
+
32
+ function makeSyncConfigContent(options: {
33
+ id?: number;
34
+ syncConfigId?: string;
35
+ active?: boolean;
36
+ content?: string;
37
+ version_label?: string;
38
+ }): storage.PersistedSyncConfigContent {
39
+ const id = options.id ?? 1;
40
+ const syncConfigId = options.syncConfigId ?? String(id);
41
+ const active = options.active ?? true;
42
+ const state = active ? storage.SyncRuleState.ACTIVE : storage.SyncRuleState.PROCESSING;
43
+ const lastKeepaliveTs = new Date('2026-01-01T00:00:00.000Z');
44
+ const lastCheckpointTs = new Date('2026-01-01T00:00:00.000Z');
45
+ const syncConfigStatus = {
46
+ id: syncConfigId,
47
+ replicationStreamId: id,
48
+ state,
49
+ last_checkpoint_lsn: null,
50
+ last_fatal_error: null,
51
+ last_fatal_error_ts: null,
52
+ last_keepalive_ts: lastKeepaliveTs,
53
+ last_checkpoint_ts: lastCheckpointTs
54
+ };
55
+ const content = {
56
+ replicationStreamId: id,
57
+ syncConfigId,
58
+ replicationStreamName: `slot_${id}`,
59
+ sync_rules_content:
60
+ options.content ??
61
+ `
62
+ bucket_definitions:
63
+ global:
64
+ data:
65
+ - SELECT id FROM test
66
+ `,
67
+ compiled_plan: null,
68
+ storageVersion: storage.LEGACY_STORAGE_VERSION,
69
+ syncConfigState: syncConfigStatus.state,
70
+ version_label: options.version_label,
71
+ parsed(options?: any) {
72
+ const syncRules = SqlSyncRules.fromYaml(content.sync_rules_content, {
73
+ ...options,
74
+ defaultSchema: 'public'
75
+ });
76
+ return {
77
+ syncConfigs: [syncRules]
78
+ } as ParsedSyncConfigSet;
79
+ },
80
+ asUpdateOptions: vi.fn(),
81
+ getStorageConfig: vi.fn(),
82
+ async getSyncConfigStatus() {
83
+ return syncConfigStatus;
84
+ }
85
+ };
86
+ return content as unknown as storage.PersistedSyncConfigContent;
87
+ }
88
+
8
89
  describe('validate', () => {
9
90
  it('reports errors with source location', async () => {
10
- const context: Context = {
11
- logger: logger,
12
- service_context: mockServiceContext(null),
13
- token_payload: new JwtPayload({
14
- sub: '',
15
- exp: 0,
16
- iat: 0
17
- })
18
- };
19
-
20
- const request: BasicRouterRequest = {
21
- headers: {},
22
- hostname: '',
23
- protocol: 'http'
24
- };
91
+ const context = makeContext();
25
92
 
26
93
  const response = await validate.handler({
27
94
  context,
@@ -45,4 +112,107 @@ bucket_definitions:
45
112
  ]);
46
113
  });
47
114
  });
115
+
116
+ describe('diagnostics', () => {
117
+ it('returns deploying config status', async () => {
118
+ const active = makeSyncConfigContent({ id: 1, syncConfigId: 'active-config', version_label: 'v5' });
119
+ const deploying = makeSyncConfigContent({ id: 2, syncConfigId: 'deploying-config', active: false });
120
+ const getInstance = vi.fn(() => ({
121
+ async getStatus() {
122
+ return {
123
+ snapshotDone: false,
124
+ resumeLsn: null
125
+ };
126
+ }
127
+ }));
128
+ const activeBucketStorage = {
129
+ getActiveSyncConfig: vi.fn(async () => ({
130
+ content: active,
131
+ replicationStream: {},
132
+ storage: getInstance()
133
+ })),
134
+ getDeployingSyncConfig: vi.fn(async () => ({
135
+ content: deploying,
136
+ replicationStream: {},
137
+ storage: getInstance()
138
+ }))
139
+ };
140
+
141
+ const response = await diagnostics.handler({
142
+ context: makeContext(activeBucketStorage),
143
+ params: {},
144
+ request
145
+ });
146
+
147
+ expect(response.deploying_sync_rules?.connections[0].slot_name).toBe('slot_2');
148
+ expect(response.active_sync_rules?.connections[0].slot_name).toBe('slot_1');
149
+ expect(response.active_sync_rules?.version_label).toBe('v5');
150
+ });
151
+ });
152
+
153
+ describe('reprocess', () => {
154
+ it('reprocesses the active sync config', async () => {
155
+ const active = makeSyncConfigContent({ id: 7, syncConfigId: 'active-config', version_label: 'v6' });
156
+ const updateSyncRules = vi.fn(async () => ({
157
+ replicationStreamId: 8,
158
+ replicationStreamName: 'new_slot',
159
+ state: storage.SyncRuleState.PROCESSING,
160
+ storageVersion: storage.LEGACY_STORAGE_VERSION,
161
+ replicationJobId: '8',
162
+ current_lock: null
163
+ }));
164
+ const activeBucketStorage = {
165
+ getDeployingSyncConfig: vi.fn(async () => null),
166
+ getActiveSyncConfig: vi.fn(async () => ({
167
+ content: active,
168
+ replicationStream: {},
169
+ storage: {}
170
+ })),
171
+ getSyncConfigContent: vi.fn(),
172
+ updateSyncRules
173
+ };
174
+
175
+ const response = await reprocess.handler({
176
+ context: makeContext(activeBucketStorage),
177
+ params: {},
178
+ request
179
+ });
180
+
181
+ expect(activeBucketStorage.getActiveSyncConfig).toHaveBeenCalledTimes(1);
182
+ expect(activeBucketStorage.getSyncConfigContent).not.toHaveBeenCalled();
183
+ expect(updateSyncRules).toHaveBeenCalledTimes(1);
184
+ expect(updateSyncRules).toHaveBeenCalledWith(
185
+ expect.objectContaining({ version_label: 'v6', forceNewReplicationStream: true })
186
+ );
187
+ expect(response.connections[0].slot_name).toBe('new_slot');
188
+ });
189
+
190
+ it('rejects reprocess while a sync config is deploying', async () => {
191
+ const activeBucketStorage = {
192
+ getDeployingSyncConfig: vi.fn(async () => ({
193
+ content: makeSyncConfigContent({ id: 2, active: false }),
194
+ replicationStream: {},
195
+ storage: {}
196
+ })),
197
+ getActiveSyncConfig: vi.fn(),
198
+ updateSyncRules: vi.fn()
199
+ };
200
+
201
+ await expect(
202
+ reprocess.handler({
203
+ context: makeContext(activeBucketStorage),
204
+ params: {},
205
+ request
206
+ })
207
+ ).rejects.toMatchObject({
208
+ errorData: {
209
+ status: 409,
210
+ code: 'PSYNC_S4106',
211
+ description: 'Busy processing sync config - cannot reprocess'
212
+ }
213
+ });
214
+ expect(activeBucketStorage.getActiveSyncConfig).not.toHaveBeenCalled();
215
+ expect(activeBucketStorage.updateSyncRules).not.toHaveBeenCalled();
216
+ });
217
+ });
48
218
  });
@@ -0,0 +1,54 @@
1
+ import { describe, expect, it, vi } from 'vitest';
2
+ import { CheckpointRequestPayload, checkpointRequest } from '../../../src/routes/endpoints/checkpointing.js';
3
+
4
+ describe('checkpoint request route', () => {
5
+ const payload = (checkpointRequestId: string | number | bigint) => ({
6
+ client_id: 'client-a',
7
+ checkpoint_request_id: checkpointRequestId
8
+ });
9
+
10
+ it.each([
11
+ ['safe JSON number', 1, 1n],
12
+ ['max int64 string', '9223372036854775807', 9_223_372_036_854_775_807n],
13
+ ['bigint value', 42n, 42n]
14
+ ])('decodes a positive int64 checkpoint request id from a %s', (_description, checkpointRequestId, expected) => {
15
+ expect(CheckpointRequestPayload.decode(payload(checkpointRequestId)).checkpoint_request_id).toEqual(expected);
16
+ });
17
+
18
+ it.each([1, '9223372036854775807'])(
19
+ 'accepts API payloads with positive int64 checkpoint request id %s',
20
+ (checkpointRequestId) => {
21
+ expect(checkpointRequest.validator!.validate(payload(checkpointRequestId)).valid).toBe(true);
22
+ }
23
+ );
24
+
25
+ it.each([
26
+ 0,
27
+ -1,
28
+ '0',
29
+ '-1',
30
+ '9223372036854775808',
31
+ '999999999999999999999999999999999999999999999',
32
+ Number.MAX_SAFE_INTEGER + 1
33
+ ])('rejects invalid API checkpoint request id %s', (checkpointRequestId) => {
34
+ expect(checkpointRequest.validator!.validate(payload(checkpointRequestId)).valid).toBe(false);
35
+ });
36
+
37
+ it('returns the stored checkpoint request id from the checkpoint request route', async () => {
38
+ const writeCheckpointBatcher = {
39
+ enqueue: vi.fn(async () => ({ replicationHead: 'head-1', writeCheckpoint: '43' }))
40
+ };
41
+
42
+ await expect(
43
+ checkpointRequest.handler({
44
+ context: {
45
+ token_payload: { userIdString: 'user-a' },
46
+ service_context: { writeCheckpointBatcher }
47
+ },
48
+ params: payload(42)
49
+ } as any)
50
+ ).resolves.toEqual({ checkpoint_request_id: '43' });
51
+
52
+ expect(writeCheckpointBatcher.enqueue).toHaveBeenCalledWith('user-a/client-a', 42n);
53
+ });
54
+ });
@@ -18,8 +18,12 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
18
18
 
19
19
  const storageEngine: StorageEngine = {
20
20
  activeBucketStorage: {
21
- async getActiveStorage() {
22
- return storage;
21
+ async getActiveSyncConfig() {
22
+ return storage == null
23
+ ? null
24
+ : {
25
+ storage
26
+ };
23
27
  }
24
28
  } as Partial<BucketStorageFactory>
25
29
  } as any;
@@ -1,4 +1,12 @@
1
- import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
1
+ import {
2
+ BasicRouterRequest,
3
+ CHECKPOINT_INVALIDATE_ALL,
4
+ Context,
5
+ JwtPayload,
6
+ RequestTracker,
7
+ streamResponse,
8
+ SyncRulesBucketStorage
9
+ } from '@/index.js';
2
10
  import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
3
11
  import {
4
12
  DEFAULT_HYDRATION_STATE,
@@ -9,7 +17,7 @@ import {
9
17
  import * as sqlite from 'node:sqlite';
10
18
  import { Readable, Writable } from 'stream';
11
19
  import { pipeline } from 'stream/promises';
12
- import { describe, expect, it } from 'vitest';
20
+ import { describe, expect, it, vi } from 'vitest';
13
21
  import winston from 'winston';
14
22
  import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
15
23
  import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
@@ -182,6 +190,119 @@ describe('Stream Route', () => {
182
190
  );
183
191
  });
184
192
  });
193
+
194
+ it('closes after a data fetch error while waiting for a new checkpoint', async () => {
195
+ // After 1,000 operations, the sync stream concurrently calls next() on the
196
+ // checkpoint watcher. This test then fails the next data fetch without
197
+ // producing another checkpoint. Before the fix, awaiting watcher.return()
198
+ // during cleanup queued it behind that pending next(), leaving the response
199
+ // open until an unrelated checkpoint was written.
200
+ const syncRules = SqlSyncRules.fromYaml(
201
+ `
202
+ bucket_definitions:
203
+ global:
204
+ data: []
205
+ `,
206
+ { defaultSchema: 'public' }
207
+ ).config.hydrate(defaultHydrationOptions);
208
+
209
+ let checkpointWatcherAborted = false;
210
+ let dataFetches = 0;
211
+ const storage = {
212
+ watchCheckpointChanges: async function* ({ signal }) {
213
+ yield {
214
+ base: {
215
+ checkpoint: 1n,
216
+ lsn: '1',
217
+ getParameterSets: async () => []
218
+ },
219
+ writeCheckpoint: null,
220
+ update: CHECKPOINT_INVALIDATE_ALL
221
+ };
222
+
223
+ await new Promise<void>((_resolve, reject) => {
224
+ // There is intentionally no next checkpoint: cleanup must abort this
225
+ // pending wait rather than wait for it to complete naturally.
226
+ signal.addEventListener(
227
+ 'abort',
228
+ () => {
229
+ checkpointWatcherAborted = true;
230
+ reject(new Error('Checkpoint watcher aborted'));
231
+ },
232
+ { once: true }
233
+ );
234
+ });
235
+ },
236
+ getChecksums: async (_checkpoint, buckets) =>
237
+ new Map(buckets.map(({ bucket }) => [bucket, { bucket, checksum: 1, count: 1000 }])),
238
+ getBucketDataBatch: async function* () {
239
+ if (dataFetches++ > 0) {
240
+ throw new Error('Simulated data fetch failure');
241
+ }
242
+
243
+ yield {
244
+ chunkData: {
245
+ bucket: 'global[]',
246
+ data: Array.from({ length: 1000 }, () => ({ op: 'PUT' })),
247
+ has_more: true,
248
+ after: '0',
249
+ next_after: '1000'
250
+ },
251
+ targetOp: null
252
+ };
253
+ yield { hasMore: true };
254
+ }
255
+ } as Partial<SyncRulesBucketStorage>;
256
+ const serviceContext = mockServiceContext(storage);
257
+ const controller = new AbortController();
258
+ const checkpointLogger = logger.child({});
259
+ const checkpointLogSpy = vi.spyOn(checkpointLogger, 'info');
260
+
261
+ const response = (async () => {
262
+ for await (const _line of streamResponse({
263
+ syncContext: serviceContext.syncContext,
264
+ bucketStorage: storage as SyncRulesBucketStorage,
265
+ syncRules,
266
+ params: { client_id: 'test-client', raw_data: true },
267
+ token: new JwtPayload({ sub: 'test-user', exp: Date.now() / 1000 + 10_000 }),
268
+ tracker: new RequestTracker(serviceContext.metricsEngine),
269
+ isEncodingAsBson: false,
270
+ signal: controller.signal,
271
+ logger: checkpointLogger
272
+ })) {
273
+ // Consume the response until the simulated data fetch error is propagated.
274
+ }
275
+ })();
276
+
277
+ let timeout: NodeJS.Timeout | undefined;
278
+ const outcome = await Promise.race([
279
+ response.then(
280
+ () => new Error('Sync stream unexpectedly completed'),
281
+ (error) => error
282
+ ),
283
+ new Promise((resolve) => {
284
+ timeout = setTimeout(() => resolve(new Error('Sync stream did not close')), 500);
285
+ })
286
+ ]);
287
+ clearTimeout(timeout);
288
+
289
+ // Capture this before aborting below, which would set it regardless of the stream cleanup.
290
+ const abortedByCleanup = checkpointWatcherAborted;
291
+
292
+ controller.abort();
293
+ await response.catch(() => {});
294
+
295
+ expect(outcome).toBeInstanceOf(Error);
296
+ expect((outcome as Error).message).toContain('Simulated data fetch failure');
297
+ expect(abortedByCleanup).toBe(true);
298
+ expect(checkpointLogSpy).toHaveBeenCalledWith('checkpoint_interrupted: 1', {
299
+ checkpoint: 1n,
300
+ user_id: 'test-user',
301
+ operations_synced: 1000,
302
+ data_synced_bytes: 0,
303
+ ms: expect.any(Object)
304
+ });
305
+ });
185
306
  });
186
307
 
187
308
  export async function drainWithTimeout(readable: Readable, ms = 2_000) {
@@ -0,0 +1,244 @@
1
+ import { SourceEntityDescriptor } from '@/storage/SourceEntity.js';
2
+ import { SourceTable, sourceTableIdEquals } from '@/storage/SourceTable.js';
3
+ import {
4
+ defaultSourceTableReconciler,
5
+ diffSourceTableUpdates,
6
+ materializeSourceTableResolution,
7
+ sourceIdentityCompatible,
8
+ validateSourceTableCandidateResolution
9
+ } from '@/storage/SourceTableReconciler.js';
10
+ import * as bson from 'bson';
11
+ import { describe, expect, it } from 'vitest';
12
+
13
+ function descriptor(overrides: Partial<SourceEntityDescriptor> = {}): SourceEntityDescriptor {
14
+ return {
15
+ connectionTag: 'default',
16
+ schema: 'public',
17
+ name: 'users',
18
+ objectId: 100,
19
+ replicaIdColumns: [{ name: 'id', type: 'int', typeId: 23 }],
20
+ ...overrides
21
+ };
22
+ }
23
+
24
+ function candidate(overrides: Partial<ConstructorParameters<typeof SourceTable>[0]> = {}): SourceTable {
25
+ return new SourceTable({
26
+ id: overrides.id ?? 'table-1',
27
+ ref: overrides.ref ?? { connectionTag: 'default', schema: 'public', name: 'users' },
28
+ objectId: 'objectId' in overrides ? overrides.objectId! : 100,
29
+ replicaIdColumns: overrides.replicaIdColumns ?? [{ name: 'id', type: 'int', typeId: 23 }],
30
+ snapshotComplete: overrides.snapshotComplete ?? true,
31
+ bucketDataSources: overrides.bucketDataSources ?? [],
32
+ parameterLookupSources: overrides.parameterLookupSources ?? [],
33
+ sourceMetadata: overrides.sourceMetadata
34
+ });
35
+ }
36
+
37
+ describe('sourceIdentityCompatible', () => {
38
+ it('matches identical identity', () => {
39
+ expect(sourceIdentityCompatible(descriptor(), candidate())).toBe(true);
40
+ });
41
+
42
+ it('rejects a different object id', () => {
43
+ expect(sourceIdentityCompatible(descriptor({ objectId: 200 }), candidate({ objectId: 100 }))).toBe(false);
44
+ });
45
+
46
+ it('rejects a different schema/name', () => {
47
+ expect(sourceIdentityCompatible(descriptor({ name: 'accounts' }), candidate())).toBe(false);
48
+ });
49
+
50
+ it('rejects changed replica-id columns', () => {
51
+ expect(
52
+ sourceIdentityCompatible(
53
+ descriptor(),
54
+ candidate({ replicaIdColumns: [{ name: 'id', type: 'bigint', typeId: 20 }] })
55
+ )
56
+ ).toBe(false);
57
+ });
58
+
59
+ it('treats an undefined descriptor object id as a wildcard on object id', () => {
60
+ expect(sourceIdentityCompatible(descriptor({ objectId: undefined }), candidate({ objectId: 999 }))).toBe(true);
61
+ });
62
+ });
63
+
64
+ describe('defaultSourceTableReconciler', () => {
65
+ it('returns all identity-compatible candidates and no metadata', async () => {
66
+ const a = candidate({ id: 'a' });
67
+ const b = candidate({ id: 'b', ref: { connectionTag: 'default', schema: 'public', name: 'accounts' } });
68
+ const resolution = await defaultSourceTableReconciler({ source: descriptor(), candidates: [a, b] });
69
+ expect(resolution.compatibleTables).toEqual([a]);
70
+ expect(resolution.incompatibleTables).toEqual([b]);
71
+ expect(resolution.newTableValues).toEqual({ sourceMetadata: null });
72
+ });
73
+
74
+ it('returns an empty set when nothing matches', async () => {
75
+ const resolution = await defaultSourceTableReconciler({
76
+ source: descriptor({ objectId: 200 }),
77
+ candidates: [candidate({ objectId: 100 })]
78
+ });
79
+ expect(resolution.compatibleTables).toHaveLength(0);
80
+ expect(resolution.incompatibleTables.map((table) => table.id)).toEqual(['table-1']);
81
+ });
82
+ });
83
+
84
+ describe('validateSourceTableCandidateResolution', () => {
85
+ it('accepts an explicit compatible/incompatible partition', () => {
86
+ const a = candidate({ id: 'a' });
87
+ const b = candidate({ id: 'b' });
88
+ expect(() =>
89
+ validateSourceTableCandidateResolution([a, b], {
90
+ compatibleTables: [a],
91
+ incompatibleTables: [b],
92
+ newTableValues: { sourceMetadata: null }
93
+ })
94
+ ).not.toThrow();
95
+ });
96
+
97
+ it('rejects omitted, duplicate, and unknown candidates', () => {
98
+ const a = candidate({ id: 'a' });
99
+ expect(() =>
100
+ validateSourceTableCandidateResolution([a], {
101
+ compatibleTables: [],
102
+ incompatibleTables: [],
103
+ newTableValues: { sourceMetadata: null }
104
+ })
105
+ ).toThrow(/exactly once/);
106
+ expect(() =>
107
+ validateSourceTableCandidateResolution([a], {
108
+ compatibleTables: [a],
109
+ incompatibleTables: [a],
110
+ newTableValues: { sourceMetadata: null }
111
+ })
112
+ ).toThrow(/exactly once/);
113
+ expect(() =>
114
+ validateSourceTableCandidateResolution([a], {
115
+ compatibleTables: [a],
116
+ incompatibleTables: [candidate({ id: 'unknown' })],
117
+ newTableValues: { sourceMetadata: null }
118
+ })
119
+ ).toThrow(/unknown candidate/);
120
+ });
121
+ });
122
+
123
+ describe('diffSourceTableUpdates', () => {
124
+ function resolution(compatibleTables: SourceTable[]) {
125
+ return { compatibleTables, incompatibleTables: [], newTableValues: { sourceMetadata: null } };
126
+ }
127
+
128
+ it('returns nothing when the reconciler returned the candidates untouched', () => {
129
+ const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
130
+ expect(diffSourceTableUpdates([a], resolution([a]))).toEqual([]);
131
+ });
132
+
133
+ it('compares by value, not by reference', () => {
134
+ // A new object with equal metadata should not cause a write.
135
+ const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
136
+ const rebuilt = a.withSourceMetadata({ captureTableObjectId: 7 });
137
+
138
+ expect(rebuilt.sourceMetadata).not.toBe(a.sourceMetadata);
139
+ expect(diffSourceTableUpdates([a], resolution([rebuilt]))).toEqual([]);
140
+ });
141
+
142
+ it('returns changed metadata', () => {
143
+ const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
144
+ expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata({ captureTableObjectId: 8 })]))).toEqual([
145
+ { id: 'a', sourceMetadata: { captureTableObjectId: 8 } }
146
+ ]);
147
+ });
148
+
149
+ it('returns metadata added to a legacy record', () => {
150
+ const legacy = candidate({ id: 'a' });
151
+ expect(legacy.sourceMetadata).toBeNull();
152
+ expect(
153
+ diffSourceTableUpdates([legacy], resolution([legacy.withSourceMetadata({ captureTableObjectId: 7 })]))
154
+ ).toEqual([{ id: 'a', sourceMetadata: { captureTableObjectId: 7 } }]);
155
+ });
156
+
157
+ it('returns cleared metadata as null', () => {
158
+ const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
159
+ expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata(null)]))).toEqual([
160
+ { id: 'a', sourceMetadata: null }
161
+ ]);
162
+ });
163
+
164
+ it('materializes only source metadata from reconciler results', () => {
165
+ const persisted = candidate({ id: 'a', snapshotComplete: false });
166
+ const modified = persisted.withSourceMetadata({ captureTableObjectId: 8 });
167
+ modified.snapshotComplete = true;
168
+ modified.syncData = false;
169
+
170
+ const [materialized] = materializeSourceTableResolution([persisted], resolution([modified])).compatibleTables;
171
+
172
+ expect(materialized.sourceMetadata).toEqual({ captureTableObjectId: 8 });
173
+ expect(materialized.snapshotComplete).toBe(false);
174
+ expect(materialized.syncData).toBe(persisted.syncData);
175
+ });
176
+
177
+ it('does not trust incompatible candidate mutations made through a cast', () => {
178
+ const persisted = candidate({ id: 'a', snapshotComplete: false });
179
+ const exposed = persisted.clone();
180
+ // Simulate callback code bypassing the read-only type boundary.
181
+ (exposed as any).snapshotComplete = true;
182
+
183
+ const materialized = materializeSourceTableResolution([persisted], {
184
+ compatibleTables: [],
185
+ incompatibleTables: [exposed],
186
+ newTableValues: { sourceMetadata: null }
187
+ }).incompatibleTables[0];
188
+
189
+ expect(exposed.snapshotComplete).toBe(true);
190
+ expect(materialized).toBe(persisted);
191
+ expect(materialized.snapshotComplete).toBe(false);
192
+ expect(materialized.syncData).toBe(persisted.syncData);
193
+ });
194
+
195
+ it('only reports the candidates that changed', () => {
196
+ const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
197
+ const b = candidate({ id: 'b', sourceMetadata: { captureTableObjectId: 7 } });
198
+ const updates = diffSourceTableUpdates([a, b], resolution([a, b.withSourceMetadata({ captureTableObjectId: 9 })]));
199
+ expect(updates).toEqual([{ id: 'b', sourceMetadata: { captureTableObjectId: 9 } }]);
200
+ });
201
+
202
+ it('rejects a compatible table that was never a candidate', () => {
203
+ expect(() => diffSourceTableUpdates([candidate({ id: 'a' })], resolution([candidate({ id: 'ghost' })]))).toThrow(
204
+ /unknown candidate/
205
+ );
206
+ });
207
+ });
208
+
209
+ describe('SourceTable.clone', () => {
210
+ it('isolates mutable identity and metadata values', () => {
211
+ const persisted = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
212
+ const exposed = persisted.clone();
213
+
214
+ (exposed.ref as any).name = 'changed';
215
+ exposed.replicaIdColumns[0].name = 'changed';
216
+ (exposed.sourceMetadata as any).captureTableObjectId = 8;
217
+
218
+ expect(persisted.name).toBe('users');
219
+ expect(persisted.replicaIdColumns[0].name).toBe('id');
220
+ expect(persisted.sourceMetadata).toEqual({ captureTableObjectId: 7 });
221
+ });
222
+ });
223
+
224
+ describe('sourceTableIdEquals', () => {
225
+ it('compares string ids by value', () => {
226
+ expect(sourceTableIdEquals('table-1', 'table-1')).toBe(true);
227
+ expect(sourceTableIdEquals('table-1', 'table-2')).toBe(false);
228
+ });
229
+
230
+ it('compares BSON ObjectIds by value', () => {
231
+ const id = new bson.ObjectId();
232
+ const copy = new bson.ObjectId(id.toHexString());
233
+
234
+ expect(id).not.toBe(copy);
235
+ expect(sourceTableIdEquals(id, copy)).toBe(true);
236
+ });
237
+
238
+ it('does not mix string and BSON ObjectId representations', () => {
239
+ const id = new bson.ObjectId();
240
+
241
+ expect(sourceTableIdEquals(id.toHexString(), id)).toBe(false);
242
+ expect(sourceTableIdEquals(id, id.toHexString())).toBe(false);
243
+ });
244
+ });