@powersync/service-core 0.0.0-dev-20260313100403 → 0.0.0-dev-20260515144844

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (235) hide show
  1. package/CHANGELOG.md +86 -7
  2. package/dist/api/RouteAPI.d.ts +17 -3
  3. package/dist/api/api-index.d.ts +1 -1
  4. package/dist/api/api-index.js +1 -1
  5. package/dist/api/api-index.js.map +1 -1
  6. package/dist/api/api-metrics.js.map +1 -1
  7. package/dist/api/diagnostics.d.ts +1 -1
  8. package/dist/api/diagnostics.js +32 -14
  9. package/dist/api/diagnostics.js.map +1 -1
  10. package/dist/auth/CachedKeyCollector.js +1 -1
  11. package/dist/auth/CachedKeyCollector.js.map +1 -1
  12. package/dist/auth/CompoundKeyCollector.js.map +1 -1
  13. package/dist/auth/KeyStore.js.map +1 -1
  14. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  15. package/dist/auth/StaticKeyCollector.d.ts +1 -1
  16. package/dist/auth/StaticKeyCollector.js.map +1 -1
  17. package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
  18. package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
  19. package/dist/entry/commands/compact-action.js +26 -5
  20. package/dist/entry/commands/compact-action.js.map +1 -1
  21. package/dist/entry/commands/teardown-action.js +2 -2
  22. package/dist/entry/commands/teardown-action.js.map +1 -1
  23. package/dist/entry/entry-index.d.ts +1 -1
  24. package/dist/entry/entry-index.js +1 -1
  25. package/dist/entry/entry-index.js.map +1 -1
  26. package/dist/events/EventsEngine.js +1 -1
  27. package/dist/events/EventsEngine.js.map +1 -1
  28. package/dist/index.d.ts +1 -0
  29. package/dist/index.js +1 -0
  30. package/dist/index.js.map +1 -1
  31. package/dist/metrics/MetricsEngine.d.ts +1 -1
  32. package/dist/metrics/RollingBucketMax.d.ts +28 -0
  33. package/dist/metrics/RollingBucketMax.js +80 -0
  34. package/dist/metrics/RollingBucketMax.js.map +1 -0
  35. package/dist/metrics/metrics-index.d.ts +3 -2
  36. package/dist/metrics/metrics-index.js +3 -2
  37. package/dist/metrics/metrics-index.js.map +1 -1
  38. package/dist/metrics/open-telemetry/util.js +1 -1
  39. package/dist/metrics/open-telemetry/util.js.map +1 -1
  40. package/dist/metrics/register-metrics.js +2 -2
  41. package/dist/metrics/register-metrics.js.map +1 -1
  42. package/dist/modules/AbstractModule.d.ts +2 -2
  43. package/dist/modules/AbstractModule.js.map +1 -1
  44. package/dist/modules/modules-index.d.ts +1 -1
  45. package/dist/modules/modules-index.js +1 -1
  46. package/dist/modules/modules-index.js.map +1 -1
  47. package/dist/replication/AbstractReplicationJob.d.ts +2 -2
  48. package/dist/replication/AbstractReplicationJob.js +1 -1
  49. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  50. package/dist/replication/AbstractReplicator.d.ts +7 -7
  51. package/dist/replication/AbstractReplicator.js +31 -28
  52. package/dist/replication/AbstractReplicator.js.map +1 -1
  53. package/dist/replication/ReplicationLagTracker.d.ts +50 -0
  54. package/dist/replication/ReplicationLagTracker.js +78 -0
  55. package/dist/replication/ReplicationLagTracker.js.map +1 -0
  56. package/dist/replication/replication-index.d.ts +3 -2
  57. package/dist/replication/replication-index.js +3 -2
  58. package/dist/replication/replication-index.js.map +1 -1
  59. package/dist/replication/replication-metrics.js.map +1 -1
  60. package/dist/routes/configure-fastify.d.ts +59 -32
  61. package/dist/routes/endpoints/admin.d.ts +108 -54
  62. package/dist/routes/endpoints/admin.js +7 -3
  63. package/dist/routes/endpoints/admin.js.map +1 -1
  64. package/dist/routes/endpoints/checkpointing.js +1 -1
  65. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  66. package/dist/routes/endpoints/socket-route.js +1 -1
  67. package/dist/routes/endpoints/socket-route.js.map +1 -1
  68. package/dist/routes/endpoints/sync-rules.js +10 -10
  69. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  70. package/dist/routes/endpoints/sync-stream.d.ts +10 -10
  71. package/dist/routes/endpoints/sync-stream.js +2 -2
  72. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  73. package/dist/routes/hooks.js +1 -1
  74. package/dist/routes/hooks.js.map +1 -1
  75. package/dist/routes/route-register.js.map +1 -1
  76. package/dist/runner/teardown.js +4 -4
  77. package/dist/runner/teardown.js.map +1 -1
  78. package/dist/storage/BucketStorage.d.ts +9 -9
  79. package/dist/storage/BucketStorage.js +9 -9
  80. package/dist/storage/BucketStorageBatch.d.ts +1 -1
  81. package/dist/storage/BucketStorageFactory.d.ts +27 -20
  82. package/dist/storage/BucketStorageFactory.js +19 -16
  83. package/dist/storage/BucketStorageFactory.js.map +1 -1
  84. package/dist/storage/ChecksumCache.js.map +1 -1
  85. package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
  86. package/dist/storage/PersistedSyncRulesContent.js +24 -5
  87. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  88. package/dist/storage/ReplicationEventPayload.d.ts +1 -1
  89. package/dist/storage/ReportStorage.d.ts +3 -3
  90. package/dist/storage/SourceTable.d.ts +4 -4
  91. package/dist/storage/SourceTable.js +3 -3
  92. package/dist/storage/SourceTable.js.map +1 -1
  93. package/dist/storage/StorageVersionConfig.d.ts +1 -1
  94. package/dist/storage/StorageVersionConfig.js +1 -1
  95. package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
  96. package/dist/storage/SyncRulesBucketStorage.js +14 -0
  97. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  98. package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
  99. package/dist/storage/WriteCheckpointAPI.js +1 -1
  100. package/dist/storage/bson.d.ts +0 -1
  101. package/dist/storage/bson.js +0 -4
  102. package/dist/storage/bson.js.map +1 -1
  103. package/dist/storage/storage-index.d.ts +8 -8
  104. package/dist/storage/storage-index.js +8 -8
  105. package/dist/storage/storage-index.js.map +1 -1
  106. package/dist/storage/storage-metrics.js.map +1 -1
  107. package/dist/streams/streams-index.d.ts +2 -2
  108. package/dist/streams/streams-index.js +2 -2
  109. package/dist/streams/streams-index.js.map +1 -1
  110. package/dist/sync/BucketChecksumState.d.ts +2 -5
  111. package/dist/sync/BucketChecksumState.js +119 -75
  112. package/dist/sync/BucketChecksumState.js.map +1 -1
  113. package/dist/sync/RequestTracker.js +1 -1
  114. package/dist/sync/RequestTracker.js.map +1 -1
  115. package/dist/sync/sync-index.d.ts +2 -2
  116. package/dist/sync/sync-index.js +2 -2
  117. package/dist/sync/sync-index.js.map +1 -1
  118. package/dist/sync/sync.js.map +1 -1
  119. package/dist/sync/util.js.map +1 -1
  120. package/dist/system/ServiceContext.d.ts +1 -1
  121. package/dist/system/ServiceContext.js +1 -1
  122. package/dist/system/ServiceContext.js.map +1 -1
  123. package/dist/tracing/PerformanceTracer.d.ts +44 -0
  124. package/dist/tracing/PerformanceTracer.js +102 -0
  125. package/dist/tracing/PerformanceTracer.js.map +1 -0
  126. package/dist/tracing/TraceWriter.d.ts +22 -0
  127. package/dist/tracing/TraceWriter.js +63 -0
  128. package/dist/tracing/TraceWriter.js.map +1 -0
  129. package/dist/util/checkpointing.js +1 -1
  130. package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
  131. package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
  132. package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
  133. package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
  134. package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
  135. package/dist/util/config/compound-config-collector.d.ts +1 -1
  136. package/dist/util/config/compound-config-collector.js +2 -2
  137. package/dist/util/config/compound-config-collector.js.map +1 -1
  138. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
  139. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  140. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  141. package/dist/util/config.js +1 -1
  142. package/dist/util/config.js.map +1 -1
  143. package/dist/util/env.js +1 -1
  144. package/dist/util/errors.d.ts +3 -0
  145. package/dist/util/errors.js +15 -0
  146. package/dist/util/errors.js.map +1 -0
  147. package/dist/util/protocol-types.d.ts +3 -3
  148. package/dist/util/protocol-types.js +1 -1
  149. package/dist/util/util-index.d.ts +1 -1
  150. package/dist/util/util-index.js +1 -1
  151. package/dist/util/util-index.js.map +1 -1
  152. package/dist/util/utils.d.ts +1 -1
  153. package/package.json +11 -11
  154. package/src/api/RouteAPI.ts +20 -3
  155. package/src/api/api-index.ts +1 -1
  156. package/src/api/api-metrics.ts +1 -1
  157. package/src/api/diagnostics.ts +42 -20
  158. package/src/auth/CachedKeyCollector.ts +2 -3
  159. package/src/auth/CompoundKeyCollector.ts +2 -3
  160. package/src/auth/KeyStore.ts +1 -1
  161. package/src/auth/RemoteJWKSCollector.ts +0 -1
  162. package/src/auth/StaticKeyCollector.ts +1 -1
  163. package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
  164. package/src/entry/commands/compact-action.ts +29 -5
  165. package/src/entry/commands/teardown-action.ts +2 -2
  166. package/src/entry/entry-index.ts +1 -1
  167. package/src/events/EventsEngine.ts +1 -1
  168. package/src/index.ts +2 -0
  169. package/src/metrics/MetricsEngine.ts +1 -1
  170. package/src/metrics/RollingBucketMax.ts +109 -0
  171. package/src/metrics/metrics-index.ts +3 -2
  172. package/src/metrics/open-telemetry/util.ts +1 -1
  173. package/src/metrics/register-metrics.ts +3 -3
  174. package/src/modules/AbstractModule.ts +2 -2
  175. package/src/modules/modules-index.ts +1 -1
  176. package/src/replication/AbstractReplicationJob.ts +3 -3
  177. package/src/replication/AbstractReplicator.ts +32 -30
  178. package/src/replication/ReplicationLagTracker.ts +86 -0
  179. package/src/replication/replication-index.ts +3 -2
  180. package/src/replication/replication-metrics.ts +1 -1
  181. package/src/routes/endpoints/admin.ts +7 -3
  182. package/src/routes/endpoints/checkpointing.ts +1 -1
  183. package/src/routes/endpoints/socket-route.ts +1 -1
  184. package/src/routes/endpoints/sync-rules.ts +10 -12
  185. package/src/routes/endpoints/sync-stream.ts +2 -2
  186. package/src/routes/hooks.ts +2 -2
  187. package/src/routes/route-register.ts +2 -10
  188. package/src/runner/teardown.ts +4 -4
  189. package/src/storage/BucketStorage.ts +9 -9
  190. package/src/storage/BucketStorageBatch.ts +1 -1
  191. package/src/storage/BucketStorageFactory.ts +45 -34
  192. package/src/storage/ChecksumCache.ts +1 -1
  193. package/src/storage/PersistedSyncRulesContent.ts +30 -6
  194. package/src/storage/ReplicationEventPayload.ts +1 -1
  195. package/src/storage/ReportStorage.ts +3 -3
  196. package/src/storage/SourceTable.ts +4 -4
  197. package/src/storage/StorageVersionConfig.ts +1 -1
  198. package/src/storage/SyncRulesBucketStorage.ts +46 -7
  199. package/src/storage/WriteCheckpointAPI.ts +6 -6
  200. package/src/storage/bson.ts +0 -5
  201. package/src/storage/storage-index.ts +8 -8
  202. package/src/storage/storage-metrics.ts +2 -2
  203. package/src/streams/streams-index.ts +2 -2
  204. package/src/sync/BucketChecksumState.ts +141 -93
  205. package/src/sync/RequestTracker.ts +1 -1
  206. package/src/sync/sync-index.ts +2 -2
  207. package/src/sync/sync.ts +2 -8
  208. package/src/sync/util.ts +1 -1
  209. package/src/system/ServiceContext.ts +1 -1
  210. package/src/tracing/PerformanceTracer.ts +126 -0
  211. package/src/tracing/TraceWriter.ts +67 -0
  212. package/src/util/checkpointing.ts +1 -1
  213. package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
  214. package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
  215. package/src/util/config/compound-config-collector.ts +3 -3
  216. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
  217. package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
  218. package/src/util/config.ts +1 -1
  219. package/src/util/env.ts +1 -1
  220. package/src/util/errors.ts +21 -0
  221. package/src/util/protocol-types.ts +1 -1
  222. package/src/util/util-index.ts +1 -1
  223. package/src/util/utils.ts +1 -1
  224. package/test/src/ReplicationLagTracker.test.ts +53 -0
  225. package/test/src/RollingBucketMax.test.ts +106 -0
  226. package/test/src/auth.test.ts +115 -7
  227. package/test/src/diagnostics.test.ts +151 -0
  228. package/test/src/module-loader.test.ts +1 -1
  229. package/test/src/routes/mocks.ts +1 -1
  230. package/test/src/routes/stream.test.ts +1 -2
  231. package/test/src/sync/BucketChecksumState.test.ts +223 -67
  232. package/test/src/util/protocol_types.test.ts +1 -1
  233. package/test/tsconfig.json +0 -1
  234. package/tsconfig.tsbuildinfo +1 -1
  235. package/vitest.config.ts +1 -1
@@ -1,31 +1,33 @@
1
1
  import { BaseObserver, logger } from '@powersync/lib-services-framework';
2
- import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
3
- import { ReplicationEventPayload } from './ReplicationEventPayload.js';
4
- import { ReplicationLock } from './ReplicationLock.js';
5
- import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
6
- import { ReportStorage } from './ReportStorage.js';
7
2
  import {
8
3
  PrecompiledSyncConfig,
9
4
  SerializedCompatibilityContext,
10
5
  serializeSyncPlan,
11
6
  SqlSyncRules,
12
- SyncConfig
7
+ SyncConfigWithErrors
13
8
  } from '@powersync/service-sync-rules';
9
+ import { ReplicationError } from '@powersync/service-types';
10
+ import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
11
+ import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
12
+ import { ReplicationEventPayload } from './ReplicationEventPayload.js';
13
+ import { ReplicationLock } from './ReplicationLock.js';
14
+ import { ReportStorage } from './ReportStorage.js';
15
+ import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
14
16
 
15
17
  /**
16
18
  * Represents a configured storage provider.
17
19
  *
18
- * The provider can handle multiple copies of sync rules concurrently, each with their own storage.
19
- * This is to handle replication of a new version of sync rules, while the old version is still active.
20
+ * The provider can handle multiple replication streams concurrently, each with their own storage.
21
+ * This is to handle replication of a new version of sync config, while the old replication stream is still active.
20
22
  *
21
- * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
23
+ * Storage APIs for a specific replication stream are provided by the `SyncRulesBucketStorage` instances.
22
24
  */
23
25
  export abstract class BucketStorageFactory
24
26
  extends BaseObserver<BucketStorageFactoryListener>
25
27
  implements AsyncDisposable
26
28
  {
27
29
  /**
28
- * Update sync rules from configuration, if changed.
30
+ * Update sync config from configuration, if changed.
29
31
  */
30
32
  async configureSyncRules(
31
33
  options: UpdateSyncRulesOptions
@@ -34,42 +36,42 @@ export abstract class BucketStorageFactory
34
36
  const active = await this.getActiveSyncRulesContent();
35
37
 
36
38
  if (next?.sync_rules_content == options.config.yaml) {
37
- logger.info('Sync rules from configuration unchanged');
39
+ logger.info('Sync config unchanged');
38
40
  return { updated: false };
39
41
  } else if (next == null && active?.sync_rules_content == options.config.yaml) {
40
- logger.info('Sync rules from configuration unchanged');
42
+ logger.info('Sync config unchanged');
41
43
  return { updated: false };
42
44
  } else {
43
- logger.info('Sync rules updated from configuration');
45
+ logger.info('Sync config updated');
44
46
  const persisted_sync_rules = await this.updateSyncRules(options);
45
47
  return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
46
48
  }
47
49
  }
48
50
 
49
51
  /**
50
- * Get a storage instance to query sync data for specific sync rules.
52
+ * Get a storage instance to query sync data for specific sync config.
51
53
  */
52
54
  abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
53
55
 
54
56
  /**
55
- * Deploy new sync rules.
57
+ * Deploy new sync config.
56
58
  */
57
59
  abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
58
60
 
59
61
  /**
60
62
  * Indicate that a slot was removed, and we should re-sync by creating
61
- * a new sync rules instance.
63
+ * a new replication stream.
62
64
  *
63
65
  * This is roughly the same as deploying a new version of the current sync
64
- * rules, but also accounts for cases where the current sync rules are not
65
- * the latest ones.
66
+ * config, but also accounts for cases where the current sync config is not
67
+ * the latest one.
66
68
  *
67
69
  * Replication should be restarted after this.
68
70
  */
69
71
  abstract restartReplication(sync_rules_group_id: number): Promise<void>;
70
72
 
71
73
  /**
72
- * Get the sync rules used for querying.
74
+ * Get the sync config used for querying.
73
75
  */
74
76
  async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
75
77
  const content = await this.getActiveSyncRulesContent();
@@ -77,12 +79,12 @@ export abstract class BucketStorageFactory
77
79
  }
78
80
 
79
81
  /**
80
- * Get the sync rules used for querying.
82
+ * Get the sync config used for querying.
81
83
  */
82
84
  abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
83
85
 
84
86
  /**
85
- * Get the sync rules that will be active next once done with initial replicatino.
87
+ * Get the sync config that will be active next once done with initial replicatino.
86
88
  */
87
89
  async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
88
90
  const content = await this.getNextSyncRulesContent();
@@ -90,17 +92,17 @@ export abstract class BucketStorageFactory
90
92
  }
91
93
 
92
94
  /**
93
- * Get the sync rules that will be active next once done with initial replicatino.
95
+ * Get the sync config that will be active next once done with initial replicatino.
94
96
  */
95
97
  abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
96
98
 
97
99
  /**
98
- * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
100
+ * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
99
101
  */
100
102
  abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
101
103
 
102
104
  /**
103
- * Get all sync rules stopped but not terminated yet.
105
+ * Get all sync config stopped but not terminated yet.
104
106
  */
105
107
  abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
106
108
 
@@ -110,12 +112,12 @@ export abstract class BucketStorageFactory
110
112
  abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
111
113
 
112
114
  /**
113
- * Get storage size of active sync rules.
115
+ * Get storage size of active replication stream.
114
116
  */
115
117
  abstract getStorageMetrics(): Promise<StorageMetrics>;
116
118
 
117
119
  /**
118
- * Get the unique identifier for this instance of Powersync
120
+ * Get the unique identifier for this instance of Powersync.
119
121
  */
120
122
  abstract getPowerSyncInstanceId(): Promise<string>;
121
123
 
@@ -159,6 +161,12 @@ export interface UpdateSyncRulesOptions {
159
161
  * compiler.
160
162
  */
161
163
  plan: SerializedSyncPlan | null;
164
+
165
+ /**
166
+ * Parsed sync config, primarily to generate a definition mapping.
167
+ * Not persisted, and the defaultSchema used for parsing is not relevant.
168
+ */
169
+ parsed: SyncConfigWithErrors;
162
170
  };
163
171
  lock?: boolean;
164
172
  storageVersion?: number;
@@ -178,13 +186,14 @@ export interface SerializedSyncPlan {
178
186
  * them.
179
187
  */
180
188
  eventDescriptors: Record<string, string[]>;
189
+ errors?: ReplicationError[];
181
190
  }
182
191
 
183
192
  export function updateSyncRulesFromYaml(
184
193
  content: string,
185
194
  options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
186
195
  ): UpdateSyncRulesOptions {
187
- const { config } = SqlSyncRules.fromYaml(content, {
196
+ const config = SqlSyncRules.fromYaml(content, {
188
197
  // No schema-based validation at this point
189
198
  schema: undefined,
190
199
  defaultSchema: 'not_applicable', // Not needed for validation
@@ -195,24 +204,26 @@ export function updateSyncRulesFromYaml(
195
204
  }
196
205
 
197
206
  export function updateSyncRulesFromConfig(
198
- parsed: SyncConfig,
207
+ parsed: SyncConfigWithErrors,
199
208
  options?: Omit<UpdateSyncRulesOptions, 'config'>
200
209
  ): UpdateSyncRulesOptions {
201
210
  let plan: SerializedSyncPlan | null = null;
202
- if (parsed instanceof PrecompiledSyncConfig) {
211
+ const { config, errors } = parsed;
212
+ if (config instanceof PrecompiledSyncConfig) {
203
213
  const eventDescriptors: Record<string, string[]> = {};
204
- for (const event of parsed.eventDescriptors) {
214
+ for (const event of config.eventDescriptors) {
205
215
  eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
206
216
  }
207
217
 
208
218
  plan = {
209
- compatibility: parsed.compatibility.serialize(),
210
- plan: serializeSyncPlan(parsed.plan),
211
- eventDescriptors
219
+ compatibility: config.compatibility.serialize(),
220
+ plan: serializeSyncPlan(config.plan),
221
+ eventDescriptors,
222
+ errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
212
223
  };
213
224
  }
214
225
 
215
- return { config: { yaml: parsed.content, plan }, ...options };
226
+ return { config: { yaml: config.content, plan, parsed }, ...options };
216
227
  }
217
228
 
218
229
  export interface GetIntanceOptions {
@@ -1,6 +1,6 @@
1
1
  import { OrderedSet } from '@js-sdsl/ordered-set';
2
- import { LRUCache } from 'lru-cache/min';
3
2
  import { BucketDataSource } from '@powersync/service-sync-rules';
3
+ import { LRUCache } from 'lru-cache/min';
4
4
  import { BucketChecksum } from '../util/protocol-types.js';
5
5
  import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
6
6
  import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
@@ -1,9 +1,10 @@
1
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
1
+ import { logger as defaultLogger, ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
2
2
  import {
3
3
  CompatibilityContext,
4
4
  CompatibilityOption,
5
5
  DEFAULT_HYDRATION_STATE,
6
6
  deserializeSyncPlan,
7
+ ErrorLocation,
7
8
  HydratedSyncRules,
8
9
  HydrationState,
9
10
  javaScriptExpressionEngine,
@@ -11,7 +12,8 @@ import {
11
12
  SqlEventDescriptor,
12
13
  SqlSyncRules,
13
14
  SyncConfigWithErrors,
14
- versionedHydrationState
15
+ versionedHydrationState,
16
+ YamlError
15
17
  } from '@powersync/service-sync-rules';
16
18
  import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
17
19
  import { ReplicationLock } from './ReplicationLock.js';
@@ -27,7 +29,7 @@ export interface PersistedSyncRulesContentData {
27
29
  readonly compiled_plan: SerializedSyncPlan | null;
28
30
  readonly slot_name: string;
29
31
  /**
30
- * True if this is the "active" copy of the sync rules.
32
+ * True if this is the "active" copy of the sync config.
31
33
  */
32
34
  readonly active: boolean;
33
35
  readonly storageVersion: number;
@@ -47,6 +49,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
47
49
  readonly slot_name!: string;
48
50
  readonly active!: boolean;
49
51
  readonly storageVersion!: number;
52
+ readonly logger: Logger;
50
53
 
51
54
  readonly last_checkpoint_lsn!: string | null;
52
55
 
@@ -59,6 +62,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
59
62
 
60
63
  constructor(data: PersistedSyncRulesContentData) {
61
64
  Object.assign(this, data);
65
+ this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` });
62
66
  }
63
67
 
64
68
  /**
@@ -71,7 +75,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
71
75
  if (storageConfig == null) {
72
76
  throw new ServiceError(
73
77
  ErrorCode.PSYNC_S1005,
74
- `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
78
+ `Unsupported storage version ${this.storageVersion} for replication stream ${this.id}`
75
79
  );
76
80
  }
77
81
  return storageConfig;
@@ -101,7 +105,25 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
101
105
  sourceText: this.sync_rules_content
102
106
  });
103
107
 
104
- config = { config: precompiled, errors: [] };
108
+ // Note: If the original content did not define a storage version, this will still set the storage version.
109
+ // This means asUpdateOptions will not change the storage version, even if the default changes.
110
+ precompiled.storageVersion = this.storageVersion;
111
+
112
+ const errors: YamlError[] = [];
113
+ if (this.compiled_plan.errors) {
114
+ for (const error of this.compiled_plan.errors) {
115
+ const location: ErrorLocation | undefined = error.location && {
116
+ start: error.location.start_offset,
117
+ end: error.location.end_offset
118
+ };
119
+ const asYamlError = new YamlError(new Error(error.message), location);
120
+ asYamlError.type = error.level;
121
+
122
+ errors.push(asYamlError);
123
+ }
124
+ }
125
+
126
+ config = { config: precompiled, errors };
105
127
  } else {
106
128
  config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
107
129
  }
@@ -128,8 +150,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
128
150
  }
129
151
 
130
152
  asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
153
+ // defaultSchema is not relevant for the parsed version here
154
+ const parsed = this.parsed({ defaultSchema: 'not_applicable' });
131
155
  return {
132
- config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
156
+ config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules },
133
157
  ...options
134
158
  };
135
159
  }
@@ -1,6 +1,6 @@
1
1
  import * as sync_rules from '@powersync/service-sync-rules';
2
- import { SourceTable } from './SourceTable.js';
3
2
  import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
3
+ import { SourceTable } from './SourceTable.js';
4
4
 
5
5
  export type EventData = {
6
6
  op: SaveOp;
@@ -21,21 +21,21 @@ export interface ReportStorage extends AsyncDisposable {
21
21
  * This will return any short or long term connected clients.
22
22
  * Clients that have no disconnected_at timestamp and that have a valid jwt_exp timestamp are considered connected.
23
23
  */
24
- getCurrentConnections(): Promise<event_types.ClientConnectionReportResponse>;
24
+ getConnectedClients(): Promise<event_types.ClientConnectionReportResponse>;
25
25
  /**
26
26
  * Get a report of client connections over a day, week or month.
27
27
  * This is internally used to generate reports over it always returns the previous day, week or month.
28
28
  * Usually this is call on the start of the new day, week or month. It will return all unique completed connections
29
29
  * as well as uniques currently connected clients.
30
30
  */
31
- getClientConnectionsSummary(
31
+ getClientConnectionReports(
32
32
  data: event_types.ClientConnectionReportRequest
33
33
  ): Promise<event_types.ClientConnectionReportResponse>;
34
34
  /**
35
35
  * Get a paginated list of client connection events
36
36
  * This will return a paginated list of connections for a client/ user ID or all if neither is provided, within a date range if provided
37
37
  */
38
- getClientSessions(
38
+ getGeneralClientConnectionAnalytics(
39
39
  data: event_types.ClientConnectionAnalyticsRequest
40
40
  ): Promise<event_types.PaginatedResponse<event_types.ClientConnection>>;
41
41
  /**
@@ -1,7 +1,7 @@
1
1
  import { DEFAULT_TAG } from '@powersync/service-sync-rules';
2
+ import { bson } from '../index.js';
2
3
  import * as util from '../util/util-index.js';
3
4
  import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
4
- import { bson } from '../index.js';
5
5
 
6
6
  /**
7
7
  * Format of the id depends on the bucket storage module. It should be consistent within the module.
@@ -28,7 +28,7 @@ export class SourceTable implements SourceEntityDescriptor {
28
28
  static readonly DEFAULT_TAG = DEFAULT_TAG;
29
29
 
30
30
  /**
31
- * True if the table is used in sync rules for data queries.
31
+ * True if the table is used in sync config for data queries.
32
32
  *
33
33
  * This value is resolved externally, and cached here.
34
34
  *
@@ -37,7 +37,7 @@ export class SourceTable implements SourceEntityDescriptor {
37
37
  public syncData = true;
38
38
 
39
39
  /**
40
- * True if the table is used in sync rules for data queries.
40
+ * True if the table is used in sync config for data queries.
41
41
  *
42
42
  * This value is resolved externally, and cached here.
43
43
  *
@@ -46,7 +46,7 @@ export class SourceTable implements SourceEntityDescriptor {
46
46
  public syncParameters = true;
47
47
 
48
48
  /**
49
- * True if the table is used in sync rules for events.
49
+ * True if the table is used in sync config for events.
50
50
  *
51
51
  * This value is resolved externally, and cached here.
52
52
  *
@@ -45,7 +45,7 @@ export const STORAGE_VERSION_3 = 3;
45
45
  export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
46
46
 
47
47
  /**
48
- * Default storage version for newly persisted sync rules.
48
+ * Default storage version for newly persisted replication streams.
49
49
  */
50
50
  export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
51
51
 
@@ -2,9 +2,10 @@ import { Logger, ObserverClient } from '@powersync/lib-services-framework';
2
2
  import {
3
3
  BucketDataSource,
4
4
  HydratedSyncRules,
5
- ScopedParameterLookup,
6
- SqliteJsonRow
5
+ ParameterLookupRows,
6
+ ScopedParameterLookup
7
7
  } from '@powersync/service-sync-rules';
8
+ import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
8
9
  import * as util from '../util/util-index.js';
9
10
  import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
10
11
  import { BucketStorageFactory } from './BucketStorageFactory.js';
@@ -14,7 +15,7 @@ import { SourceTable } from './SourceTable.js';
14
15
  import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
15
16
 
16
17
  /**
17
- * Storage for a specific copy of sync rules.
18
+ * Storage for a specific replication stream.
18
19
  */
19
20
  export interface SyncRulesBucketStorage
20
21
  extends ObserverClient<SyncRulesBucketStorageListener>,
@@ -23,6 +24,7 @@ export interface SyncRulesBucketStorage
23
24
  readonly slot_name: string;
24
25
 
25
26
  readonly factory: BucketStorageFactory;
27
+ readonly logger: Logger;
26
28
 
27
29
  /**
28
30
  * Resolve a table, keeping track of it internally.
@@ -47,11 +49,11 @@ export interface SyncRulesBucketStorage
47
49
  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
48
50
 
49
51
  /**
50
- * Terminate the sync rules.
52
+ * Terminate the replication stream.
51
53
  *
52
54
  * This clears the storage, and sets state to TERMINATED.
53
55
  *
54
- * Must only be called on stopped sync rules.
56
+ * Must only be called on stopped replication streams.
55
57
  */
56
58
  terminate(options?: TerminateOptions): Promise<void>;
57
59
 
@@ -98,7 +100,7 @@ export interface SyncRulesBucketStorage
98
100
  /**
99
101
  * Yields the latest user write checkpoint whenever the sync checkpoint updates.
100
102
  *
101
- * The stream stops or errors if this is not the active sync rules (anymore).
103
+ * The stream stops or errors if this is not the active sync config (anymore).
102
104
  */
103
105
  watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable<StorageCheckpointUpdate>;
104
106
 
@@ -196,6 +198,8 @@ export interface CreateWriterOptions extends ParseSyncRulesOptions {
196
198
  */
197
199
  markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
198
200
 
201
+ tracer?: PerformanceTracer<'storage' | 'evaluate'>;
202
+
199
203
  logger?: Logger;
200
204
  }
201
205
 
@@ -265,6 +269,8 @@ export interface CompactOptions {
265
269
  compactParameterCacheLimit?: number;
266
270
 
267
271
  signal?: AbortSignal;
272
+
273
+ logger?: Logger;
268
274
  }
269
275
 
270
276
  export interface PopulateChecksumCacheOptions {
@@ -320,8 +326,41 @@ export interface ReplicationCheckpoint {
320
326
  * Used to resolve "dynamic" parameter queries.
321
327
  *
322
328
  * This gets parameter sets specific to this checkpoint.
329
+ *
330
+ * @throws {@link ParameterSetLimitExceededError}
331
+ * Thrown if resolved lookups in bucket storage exceed the `limit` parameter.
332
+ */
333
+ getParameterSets(lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
334
+ }
335
+
336
+ /**
337
+ * An exception thrown by {@link ReplicationCheckpoint} implementations if there are too many parameter results.
338
+ *
339
+ * This is not a suitable exception to show to users, `BucketParameterState` adds additional context.
340
+ */
341
+ export class ParameterSetLimitExceededError extends Error {
342
+ constructor(
343
+ readonly limit: number,
344
+ readonly breakdown?: ParameterQueryInvocationLog[]
345
+ ) {
346
+ super(`Too many parameter results (limit was ${limit})`);
347
+ }
348
+ }
349
+
350
+ export interface ParameterQueryInvocationLog {
351
+ /**
352
+ * The definition for which a parameter query was invoked.
353
+ *
354
+ * The exact format of definition is unspecified, it's shown to users to help them debug this failure.
355
+ */
356
+ definition: string;
357
+ /**
358
+ * If {@link didExceedLimit} is false, the amount of rows returned by the invocation.
359
+ *
360
+ * Otherwise, the maximum amount of rows this invocation was allowed to return.
323
361
  */
324
- getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]>;
362
+ resultsOrLimit: number;
363
+ didExceedLimit: boolean;
325
364
  }
326
365
 
327
366
  export interface WatchWriteCheckpointOptions {
@@ -1,7 +1,7 @@
1
1
  export enum WriteCheckpointMode {
2
2
  /**
3
3
  * Raw mappings of `user_id` to `write_checkpoint`s should
4
- * be supplied for each set of sync rules.
4
+ * be supplied for each replication stream.
5
5
  */
6
6
  CUSTOM = 'custom',
7
7
  /**
@@ -21,7 +21,7 @@ export interface BaseWriteCheckpointIdentifier {
21
21
 
22
22
  export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
23
23
  /**
24
- * Sync rules which were active when this checkpoint was created.
24
+ * Replication stream which was active when this checkpoint was created.
25
25
  */
26
26
  sync_rules_id: number;
27
27
  }
@@ -35,7 +35,7 @@ export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpoint
35
35
 
36
36
  export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
37
37
  /**
38
- * Sync rules which were active when this checkpoint was created.
38
+ * Replication stream which was active when this checkpoint was created.
39
39
  */
40
40
  sync_rules_id: number;
41
41
  }
@@ -63,8 +63,8 @@ export interface BaseWriteCheckpointAPI {
63
63
 
64
64
  /**
65
65
  * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
66
- * This storage corresponds with a set of sync rules. These APIs don't require specifying a
67
- * sync rules id.
66
+ * This storage corresponds with a replication stream. These APIs don't require specifying a
67
+ * replication stream id.
68
68
  */
69
69
  export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
70
70
  lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
@@ -72,7 +72,7 @@ export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
72
72
 
73
73
  /**
74
74
  * Write Checkpoint API which is interfaced directly with the storage layer. This requires
75
- * sync rules identifiers for custom write checkpoints.
75
+ * replication stream identifiers for custom write checkpoints.
76
76
  */
77
77
  export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
78
78
  lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
@@ -40,11 +40,6 @@ export const deserializeParameterLookup = (lookup: bson.Binary) => {
40
40
  return parsed;
41
41
  };
42
42
 
43
- export const getLookupBucketDefinitionName = (lookup: bson.Binary) => {
44
- const parsed = deserializeParameterLookup(lookup);
45
- return parsed[0] as string;
46
- };
47
-
48
43
  /**
49
44
  * True if this is a bson.UUID.
50
45
  *
@@ -1,17 +1,17 @@
1
1
  export * from './bson.js';
2
2
  export * from './BucketStorage.js';
3
+ export * from './BucketStorageBatch.js';
4
+ export * from './BucketStorageFactory.js';
3
5
  export * from './ChecksumCache.js';
6
+ export * from './PersistedSyncRulesContent.js';
4
7
  export * from './ReplicationEventPayload.js';
8
+ export * from './ReplicationLock.js';
9
+ export * from './ReportStorage.js';
5
10
  export * from './SourceEntity.js';
6
11
  export * from './SourceTable.js';
12
+ export * from './storage-metrics.js';
7
13
  export * from './StorageEngine.js';
8
14
  export * from './StorageProvider.js';
9
- export * from './storage-metrics.js';
10
- export * from './WriteCheckpointAPI.js';
11
- export * from './BucketStorageFactory.js';
12
- export * from './BucketStorageBatch.js';
13
- export * from './SyncRulesBucketStorage.js';
14
- export * from './PersistedSyncRulesContent.js';
15
- export * from './ReplicationLock.js';
16
- export * from './ReportStorage.js';
17
15
  export * from './StorageVersionConfig.js';
16
+ export * from './SyncRulesBucketStorage.js';
17
+ export * from './WriteCheckpointAPI.js';
@@ -1,7 +1,7 @@
1
- import { MetricsEngine } from '../metrics/MetricsEngine.js';
2
1
  import { logger } from '@powersync/lib-services-framework';
3
- import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
4
2
  import { StorageMetric } from '@powersync/service-types';
3
+ import { MetricsEngine } from '../metrics/MetricsEngine.js';
4
+ import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
5
5
 
6
6
  export function createCoreStorageMetrics(engine: MetricsEngine): void {
7
7
  engine.createObservableGauge({
@@ -1,3 +1,3 @@
1
- export * from './merge.js';
2
- export * from './LastValueSink.js';
3
1
  export * from './BroadcastIterable.js';
2
+ export * from './LastValueSink.js';
3
+ export * from './merge.js';