@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +152 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +36 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +30 -22
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/metrics/metrics-interfaces.d.ts +5 -1
  9. package/dist/metrics/metrics-interfaces.js.map +1 -1
  10. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
  11. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
  12. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
  13. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
  14. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
  15. package/dist/metrics/open-telemetry/util.js +7 -3
  16. package/dist/metrics/open-telemetry/util.js.map +1 -1
  17. package/dist/modules/AbstractModule.d.ts +6 -2
  18. package/dist/modules/AbstractModule.js.map +1 -1
  19. package/dist/replication/AbstractReplicator.d.ts +31 -7
  20. package/dist/replication/AbstractReplicator.js +168 -74
  21. package/dist/replication/AbstractReplicator.js.map +1 -1
  22. package/dist/replication/ReplicationModule.d.ts +3 -3
  23. package/dist/replication/ReplicationModule.js.map +1 -1
  24. package/dist/routes/RouterEngine.d.ts +1 -0
  25. package/dist/routes/RouterEngine.js +6 -1
  26. package/dist/routes/RouterEngine.js.map +1 -1
  27. package/dist/routes/configure-fastify.d.ts +104 -56
  28. package/dist/routes/configure-fastify.js +15 -2
  29. package/dist/routes/configure-fastify.js.map +1 -1
  30. package/dist/routes/endpoints/admin.d.ts +42 -24
  31. package/dist/routes/endpoints/admin.js +49 -36
  32. package/dist/routes/endpoints/admin.js.map +1 -1
  33. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  34. package/dist/routes/endpoints/checkpointing.js +65 -7
  35. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  36. package/dist/routes/endpoints/socket-route.js +3 -2
  37. package/dist/routes/endpoints/socket-route.js.map +1 -1
  38. package/dist/routes/endpoints/sync-rules.js +25 -14
  39. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  40. package/dist/routes/endpoints/sync-stream.js +1 -1
  41. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  42. package/dist/runner/teardown.js +8 -8
  43. package/dist/runner/teardown.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +64 -17
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/BucketStorageFactory.d.ts +50 -31
  47. package/dist/storage/BucketStorageFactory.js +10 -20
  48. package/dist/storage/BucketStorageFactory.js.map +1 -1
  49. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  50. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  51. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  52. package/dist/storage/ChecksumCache.d.ts +11 -6
  53. package/dist/storage/ChecksumCache.js +5 -4
  54. package/dist/storage/ChecksumCache.js.map +1 -1
  55. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  56. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  57. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  58. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  59. package/dist/storage/PersistedReplicationStream.js +26 -0
  60. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  61. package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
  62. package/dist/storage/PersistedSyncConfigContent.js +138 -0
  63. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  64. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  65. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  66. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  67. package/dist/storage/ReplicationEventPayload.d.ts +6 -1
  68. package/dist/storage/SourceEntity.d.ts +13 -2
  69. package/dist/storage/SourceTable.d.ts +53 -4
  70. package/dist/storage/SourceTable.js +62 -7
  71. package/dist/storage/SourceTable.js.map +1 -1
  72. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  73. package/dist/storage/SourceTableReconciler.js +95 -0
  74. package/dist/storage/SourceTableReconciler.js.map +1 -0
  75. package/dist/storage/StorageVersionConfig.d.ts +4 -0
  76. package/dist/storage/StorageVersionConfig.js +10 -1
  77. package/dist/storage/StorageVersionConfig.js.map +1 -1
  78. package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
  79. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  80. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  81. package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
  82. package/dist/storage/WriteCheckpointAPI.js +33 -0
  83. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  84. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
  85. package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
  86. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  87. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  88. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  89. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  90. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  91. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
  92. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  93. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  94. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  95. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  96. package/dist/storage/storage-index.d.ts +10 -1
  97. package/dist/storage/storage-index.js +10 -1
  98. package/dist/storage/storage-index.js.map +1 -1
  99. package/dist/storage/storage-metrics.js +120 -4
  100. package/dist/storage/storage-metrics.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +12 -2
  102. package/dist/sync/BucketChecksumState.js +69 -21
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.d.ts +7 -0
  105. package/dist/sync/RequestTracker.js +12 -0
  106. package/dist/sync/RequestTracker.js.map +1 -1
  107. package/dist/sync/sync.js +184 -43
  108. package/dist/sync/sync.js.map +1 -1
  109. package/dist/sync/util.d.ts +5 -0
  110. package/dist/sync/util.js +19 -1
  111. package/dist/sync/util.js.map +1 -1
  112. package/dist/system/ServiceContext.d.ts +2 -0
  113. package/dist/system/ServiceContext.js +2 -0
  114. package/dist/system/ServiceContext.js.map +1 -1
  115. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  116. package/dist/tracing/PerformanceTracer.js +11 -0
  117. package/dist/tracing/PerformanceTracer.js.map +1 -1
  118. package/dist/util/checkpointing.d.ts +4 -8
  119. package/dist/util/checkpointing.js +1 -16
  120. package/dist/util/checkpointing.js.map +1 -1
  121. package/dist/util/config/compound-config-collector.js +23 -2
  122. package/dist/util/config/compound-config-collector.js.map +1 -1
  123. package/dist/util/config/defaults.d.ts +2 -0
  124. package/dist/util/config/defaults.js +2 -0
  125. package/dist/util/config/defaults.js.map +1 -1
  126. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  127. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  128. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  130. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  131. package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
  132. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  133. package/dist/util/config/types.d.ts +4 -1
  134. package/dist/util/lsn.d.ts +4 -0
  135. package/dist/util/lsn.js +10 -0
  136. package/dist/util/lsn.js.map +1 -1
  137. package/dist/util/protocol-types.d.ts +4 -4
  138. package/dist/util/protocol-types.js +3 -2
  139. package/dist/util/protocol-types.js.map +1 -1
  140. package/dist/util/util-index.d.ts +1 -0
  141. package/dist/util/util-index.js +1 -0
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +3 -2
  144. package/dist/util/utils.js +17 -2
  145. package/dist/util/utils.js.map +1 -1
  146. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  147. package/dist/util/write-checkpoint-batcher.js +100 -0
  148. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  149. package/package.json +7 -7
  150. package/src/api/RouteAPI.ts +43 -14
  151. package/src/api/diagnostics.ts +45 -26
  152. package/src/entry/commands/compact-action.ts +37 -23
  153. package/src/metrics/metrics-interfaces.ts +6 -1
  154. package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
  155. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
  156. package/src/metrics/open-telemetry/util.ts +7 -3
  157. package/src/modules/AbstractModule.ts +6 -2
  158. package/src/replication/AbstractReplicator.ts +212 -78
  159. package/src/replication/ReplicationModule.ts +2 -3
  160. package/src/routes/RouterEngine.ts +7 -1
  161. package/src/routes/configure-fastify.ts +25 -2
  162. package/src/routes/endpoints/admin.ts +75 -51
  163. package/src/routes/endpoints/checkpointing.ts +79 -8
  164. package/src/routes/endpoints/socket-route.ts +3 -2
  165. package/src/routes/endpoints/sync-rules.ts +26 -14
  166. package/src/routes/endpoints/sync-stream.ts +1 -1
  167. package/src/runner/teardown.ts +8 -8
  168. package/src/storage/BucketStorageBatch.ts +65 -18
  169. package/src/storage/BucketStorageFactory.ts +73 -46
  170. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  171. package/src/storage/ChecksumCache.ts +27 -14
  172. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  173. package/src/storage/PersistedReplicationStream.ts +55 -0
  174. package/src/storage/PersistedSyncConfigContent.ts +207 -0
  175. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  176. package/src/storage/ReplicationEventPayload.ts +6 -1
  177. package/src/storage/SourceEntity.ts +12 -2
  178. package/src/storage/SourceTable.ts +96 -8
  179. package/src/storage/SourceTableReconciler.ts +188 -0
  180. package/src/storage/StorageVersionConfig.ts +11 -1
  181. package/src/storage/SyncRulesBucketStorage.ts +189 -27
  182. package/src/storage/WriteCheckpointAPI.ts +140 -6
  183. package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
  184. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  185. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
  186. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  187. package/src/storage/storage-index.ts +11 -1
  188. package/src/storage/storage-metrics.ts +154 -8
  189. package/src/sync/BucketChecksumState.ts +105 -21
  190. package/src/sync/RequestTracker.ts +20 -0
  191. package/src/sync/sync.ts +214 -50
  192. package/src/sync/util.ts +33 -8
  193. package/src/system/ServiceContext.ts +7 -0
  194. package/src/tracing/PerformanceTracer.ts +41 -1
  195. package/src/util/checkpointing.ts +4 -22
  196. package/src/util/config/compound-config-collector.ts +31 -1
  197. package/src/util/config/defaults.ts +2 -0
  198. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  199. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  200. package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
  201. package/src/util/config/types.ts +4 -1
  202. package/src/util/lsn.ts +9 -0
  203. package/src/util/protocol-types.ts +3 -2
  204. package/src/util/util-index.ts +1 -0
  205. package/src/util/utils.ts +16 -2
  206. package/src/util/write-checkpoint-batcher.ts +142 -0
  207. package/test/src/AbstractReplicator.test.ts +177 -0
  208. package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
  209. package/test/src/PersistedSyncConfigContent.test.ts +152 -0
  210. package/test/src/checksum_cache.test.ts +22 -0
  211. package/test/src/config.test.ts +142 -0
  212. package/test/src/diagnostics.test.ts +87 -34
  213. package/test/src/module-loader.test.ts +33 -1
  214. package/test/src/routes/admin.test.ts +188 -18
  215. package/test/src/routes/checkpointing.test.ts +54 -0
  216. package/test/src/routes/mocks.ts +6 -2
  217. package/test/src/routes/stream.test.ts +123 -2
  218. package/test/src/source-table-reconciler.test.ts +244 -0
  219. package/test/src/storage/SourceTable.test.ts +92 -0
  220. package/test/src/storage-metrics.test.ts +148 -0
  221. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  222. package/test/src/sync/util.test.ts +28 -1
  223. package/test/src/util/checkpointing.test.ts +249 -0
  224. package/test/src/util/utils.test.ts +23 -0
  225. package/tsconfig.json +1 -4
  226. package/tsconfig.tsbuildinfo +1 -1
  227. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  228. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  229. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  230. package/src/storage/PersistedSyncRulesContent.ts +0 -174
@@ -1,4 +1,5 @@
1
1
  import { logger, LookupOptions } from '@powersync/lib-services-framework';
2
+ import { validateStorageVersion } from '@powersync/service-sync-rules';
2
3
  import { configFile } from '@powersync/service-types';
3
4
  import * as auth from '../../auth/auth-index.js';
4
5
  import { ConfigCollector } from './collectors/config-collector.js';
@@ -6,6 +7,8 @@ import { Base64ConfigCollector } from './collectors/impl/base64-config-collector
6
7
  import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
7
8
  import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
8
9
  import {
10
+ DEFAULT_CHECKPOINT_REQUEST_RETENTION_MINUTES,
11
+ DEFAULT_CHECKSUM_CACHE_TTL_MINUTES,
9
12
  DEFAULT_MAX_BUCKETS_PER_CONNECTION,
10
13
  DEFAULT_MAX_CONCURRENT_CONNECTIONS,
11
14
  DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
@@ -60,6 +63,11 @@ export class CompoundConfigCollector {
60
63
  async collectConfig(runnerConfig: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
61
64
  const baseConfig = await this.collectBaseConfig(runnerConfig);
62
65
 
66
+ const defaultStorageVersion = baseConfig.storage?.default_storage_version;
67
+ if (defaultStorageVersion != null && validateStorageVersion(defaultStorageVersion) == null) {
68
+ throw new Error(`Storage version ${defaultStorageVersion} is not supported`);
69
+ }
70
+
63
71
  const dataSources = baseConfig.replication?.connections ?? [];
64
72
  if (dataSources.length > 1) {
65
73
  throw new Error('Only a single replication data source is supported currently');
@@ -193,7 +201,13 @@ export class CompoundConfigCollector {
193
201
  max_concurrent_connections:
194
202
  baseConfig.api?.parameters?.max_concurrent_connections ?? DEFAULT_MAX_CONCURRENT_CONNECTIONS,
195
203
  max_data_fetch_concurrency:
196
- baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY
204
+ baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
205
+ checkpoint_request_retention_minutes: normalizeCheckpointRequestRetentionMinutes(
206
+ baseConfig.api?.parameters?.checkpoint_request_retention_minutes
207
+ ),
208
+ bucket_count_cache_ttl_minutes: normalizeChecksumCacheTtlMinutes(
209
+ baseConfig.api?.parameters?.bucket_count_cache_ttl_minutes
210
+ )
197
211
  },
198
212
  // TODO maybe move this out of the connection or something
199
213
  slot_name_prefix: baseConfig.replication?.connections?.[0]?.slot_name_prefix ?? 'powersync_',
@@ -249,3 +263,19 @@ export class CompoundConfigCollector {
249
263
  };
250
264
  }
251
265
  }
266
+
267
+ function normalizeChecksumCacheTtlMinutes(ttlMinutes: number | undefined): number {
268
+ const normalized = ttlMinutes ?? DEFAULT_CHECKSUM_CACHE_TTL_MINUTES;
269
+ if (!Number.isFinite(normalized) || !Number.isInteger(normalized) || normalized < 1) {
270
+ throw new Error('api.parameters.bucket_count_cache_ttl_minutes must be a positive integer');
271
+ }
272
+ return normalized;
273
+ }
274
+
275
+ function normalizeCheckpointRequestRetentionMinutes(retentionMinutes: number | undefined): number {
276
+ const normalized = retentionMinutes ?? DEFAULT_CHECKPOINT_REQUEST_RETENTION_MINUTES;
277
+ if (!Number.isFinite(normalized) || !Number.isInteger(normalized) || normalized < 1) {
278
+ throw new Error('api.parameters.checkpoint_request_retention_minutes must be a positive integer');
279
+ }
280
+ return normalized;
281
+ }
@@ -3,3 +3,5 @@ export const DEFAULT_MAX_CONCURRENT_CONNECTIONS = 200;
3
3
  export const DEFAULT_MAX_DATA_FETCH_CONCURRENCY = 10;
4
4
  export const DEFAULT_MAX_BUCKETS_PER_CONNECTION = 1000;
5
5
  export const DEFAULT_MAX_PARAMETER_QUERY_RESULTS = 1000;
6
+ export const DEFAULT_CHECKPOINT_REQUEST_RETENTION_MINUTES = 60;
7
+ export const DEFAULT_CHECKSUM_CACHE_TTL_MINUTES = 1 * 60;
@@ -15,6 +15,7 @@ export class Base64SyncRulesCollector extends SyncRulesCollector {
15
15
 
16
16
  return {
17
17
  present: true,
18
+ version_label: baseConfig.sync_config?.version_label,
18
19
  exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
19
20
  content: Buffer.from(sync_config_base64, 'base64').toString()
20
21
  };
@@ -20,6 +20,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
20
20
  // Only persist the path here, and load on demand using `loadSyncRules()`.
21
21
  return {
22
22
  present: true,
23
+ version_label: baseConfig.sync_config?.version_label,
23
24
  exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
24
25
  path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
25
26
  };
@@ -5,6 +5,7 @@ export interface SyncRulesProvider {
5
5
  get(): Promise<string | undefined>;
6
6
 
7
7
  readonly exitOnError: boolean;
8
+ readonly versionLabel: string | undefined;
8
9
  }
9
10
 
10
11
  export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
@@ -21,4 +22,8 @@ export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
21
22
  get exitOnError() {
22
23
  return this.config.exit_on_error;
23
24
  }
25
+
26
+ get versionLabel() {
27
+ return this.config.version_label;
28
+ }
24
29
  }
@@ -22,6 +22,7 @@ export type Runner = (config: RunnerConfig) => Promise<void>;
22
22
 
23
23
  export type SyncRulesConfig = {
24
24
  present: boolean;
25
+ version_label?: string;
25
26
  content?: string;
26
27
  path?: string;
27
28
  exit_on_error: boolean;
@@ -53,9 +54,11 @@ export type ResolvedPowerSyncConfig = {
53
54
  max_data_fetch_concurrency: number;
54
55
  max_buckets_per_connection: number;
55
56
  max_parameter_query_results: number;
57
+ checkpoint_request_retention_minutes: number;
58
+ bucket_count_cache_ttl_minutes: number;
56
59
  };
57
60
 
58
- /** Prefix for postgres replication slot names. May eventually be connection-specific. */
61
+ /** Prefix for postgres replication slot names, as well as replication stream names. */
59
62
  slot_name_prefix: string;
60
63
 
61
64
  healthcheck: {
package/src/util/lsn.ts CHANGED
@@ -6,3 +6,12 @@ export function maxLsn(a: string | null | undefined, b: string | null | undefine
6
6
  if (b == null) return a;
7
7
  return a > b ? a : b;
8
8
  }
9
+
10
+ /**
11
+ * Return the smaller of two LSNs, ignoring nulls.
12
+ */
13
+ export function minLsn(a: string | null | undefined, b: string | null | undefined): string | null {
14
+ if (a == null) return b ?? null;
15
+ if (b == null) return a;
16
+ return a < b ? a : b;
17
+ }
@@ -1,5 +1,6 @@
1
1
  import { JsonContainer } from '@powersync/service-jsonbig';
2
2
  import { BucketPriority, SqliteJsonRow } from '@powersync/service-sync-rules';
3
+ import { orNull } from '@powersync/service-types';
3
4
  import * as t from 'ts-codec';
4
5
 
5
6
  export const BucketRequest = t.object({
@@ -24,14 +25,14 @@ export const RequestedStreamSubscription = t.object({
24
25
  /**
25
26
  * An optional dictionary of parameters to pass to this specific stream.
26
27
  */
27
- parameters: t.union(t.record(t.any), t.Null),
28
+ parameters: orNull(t.record(t.any)),
28
29
  /**
29
30
  * Set when the client wishes to re-assign a different priority to this stream.
30
31
  *
31
32
  * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
32
33
  * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
33
34
  */
34
- override_priority: t.union(t.number, t.Null)
35
+ override_priority: orNull(t.number)
35
36
  });
36
37
 
37
38
  export type RequestedStreamSubscription = t.Decoded<typeof RequestedStreamSubscription>;
@@ -8,6 +8,7 @@ export * from './protocol-types.js';
8
8
  export * from './secs.js';
9
9
  export * from './utils.js';
10
10
  export * from './version.js';
11
+ export * from './write-checkpoint-batcher.js';
11
12
 
12
13
  export * from './config.js';
13
14
  export * from './config/compound-config-collector.js';
package/src/util/utils.ts CHANGED
@@ -33,7 +33,7 @@ export interface PartialChecksum {
33
33
  */
34
34
  export type InternalOpId = bigint;
35
35
 
36
- export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
36
+ export const ID_NAMESPACE = uuid.parse('a396dd91-09fc-4017-a28d-3df722f651e9');
37
37
 
38
38
  export function escapeIdentifier(identifier: string) {
39
39
  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
@@ -223,7 +223,7 @@ export function isCompleteRow<V>(
223
223
  * All other operations are replaced with a single CLEAR operation,
224
224
  * summing their checksums, and using a 0 as an op_id.
225
225
  *
226
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
226
+ * This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
227
227
  *
228
228
  * Used for tests.
229
229
  */
@@ -309,3 +309,17 @@ export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefine
309
309
  }
310
310
  return size;
311
311
  }
312
+
313
+ export function formatBytes(bytes: number | bigint): string {
314
+ if (typeof bytes == 'bigint') {
315
+ bytes = Number(bytes); // We round either way
316
+ }
317
+ if (bytes >= 1024 * 1024 * 1024) {
318
+ return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
319
+ } else if (bytes >= 1024 * 1024) {
320
+ return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
321
+ } else if (bytes >= 1024) {
322
+ return `${(bytes / 1024).toFixed(1)}KB`;
323
+ }
324
+ return `${bytes}B`;
325
+ }
@@ -0,0 +1,142 @@
1
+ import { ErrorCode, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
2
+ import { RouteAPI } from '../api/RouteAPI.js';
3
+ import {
4
+ BucketStorageFactory,
5
+ CreateManagedWriteCheckpointsResult,
6
+ ManagedWriteCheckpointOptions,
7
+ SyncRulesBucketStorage
8
+ } from '../storage/storage-index.js';
9
+
10
+ // Keep up to three source-head/storage batches executing concurrently under load.
11
+ // There is intentionally no explicit batch-size cap here: the HTTP request queue
12
+ // already bounds how many requests can be waiting in this process.
13
+ // Smaller values here have better efficiency, while larger numbers here may give slight improvements in latency.
14
+ // We want to limit the number of database connections in use for write checkpoints, so we keep the numbers low here.
15
+ const MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES = 3;
16
+
17
+ export interface CreateWriteCheckpointResult {
18
+ writeCheckpoint: string;
19
+ replicationHead: string;
20
+ }
21
+
22
+ interface QueuedWriteCheckpoint {
23
+ userId: string;
24
+ checkpointRequestId: bigint | undefined;
25
+ resolvers: PromiseWithResolvers<CreateWriteCheckpointResult>;
26
+ }
27
+
28
+ export class WriteCheckpointBatcher {
29
+ private pending: QueuedWriteCheckpoint[] = [];
30
+ private inFlight = 0;
31
+ private scheduledPump: NodeJS.Timeout | undefined;
32
+
33
+ constructor(
34
+ private readonly getAPI: () => RouteAPI,
35
+ private readonly getStorage: () => BucketStorageFactory
36
+ ) {}
37
+
38
+ enqueue(userId: string, checkpointRequestId?: bigint): Promise<CreateWriteCheckpointResult> {
39
+ const resolvers = Promise.withResolvers<CreateWriteCheckpointResult>();
40
+ this.pending.push({ userId, checkpointRequestId, resolvers });
41
+ this.schedulePump();
42
+ return resolvers.promise;
43
+ }
44
+
45
+ private schedulePump() {
46
+ if (this.scheduledPump != null) {
47
+ return;
48
+ }
49
+
50
+ // Delay dispatch by one timer turn so requests arriving together can share
51
+ // one source head and one storage write batch.
52
+ // While normally we wouldn't have http requests arriving at the exact same time, this is relevant when
53
+ // requests are queued by fastify: All current requests are typically batched together, and finish at the
54
+ // same time. Then we get many requests from the queue arriving at the same time. Without this delayed dispatch,
55
+ // the first request would form its own batch, and the rest will then wait for a new batch to become available.
56
+ // The delayed dispatch allow all those requests to be batched together.
57
+ this.scheduledPump = setTimeout(() => {
58
+ this.scheduledPump = undefined;
59
+ this.pump();
60
+ }, 0);
61
+ }
62
+
63
+ private pump() {
64
+ while (this.inFlight < MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES && this.pending.length > 0) {
65
+ const batch = this.pending;
66
+ this.pending = [];
67
+ this.inFlight++;
68
+
69
+ void this.executeBatch(batch).finally(() => {
70
+ this.inFlight--;
71
+ this.schedulePump();
72
+ });
73
+ }
74
+ }
75
+
76
+ private async executeBatch(batch: QueuedWriteCheckpoint[]) {
77
+ try {
78
+ const syncBucketStorage = (await this.getStorage().getActiveSyncConfig())?.storage;
79
+ if (!syncBucketStorage) {
80
+ throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
81
+ }
82
+
83
+ // The source adapter reads the head, hands it to this callback to persist the
84
+ // write-checkpoint mapping, then forces a source marker only when storage
85
+ // reports an advance. Keeping the marker inside the callback means it is
86
+ // causally ordered after the head within a single source session.
87
+ const { writeCheckpoints, currentCheckpoint } = await this.getAPI().createReplicationHead(
88
+ async (currentCheckpoint) => {
89
+ const { writeCheckpoints, shouldAdvance } = await this.createBatchWriteCheckpoints(
90
+ syncBucketStorage,
91
+ batch,
92
+ currentCheckpoint
93
+ );
94
+ return { response: { writeCheckpoints, currentCheckpoint }, shouldAdvance };
95
+ }
96
+ );
97
+
98
+ const results = batch.map((request) => {
99
+ const writeCheckpoint = writeCheckpoints.get(request.userId);
100
+ if (writeCheckpoint == null) {
101
+ throw new ServiceAssertionError(
102
+ `Write checkpoint storage did not return a checkpoint for user ${request.userId}`
103
+ );
104
+ }
105
+
106
+ return { request, writeCheckpoint };
107
+ });
108
+
109
+ for (const { request, writeCheckpoint } of results) {
110
+ request.resolvers.resolve({
111
+ writeCheckpoint: String(writeCheckpoint),
112
+ replicationHead: currentCheckpoint
113
+ });
114
+ }
115
+ } catch (error) {
116
+ // Do not retry here. The route should pass failures through so clients can
117
+ // apply their normal write-checkpoint retry policy.
118
+ for (const request of batch) {
119
+ request.resolvers.reject(error);
120
+ }
121
+ }
122
+ }
123
+
124
+ private async createBatchWriteCheckpoints(
125
+ syncBucketStorage: SyncRulesBucketStorage,
126
+ batch: QueuedWriteCheckpoint[],
127
+ currentCheckpoint: string
128
+ ): Promise<CreateManagedWriteCheckpointsResult> {
129
+ return syncBucketStorage.createManagedWriteCheckpoints(
130
+ batch.map((request) => {
131
+ const checkpoint: ManagedWriteCheckpointOptions = {
132
+ user_id: request.userId,
133
+ heads: { '1': currentCheckpoint }
134
+ };
135
+ if (request.checkpointRequestId != null) {
136
+ checkpoint.checkpoint_request_id = request.checkpointRequestId;
137
+ }
138
+ return checkpoint;
139
+ })
140
+ );
141
+ }
142
+ }
@@ -0,0 +1,177 @@
1
+ import { AbstractReplicationJob } from '@/replication/AbstractReplicationJob.js';
2
+ import { AbstractReplicator, AbstractReplicatorOptions, CreateJobOptions } from '@/replication/AbstractReplicator.js';
3
+ import { PersistedReplicationStream } from '@/storage/PersistedReplicationStream.js';
4
+ import { SyncRulesBucketStorage } from '@/storage/SyncRulesBucketStorage.js';
5
+ import { describe, expect, it, vi } from 'vitest';
6
+
7
+ class TestReplicator extends AbstractReplicator {
8
+ constructor(
9
+ private readonly cleanup: (storage: SyncRulesBucketStorage) => Promise<void>,
10
+ options: AbstractReplicatorOptions = {
11
+ id: 'test',
12
+ storageEngine: {} as AbstractReplicatorOptions['storageEngine'],
13
+ metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
14
+ syncRuleProvider: {} as AbstractReplicatorOptions['syncRuleProvider'],
15
+ rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
16
+ }
17
+ ) {
18
+ super(options);
19
+ }
20
+
21
+ createJob(_options: CreateJobOptions): AbstractReplicationJob {
22
+ throw new Error('Not implemented');
23
+ }
24
+
25
+ cleanUp(storage: SyncRulesBucketStorage): Promise<void> {
26
+ return this.cleanup(storage);
27
+ }
28
+
29
+ async testConnection() {
30
+ return { connectionDescription: 'test' };
31
+ }
32
+
33
+ terminateStoppedStream(
34
+ replicationStream: PersistedReplicationStream,
35
+ syncRuleStorage: SyncRulesBucketStorage
36
+ ): Promise<void> {
37
+ return this.terminateStoppedReplicationStream(replicationStream, syncRuleStorage);
38
+ }
39
+
40
+ addClearingJob(replicationStreamId: number, promise: Promise<void>): void {
41
+ this.clearingJobs.set(replicationStreamId, promise);
42
+ }
43
+
44
+ get heartbeatIntervalNanosForTest(): bigint | null {
45
+ return (this as any).heartbeatIntervalNanos;
46
+ }
47
+
48
+ shouldHandleStreamForTest(
49
+ replicationStream: PersistedReplicationStream,
50
+ loadedSyncRules: string | undefined,
51
+ loadedVersionLabel: string | undefined
52
+ ): boolean {
53
+ return (this as any).shouldHandleReplicationStream(replicationStream, loadedSyncRules, loadedVersionLabel);
54
+ }
55
+ }
56
+
57
+ describe('AbstractReplicator heartbeat interval', () => {
58
+ const options: AbstractReplicatorOptions = {
59
+ id: 'test',
60
+ storageEngine: {} as AbstractReplicatorOptions['storageEngine'],
61
+ metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
62
+ syncRuleProvider: {} as AbstractReplicatorOptions['syncRuleProvider'],
63
+ rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
64
+ };
65
+
66
+ it.each([undefined, null])('uses the default for %s', (heartbeatIntervalSeconds) => {
67
+ const replicator = new TestReplicator(async () => {}, { ...options, heartbeatIntervalSeconds });
68
+
69
+ expect(replicator.heartbeatIntervalNanosForTest).toBe(60_000_000_000n);
70
+ });
71
+
72
+ it('disables the heartbeat interval with 0', () => {
73
+ const replicator = new TestReplicator(async () => {}, { ...options, heartbeatIntervalSeconds: 0 });
74
+
75
+ expect(replicator.heartbeatIntervalNanosForTest).toBeNull();
76
+ });
77
+
78
+ it('converts a positive heartbeat interval to nanoseconds', () => {
79
+ const replicator = new TestReplicator(async () => {}, { ...options, heartbeatIntervalSeconds: 5 });
80
+
81
+ expect(replicator.heartbeatIntervalNanosForTest).toBe(5_000_000_000n);
82
+ });
83
+ });
84
+
85
+ describe('AbstractReplicator stopped stream cleanup', () => {
86
+ it('holds the replication stream lock across source and storage cleanup', async () => {
87
+ const calls: string[] = [];
88
+ const release = vi.fn(async () => {
89
+ calls.push('release');
90
+ });
91
+ const replicationStream = {
92
+ async lock() {
93
+ calls.push('lock');
94
+ return { sync_rules_id: 1, release };
95
+ }
96
+ } as unknown as PersistedReplicationStream;
97
+ const syncRuleStorage = {
98
+ logger: { info: vi.fn() },
99
+ async terminate() {
100
+ calls.push('terminate');
101
+ }
102
+ } as unknown as SyncRulesBucketStorage;
103
+ const replicator = new TestReplicator(async () => {
104
+ calls.push('cleanup');
105
+ });
106
+
107
+ await replicator.terminateStoppedStream(replicationStream, syncRuleStorage);
108
+
109
+ expect(calls).toEqual(['lock', 'cleanup', 'terminate', 'release']);
110
+ expect(release).toHaveBeenCalledOnce();
111
+ });
112
+
113
+ it('releases the replication stream lock when cleanup fails', async () => {
114
+ const cleanupError = new Error('cleanup failed');
115
+ const release = vi.fn(async () => {});
116
+ const replicationStream = {
117
+ async lock() {
118
+ return { sync_rules_id: 1, release };
119
+ }
120
+ } as unknown as PersistedReplicationStream;
121
+ const terminate = vi.fn(async () => {});
122
+ const syncRuleStorage = {
123
+ logger: { info: vi.fn() },
124
+ terminate
125
+ } as unknown as SyncRulesBucketStorage;
126
+ const replicator = new TestReplicator(async () => {
127
+ throw cleanupError;
128
+ });
129
+
130
+ await expect(replicator.terminateStoppedStream(replicationStream, syncRuleStorage)).rejects.toBe(cleanupError);
131
+
132
+ expect(terminate).not.toHaveBeenCalled();
133
+ expect(release).toHaveBeenCalledOnce();
134
+ });
135
+
136
+ it('waits for stopped stream cleanup when stopping', async () => {
137
+ let finishCleanup: () => void;
138
+ const cleanup = new Promise<void>((resolve) => {
139
+ finishCleanup = resolve;
140
+ });
141
+ const replicator = new TestReplicator(async () => {});
142
+ replicator.addClearingJob(1, cleanup);
143
+
144
+ let stopped = false;
145
+ const stop = replicator.stop().then(() => {
146
+ stopped = true;
147
+ });
148
+ await Promise.resolve();
149
+ expect(stopped).toBe(false);
150
+
151
+ finishCleanup!();
152
+ await stop;
153
+ expect(stopped).toBe(true);
154
+ });
155
+ });
156
+
157
+ describe('AbstractReplicator rolling deployment config matching', () => {
158
+ const processingStream = (version_label: string | undefined): PersistedReplicationStream =>
159
+ ({
160
+ syncConfigContent: [
161
+ {
162
+ syncConfigState: 'PROCESSING',
163
+ sync_rules_content: 'bucket_definitions: {}',
164
+ version_label
165
+ }
166
+ ]
167
+ }) as unknown as PersistedReplicationStream;
168
+
169
+ it('requires both YAML and version label to match a processing config', () => {
170
+ const replicator = new TestReplicator(async () => {});
171
+ const stream = processingStream('v2');
172
+
173
+ expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v2')).toBe(true);
174
+ expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v1')).toBe(false);
175
+ expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', undefined)).toBe(false);
176
+ });
177
+ });
@@ -0,0 +1,96 @@
1
+ import { MetricAttributeFilteringExporter } from '@/metrics/open-telemetry/MetricAttributeFilteringExporter.js';
2
+ import { Attributes } from '@opentelemetry/api';
3
+ import { ExportResult } from '@opentelemetry/core';
4
+ import { DataPointType, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
5
+ import { describe, expect, it, vi } from 'vitest';
6
+
7
+ type ExportMetrics = (metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void) => void;
8
+
9
+ describe('MetricAttributeFilteringExporter', () => {
10
+ it('filters configured attributes without changing the collected metrics', () => {
11
+ const exportMetrics = vi.fn<ExportMetrics>();
12
+ const delegate = {
13
+ export: exportMetrics,
14
+ forceFlush: vi.fn(async () => {}),
15
+ shutdown: vi.fn(async () => {})
16
+ } satisfies PushMetricExporter;
17
+ const exporter = new MetricAttributeFilteringExporter(delegate, new Set(['version_label']));
18
+ const attributes: Attributes = {
19
+ sync_config_id: 'config-a',
20
+ sync_config_state: 'ACTIVE',
21
+ version_label: 'customer-version'
22
+ };
23
+ const metrics = {
24
+ resource: {},
25
+ scopeMetrics: [
26
+ {
27
+ scope: {},
28
+ metrics: [
29
+ {
30
+ dataPointType: DataPointType.GAUGE,
31
+ dataPoints: [{ attributes, value: 100 }]
32
+ }
33
+ ]
34
+ }
35
+ ]
36
+ } as unknown as ResourceMetrics;
37
+ const callback = vi.fn();
38
+
39
+ exporter.export(metrics, callback);
40
+
41
+ expect(exportMetrics).toHaveBeenCalledWith(
42
+ expect.objectContaining({
43
+ scopeMetrics: [
44
+ expect.objectContaining({
45
+ metrics: [
46
+ expect.objectContaining({
47
+ dataPoints: [
48
+ expect.objectContaining({
49
+ attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' }
50
+ })
51
+ ]
52
+ })
53
+ ]
54
+ })
55
+ ]
56
+ }),
57
+ callback
58
+ );
59
+ expect(metrics.scopeMetrics[0].metrics[0].dataPoints[0].attributes).toBe(attributes);
60
+ });
61
+
62
+ it('forwards rather than merges data points whose attributes become identical', () => {
63
+ const exportMetrics = vi.fn<ExportMetrics>();
64
+ const delegate = {
65
+ export: exportMetrics,
66
+ forceFlush: vi.fn(async () => {}),
67
+ shutdown: vi.fn(async () => {})
68
+ } satisfies PushMetricExporter;
69
+ const exporter = new MetricAttributeFilteringExporter(delegate, new Set(['filtered']));
70
+ const metrics = {
71
+ resource: {},
72
+ scopeMetrics: [
73
+ {
74
+ scope: {},
75
+ metrics: [
76
+ {
77
+ dataPointType: DataPointType.GAUGE,
78
+ dataPoints: [
79
+ { attributes: { retained: 'same', filtered: 'a' }, value: 10 },
80
+ { attributes: { retained: 'same', filtered: 'b' }, value: 20 }
81
+ ]
82
+ }
83
+ ]
84
+ }
85
+ ]
86
+ } as unknown as ResourceMetrics;
87
+
88
+ exporter.export(metrics, vi.fn());
89
+
90
+ const exported = exportMetrics.mock.calls[0][0];
91
+ expect(exported.scopeMetrics[0].metrics[0].dataPoints).toMatchObject([
92
+ { attributes: { retained: 'same' }, value: 10 },
93
+ { attributes: { retained: 'same' }, value: 20 }
94
+ ]);
95
+ });
96
+ });