@powersync/service-core 1.20.4 → 1.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/dist/api/RouteAPI.d.ts +17 -3
  3. package/dist/api/api-index.d.ts +1 -1
  4. package/dist/api/api-index.js +1 -1
  5. package/dist/api/api-index.js.map +1 -1
  6. package/dist/api/api-metrics.js.map +1 -1
  7. package/dist/api/diagnostics.d.ts +1 -1
  8. package/dist/api/diagnostics.js +32 -14
  9. package/dist/api/diagnostics.js.map +1 -1
  10. package/dist/auth/CachedKeyCollector.js +1 -1
  11. package/dist/auth/CachedKeyCollector.js.map +1 -1
  12. package/dist/auth/CompoundKeyCollector.js.map +1 -1
  13. package/dist/auth/KeyStore.js.map +1 -1
  14. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  15. package/dist/auth/StaticKeyCollector.d.ts +1 -1
  16. package/dist/auth/StaticKeyCollector.js.map +1 -1
  17. package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
  18. package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
  19. package/dist/entry/commands/teardown-action.js +2 -2
  20. package/dist/entry/commands/teardown-action.js.map +1 -1
  21. package/dist/entry/entry-index.d.ts +1 -1
  22. package/dist/entry/entry-index.js +1 -1
  23. package/dist/entry/entry-index.js.map +1 -1
  24. package/dist/events/EventsEngine.js +1 -1
  25. package/dist/events/EventsEngine.js.map +1 -1
  26. package/dist/index.d.ts +1 -0
  27. package/dist/index.js +1 -0
  28. package/dist/index.js.map +1 -1
  29. package/dist/metrics/MetricsEngine.d.ts +1 -1
  30. package/dist/metrics/metrics-index.d.ts +3 -3
  31. package/dist/metrics/metrics-index.js +3 -3
  32. package/dist/metrics/metrics-index.js.map +1 -1
  33. package/dist/metrics/open-telemetry/util.js +1 -1
  34. package/dist/metrics/open-telemetry/util.js.map +1 -1
  35. package/dist/metrics/register-metrics.js +2 -2
  36. package/dist/metrics/register-metrics.js.map +1 -1
  37. package/dist/modules/AbstractModule.d.ts +2 -2
  38. package/dist/modules/AbstractModule.js.map +1 -1
  39. package/dist/modules/modules-index.d.ts +1 -1
  40. package/dist/modules/modules-index.js +1 -1
  41. package/dist/modules/modules-index.js.map +1 -1
  42. package/dist/replication/AbstractReplicationJob.d.ts +1 -1
  43. package/dist/replication/AbstractReplicationJob.js +1 -1
  44. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  45. package/dist/replication/AbstractReplicator.d.ts +6 -6
  46. package/dist/replication/AbstractReplicator.js +21 -21
  47. package/dist/replication/AbstractReplicator.js.map +1 -1
  48. package/dist/replication/replication-index.d.ts +3 -3
  49. package/dist/replication/replication-index.js +3 -3
  50. package/dist/replication/replication-index.js.map +1 -1
  51. package/dist/replication/replication-metrics.js.map +1 -1
  52. package/dist/routes/configure-fastify.d.ts +59 -32
  53. package/dist/routes/endpoints/admin.d.ts +108 -54
  54. package/dist/routes/endpoints/admin.js +7 -3
  55. package/dist/routes/endpoints/admin.js.map +1 -1
  56. package/dist/routes/endpoints/checkpointing.js +1 -1
  57. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  58. package/dist/routes/endpoints/socket-route.js +1 -1
  59. package/dist/routes/endpoints/socket-route.js.map +1 -1
  60. package/dist/routes/endpoints/sync-rules.js +10 -10
  61. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  62. package/dist/routes/endpoints/sync-stream.d.ts +10 -10
  63. package/dist/routes/endpoints/sync-stream.js +2 -2
  64. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  65. package/dist/routes/hooks.js +1 -1
  66. package/dist/routes/hooks.js.map +1 -1
  67. package/dist/routes/route-register.js.map +1 -1
  68. package/dist/runner/teardown.js +4 -4
  69. package/dist/runner/teardown.js.map +1 -1
  70. package/dist/storage/BucketStorage.d.ts +9 -9
  71. package/dist/storage/BucketStorage.js +9 -9
  72. package/dist/storage/BucketStorageBatch.d.ts +1 -1
  73. package/dist/storage/BucketStorageFactory.d.ts +27 -20
  74. package/dist/storage/BucketStorageFactory.js +19 -16
  75. package/dist/storage/BucketStorageFactory.js.map +1 -1
  76. package/dist/storage/ChecksumCache.js.map +1 -1
  77. package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
  78. package/dist/storage/PersistedSyncRulesContent.js +24 -5
  79. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  80. package/dist/storage/ReplicationEventPayload.d.ts +1 -1
  81. package/dist/storage/SourceTable.d.ts +4 -4
  82. package/dist/storage/SourceTable.js +3 -3
  83. package/dist/storage/SourceTable.js.map +1 -1
  84. package/dist/storage/StorageVersionConfig.d.ts +1 -1
  85. package/dist/storage/StorageVersionConfig.js +1 -1
  86. package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
  87. package/dist/storage/SyncRulesBucketStorage.js +14 -0
  88. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  89. package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
  90. package/dist/storage/WriteCheckpointAPI.js +1 -1
  91. package/dist/storage/bson.d.ts +0 -1
  92. package/dist/storage/bson.js +0 -4
  93. package/dist/storage/bson.js.map +1 -1
  94. package/dist/storage/storage-index.d.ts +8 -8
  95. package/dist/storage/storage-index.js +8 -8
  96. package/dist/storage/storage-index.js.map +1 -1
  97. package/dist/storage/storage-metrics.js.map +1 -1
  98. package/dist/streams/streams-index.d.ts +2 -2
  99. package/dist/streams/streams-index.js +2 -2
  100. package/dist/streams/streams-index.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +2 -5
  102. package/dist/sync/BucketChecksumState.js +119 -75
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.js +1 -1
  105. package/dist/sync/RequestTracker.js.map +1 -1
  106. package/dist/sync/sync-index.d.ts +2 -2
  107. package/dist/sync/sync-index.js +2 -2
  108. package/dist/sync/sync-index.js.map +1 -1
  109. package/dist/sync/sync.js.map +1 -1
  110. package/dist/sync/util.js.map +1 -1
  111. package/dist/system/ServiceContext.d.ts +1 -1
  112. package/dist/system/ServiceContext.js +1 -1
  113. package/dist/system/ServiceContext.js.map +1 -1
  114. package/dist/tracing/PerformanceTracer.d.ts +44 -0
  115. package/dist/tracing/PerformanceTracer.js +102 -0
  116. package/dist/tracing/PerformanceTracer.js.map +1 -0
  117. package/dist/tracing/TraceWriter.d.ts +22 -0
  118. package/dist/tracing/TraceWriter.js +63 -0
  119. package/dist/tracing/TraceWriter.js.map +1 -0
  120. package/dist/util/checkpointing.js +1 -1
  121. package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
  122. package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
  123. package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
  124. package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
  125. package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
  126. package/dist/util/config/compound-config-collector.d.ts +1 -1
  127. package/dist/util/config/compound-config-collector.js +2 -2
  128. package/dist/util/config/compound-config-collector.js.map +1 -1
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
  130. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  131. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  132. package/dist/util/config.js +1 -1
  133. package/dist/util/config.js.map +1 -1
  134. package/dist/util/env.js +1 -1
  135. package/dist/util/errors.d.ts +3 -0
  136. package/dist/util/errors.js +15 -0
  137. package/dist/util/errors.js.map +1 -0
  138. package/dist/util/protocol-types.d.ts +3 -3
  139. package/dist/util/protocol-types.js +1 -1
  140. package/dist/util/util-index.d.ts +1 -1
  141. package/dist/util/util-index.js +1 -1
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +1 -1
  144. package/package.json +11 -11
  145. package/src/api/RouteAPI.ts +20 -3
  146. package/src/api/api-index.ts +1 -1
  147. package/src/api/api-metrics.ts +1 -1
  148. package/src/api/diagnostics.ts +42 -20
  149. package/src/auth/CachedKeyCollector.ts +2 -3
  150. package/src/auth/CompoundKeyCollector.ts +2 -3
  151. package/src/auth/KeyStore.ts +1 -1
  152. package/src/auth/RemoteJWKSCollector.ts +0 -1
  153. package/src/auth/StaticKeyCollector.ts +1 -1
  154. package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
  155. package/src/entry/commands/teardown-action.ts +2 -2
  156. package/src/entry/entry-index.ts +1 -1
  157. package/src/events/EventsEngine.ts +1 -1
  158. package/src/index.ts +2 -0
  159. package/src/metrics/MetricsEngine.ts +1 -1
  160. package/src/metrics/metrics-index.ts +3 -3
  161. package/src/metrics/open-telemetry/util.ts +1 -1
  162. package/src/metrics/register-metrics.ts +3 -3
  163. package/src/modules/AbstractModule.ts +2 -2
  164. package/src/modules/modules-index.ts +1 -1
  165. package/src/replication/AbstractReplicationJob.ts +2 -2
  166. package/src/replication/AbstractReplicator.ts +23 -23
  167. package/src/replication/replication-index.ts +3 -3
  168. package/src/replication/replication-metrics.ts +1 -1
  169. package/src/routes/endpoints/admin.ts +7 -3
  170. package/src/routes/endpoints/checkpointing.ts +1 -1
  171. package/src/routes/endpoints/socket-route.ts +1 -1
  172. package/src/routes/endpoints/sync-rules.ts +10 -12
  173. package/src/routes/endpoints/sync-stream.ts +2 -2
  174. package/src/routes/hooks.ts +2 -2
  175. package/src/routes/route-register.ts +2 -10
  176. package/src/runner/teardown.ts +4 -4
  177. package/src/storage/BucketStorage.ts +9 -9
  178. package/src/storage/BucketStorageBatch.ts +1 -1
  179. package/src/storage/BucketStorageFactory.ts +45 -34
  180. package/src/storage/ChecksumCache.ts +1 -1
  181. package/src/storage/PersistedSyncRulesContent.ts +30 -6
  182. package/src/storage/ReplicationEventPayload.ts +1 -1
  183. package/src/storage/SourceTable.ts +4 -4
  184. package/src/storage/StorageVersionConfig.ts +1 -1
  185. package/src/storage/SyncRulesBucketStorage.ts +46 -7
  186. package/src/storage/WriteCheckpointAPI.ts +6 -6
  187. package/src/storage/bson.ts +0 -5
  188. package/src/storage/storage-index.ts +8 -8
  189. package/src/storage/storage-metrics.ts +2 -2
  190. package/src/streams/streams-index.ts +2 -2
  191. package/src/sync/BucketChecksumState.ts +141 -93
  192. package/src/sync/RequestTracker.ts +1 -1
  193. package/src/sync/sync-index.ts +2 -2
  194. package/src/sync/sync.ts +2 -8
  195. package/src/sync/util.ts +1 -1
  196. package/src/system/ServiceContext.ts +1 -1
  197. package/src/tracing/PerformanceTracer.ts +126 -0
  198. package/src/tracing/TraceWriter.ts +67 -0
  199. package/src/util/checkpointing.ts +1 -1
  200. package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
  201. package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
  202. package/src/util/config/compound-config-collector.ts +3 -3
  203. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
  204. package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
  205. package/src/util/config.ts +1 -1
  206. package/src/util/env.ts +1 -1
  207. package/src/util/errors.ts +21 -0
  208. package/src/util/protocol-types.ts +1 -1
  209. package/src/util/util-index.ts +1 -1
  210. package/src/util/utils.ts +1 -1
  211. package/test/src/auth.test.ts +115 -7
  212. package/test/src/diagnostics.test.ts +151 -0
  213. package/test/src/module-loader.test.ts +1 -1
  214. package/test/src/routes/mocks.ts +1 -1
  215. package/test/src/routes/stream.test.ts +1 -2
  216. package/test/src/sync/BucketChecksumState.test.ts +223 -67
  217. package/test/src/util/protocol_types.test.ts +1 -1
  218. package/test/tsconfig.json +0 -1
  219. package/tsconfig.tsbuildinfo +1 -1
  220. package/vitest.config.ts +1 -1
@@ -14,6 +14,16 @@ export interface ReplicationLagOptions {
14
14
  bucketStorage: SyncRulesBucketStorage;
15
15
  }
16
16
 
17
+ export interface SlotWalBudgetInfo {
18
+ wal_status: string;
19
+ safe_wal_size?: number;
20
+ max_slot_wal_keep_size?: number;
21
+ }
22
+
23
+ export interface SlotWalBudgetOptions {
24
+ slotName: string;
25
+ }
26
+
17
27
  /**
18
28
  * Describes all the methods currently required to service the sync API endpoints.
19
29
  */
@@ -33,7 +43,7 @@ export interface RouteAPI {
33
43
  * Generates replication table information from a given pattern of tables.
34
44
  *
35
45
  * @param tablePatterns A set of table patterns which typically come from
36
- * the tables listed in sync rules definitions.
46
+ * the tables listed in sync config definitions.
37
47
  *
38
48
  * @param sqlSyncRules
39
49
  * @returns A result of all the tables and columns which should be replicated
@@ -49,6 +59,13 @@ export interface RouteAPI {
49
59
  */
50
60
  getReplicationLagBytes(options: ReplicationLagOptions): Promise<number | undefined>;
51
61
 
62
+ /**
63
+ * @returns WAL budget information for the replication slot, or undefined
64
+ * if the slot doesn't exist or the source doesn't support this.
65
+ * Only implemented by the Postgres adapter.
66
+ */
67
+ getSlotWalBudget?(options: SlotWalBudgetOptions): Promise<SlotWalBudgetInfo | undefined>;
68
+
52
69
  /**
53
70
  * Get the current LSN or equivalent replication HEAD position identifier.
54
71
  *
@@ -59,7 +76,7 @@ export interface RouteAPI {
59
76
 
60
77
  /**
61
78
  * @returns The schema for tables inside the connected database. This is typically
62
- * used to validate sync rules.
79
+ * used to validate sync config.
63
80
  */
64
81
  getConnectionSchema(): Promise<types.DatabaseSchema[]>;
65
82
 
@@ -75,7 +92,7 @@ export interface RouteAPI {
75
92
  shutdown(): Promise<void>;
76
93
 
77
94
  /**
78
- * Get the default schema (or database) when only a table name is specified in sync rules.
95
+ * Get the default schema (or database) when only a table name is specified in sync config.
79
96
  */
80
97
  getParseSyncRulesOptions(): ParseSyncRulesOptions;
81
98
  }
@@ -1,4 +1,4 @@
1
+ export * from './api-metrics.js';
1
2
  export * from './diagnostics.js';
2
3
  export * from './RouteAPI.js';
3
4
  export * from './schema.js';
4
- export * from './api-metrics.js';
@@ -1,5 +1,5 @@
1
- import { MetricsEngine } from '../metrics/MetricsEngine.js';
2
1
  import { APIMetric } from '@powersync/service-types';
2
+ import { MetricsEngine } from '../metrics/MetricsEngine.js';
3
3
 
4
4
  /**
5
5
  * Create and register the core API metrics.
@@ -1,13 +1,14 @@
1
1
  import { logger } from '@powersync/lib-services-framework';
2
- import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules, SyncConfigWithErrors } from '@powersync/service-sync-rules';
2
+ import { DEFAULT_TAG, SourceTableInterface, SyncConfigWithErrors } from '@powersync/service-sync-rules';
3
3
  import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types';
4
4
 
5
5
  import * as storage from '../storage/storage-index.js';
6
- import { RouteAPI } from './RouteAPI.js';
6
+ import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
7
+ import { RouteAPI, SlotWalBudgetInfo } from './RouteAPI.js';
7
8
 
8
9
  export interface DiagnosticsOptions {
9
10
  /**
10
- * Include sync rules content in response.
11
+ * Include sync config content in response.
11
12
  */
12
13
  include_content?: boolean;
13
14
 
@@ -62,6 +63,7 @@ export async function getSyncRulesStatus(
62
63
  const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
63
64
  const status = await systemStorage?.getStatus();
64
65
  let replication_lag_bytes: number | undefined = undefined;
66
+ let slot_wal_budget: SlotWalBudgetInfo | undefined = undefined;
65
67
 
66
68
  let tables_flat: TableInfo[] = [];
67
69
 
@@ -87,6 +89,16 @@ export async function getSyncRulesStatus(
87
89
  // Ignore
88
90
  logger.warn(`Unable to get replication lag`, e);
89
91
  }
92
+
93
+ if (apiHandler.getSlotWalBudget && sync_rules.slot_name) {
94
+ try {
95
+ slot_wal_budget = await apiHandler.getSlotWalBudget({
96
+ slotName: sync_rules.slot_name
97
+ });
98
+ } catch (e) {
99
+ logger.warn(`Unable to get WAL budget`, e);
100
+ }
101
+ }
90
102
  }
91
103
  } else {
92
104
  const source_table_patterns = rules.getSourceTables();
@@ -131,26 +143,33 @@ export async function getSyncRulesStatus(
131
143
  ts: sync_rules.last_fatal_error_ts?.toISOString()
132
144
  });
133
145
  }
134
- errors.push(
135
- ...syncRuleErrors.map(({ type, message, location }) => {
136
- const error: ReplicationError = {
137
- level: type,
138
- message,
146
+ errors.push(...syncRuleErrors.map((error) => syncConfigYamlErrorToReplicationError(error, now)));
147
+
148
+ if (
149
+ slot_wal_budget &&
150
+ slot_wal_budget.wal_status !== 'lost' &&
151
+ slot_wal_budget.safe_wal_size != null &&
152
+ slot_wal_budget.max_slot_wal_keep_size != null &&
153
+ slot_wal_budget.max_slot_wal_keep_size > 0
154
+ ) {
155
+ const budgetPct = Math.max(
156
+ 0,
157
+ Math.round((slot_wal_budget.safe_wal_size / slot_wal_budget.max_slot_wal_keep_size) * 100)
158
+ );
159
+ if (budgetPct <= 50) {
160
+ errors.push({
161
+ level: 'warning',
162
+ message:
163
+ `WAL budget is low: ${budgetPct}% remaining. ` +
164
+ `The replication slot may be invalidated if WAL consumption ` +
165
+ `continues at this rate. Consider increasing max_slot_wal_keep_size.`,
139
166
  ts: now
140
- };
141
- if (location != null) {
142
- error.location = {
143
- start_offset: location.start,
144
- end_offset: location.end
145
- };
146
- }
147
-
148
- return error;
149
- })
150
- );
167
+ });
168
+ }
169
+ }
151
170
 
152
171
  if (live_status && status?.active) {
153
- // Check replication lag for active sync rules.
172
+ // Check replication lag for active replication stream.
154
173
  // Right now we exclude mysql, since it we don't have consistent keepalives for it.
155
174
  if (sync_rules.last_checkpoint_ts == null && sync_rules.last_keepalive_ts == null) {
156
175
  errors.push({
@@ -197,6 +216,9 @@ export async function getSyncRulesStatus(
197
216
  last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
198
217
  last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
199
218
  replication_lag_bytes: replication_lag_bytes,
219
+ wal_status: slot_wal_budget?.wal_status,
220
+ safe_wal_size: slot_wal_budget?.safe_wal_size,
221
+ max_slot_wal_keep_size: slot_wal_budget?.max_slot_wal_keep_size,
200
222
  tables: tables_flat
201
223
  }
202
224
  ],
@@ -1,9 +1,8 @@
1
- import * as jose from 'jose';
1
+ import { AuthorizationError, ErrorCode, logger } from '@powersync/lib-services-framework';
2
2
  import timers from 'timers/promises';
3
+ import { KeyCollector, KeyResult } from './KeyCollector.js';
3
4
  import { KeySpec } from './KeySpec.js';
4
5
  import { LeakyBucket } from './LeakyBucket.js';
5
- import { KeyCollector, KeyResult } from './KeyCollector.js';
6
- import { AuthorizationError, ErrorCode, logger } from '@powersync/lib-services-framework';
7
6
  import { mapAuthConfigError } from './utils.js';
8
7
 
9
8
  /**
@@ -1,7 +1,6 @@
1
- import * as jose from 'jose';
2
- import { KeySpec } from './KeySpec.js';
3
- import { KeyCollector, KeyResult } from './KeyCollector.js';
4
1
  import { AuthorizationError } from '@powersync/lib-services-framework';
2
+ import { KeyCollector, KeyResult } from './KeyCollector.js';
3
+ import { KeySpec } from './KeySpec.js';
5
4
 
6
5
  export class CompoundKeyCollector implements KeyCollector {
7
6
  private collectors: KeyCollector[];
@@ -4,7 +4,7 @@ import secs from '../util/secs.js';
4
4
  import { JwtPayload } from './JwtPayload.js';
5
5
  import { KeyCollector } from './KeyCollector.js';
6
6
  import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
7
- import { debugKeyNotFound, mapAuthError, SupabaseAuthDetails, tokenDebugDetails } from './utils.js';
7
+ import { debugKeyNotFound, mapAuthError, SupabaseAuthDetails } from './utils.js';
8
8
 
9
9
  /**
10
10
  * KeyStore to get keys and verify tokens.
@@ -1,6 +1,5 @@
1
1
  import * as http from 'http';
2
2
  import * as https from 'https';
3
- import * as jose from 'jose';
4
3
  import fetch from 'node-fetch';
5
4
 
6
5
  import {
@@ -1,6 +1,6 @@
1
1
  import * as jose from 'jose';
2
- import { KeySpec } from './KeySpec.js';
3
2
  import { KeyCollector, KeyResult } from './KeyCollector.js';
3
+ import { KeySpec } from './KeySpec.js';
4
4
 
5
5
  /**
6
6
  * Set of static keys.
@@ -1,6 +1,6 @@
1
1
  import * as jose from 'jose';
2
- import { KeySpec, KeyOptions } from './KeySpec.js';
3
2
  import { KeyCollector, KeyResult } from './KeyCollector.js';
3
+ import { KeyOptions, KeySpec } from './KeySpec.js';
4
4
 
5
5
  export const SUPABASE_KEY_OPTIONS: KeyOptions = {
6
6
  requiresAudience: ['authenticated'],
@@ -1,8 +1,8 @@
1
1
  import { Command } from 'commander';
2
2
 
3
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
3
4
  import { teardown } from '../../runner/teardown.js';
4
5
  import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
5
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
6
6
 
7
7
  const COMMAND_NAME = 'teardown';
8
8
 
@@ -13,7 +13,7 @@ export function registerTearDownAction(program: Command) {
13
13
 
14
14
  return teardownCommand
15
15
  .argument('[ack]', 'Type `TEARDOWN` to confirm teardown should occur')
16
- .description('Terminate all replicating sync rules, clear remote configuration and remove all data')
16
+ .description('Terminate all replicating replication streams, clear remote configuration and remove all data')
17
17
  .action(async (ack, options) => {
18
18
  if (ack !== 'TEARDOWN') {
19
19
  throw new ServiceError(ErrorCode.PSYNC_S0102, 'TEARDOWN was not acknowledged.');
@@ -1,6 +1,6 @@
1
1
  export * from './cli-entry.js';
2
+ export * from './commands/compact-action.js';
2
3
  export * from './commands/config-command.js';
3
4
  export * from './commands/migrate-action.js';
4
5
  export * from './commands/start-action.js';
5
6
  export * from './commands/teardown-action.js';
6
- export * from './commands/compact-action.js';
@@ -1,6 +1,6 @@
1
- import EventEmitter from 'node:events';
2
1
  import { logger } from '@powersync/lib-services-framework';
3
2
  import { event_types } from '@powersync/service-types';
3
+ import EventEmitter from 'node:events';
4
4
 
5
5
  export class EventsEngine {
6
6
  private emitter: EventEmitter;
package/src/index.ts CHANGED
@@ -42,4 +42,6 @@ export * as utils from './util/util-index.js';
42
42
  export * from './streams/streams-index.js';
43
43
  export * as streams from './streams/streams-index.js';
44
44
 
45
+ export * from './tracing/PerformanceTracer.js';
46
+
45
47
  export * as bson from 'bson';
@@ -1,5 +1,5 @@
1
1
  import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
2
- import { Counter, UpDownCounter, ObservableGauge, MetricMetadata, MetricsFactory } from './metrics-interfaces.js';
2
+ import { Counter, MetricMetadata, MetricsFactory, ObservableGauge, UpDownCounter } from './metrics-interfaces.js';
3
3
 
4
4
  export interface MetricsEngineOptions {
5
5
  factory: MetricsFactory;
@@ -1,6 +1,6 @@
1
- export * from './MetricsEngine.js';
2
1
  export * from './metrics-interfaces.js';
3
- export * from './RollingBucketMax.js';
4
- export * from './register-metrics.js';
2
+ export * from './MetricsEngine.js';
5
3
  export * from './open-telemetry/OpenTelemetryMetricsFactory.js';
6
4
  export * from './open-telemetry/util.js';
5
+ export * from './register-metrics.js';
6
+ export * from './RollingBucketMax.js';
@@ -6,8 +6,8 @@ import { ServiceContext } from '../../system/ServiceContext.js';
6
6
  import { MetricsFactory } from '../metrics-interfaces.js';
7
7
  import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
8
8
 
9
- import pkg from '../../../package.json' with { type: 'json' };
10
9
  import { resourceFromAttributes } from '@opentelemetry/resources';
10
+ import pkg from '../../../package.json' with { type: 'json' };
11
11
 
12
12
  export function createOpenTelemetryMetricsFactory(context: ServiceContext): MetricsFactory {
13
13
  const { configuration, lifeCycleEngine, storageEngine } = context;
@@ -1,9 +1,9 @@
1
- import { ServiceContextContainer } from '../system/ServiceContext.js';
2
- import { createOpenTelemetryMetricsFactory } from './open-telemetry/util.js';
3
- import { MetricsEngine } from './MetricsEngine.js';
4
1
  import { createCoreAPIMetrics, initializeCoreAPIMetrics } from '../api/api-metrics.js';
5
2
  import { createCoreReplicationMetrics, initializeCoreReplicationMetrics } from '../replication/replication-metrics.js';
6
3
  import { createCoreStorageMetrics, initializeCoreStorageMetrics } from '../storage/storage-metrics.js';
4
+ import { ServiceContextContainer } from '../system/ServiceContext.js';
5
+ import { MetricsEngine } from './MetricsEngine.js';
6
+ import { createOpenTelemetryMetricsFactory } from './open-telemetry/util.js';
7
7
 
8
8
  export enum MetricModes {
9
9
  API = 'api',
@@ -1,11 +1,11 @@
1
- import { ServiceContextContainer } from '../system/ServiceContext.js';
2
1
  import { logger } from '@powersync/lib-services-framework';
3
2
  import winston from 'winston';
4
3
  import { PersistedSyncRulesContent } from '../storage/storage-index.js';
4
+ import { ServiceContextContainer } from '../system/ServiceContext.js';
5
5
 
6
6
  export interface TearDownOptions {
7
7
  /**
8
- * If required, tear down any configuration/state for the specific sync rules
8
+ * If required, tear down any configuration/state for the specific replication stream
9
9
  */
10
10
  syncRules?: PersistedSyncRulesContent[];
11
11
  }
@@ -1,3 +1,3 @@
1
- export * from './ModuleManager.js';
2
1
  export * from './AbstractModule.js';
3
2
  export * from './loader.js';
3
+ export * from './ModuleManager.js';
@@ -1,8 +1,8 @@
1
1
  import { container, logger } from '@powersync/lib-services-framework';
2
2
  import winston from 'winston';
3
+ import { MetricsEngine } from '../metrics/MetricsEngine.js';
3
4
  import * as storage from '../storage/storage-index.js';
4
5
  import { ErrorRateLimiter } from './ErrorRateLimiter.js';
5
- import { MetricsEngine } from '../metrics/MetricsEngine.js';
6
6
 
7
7
  export interface AbstractReplicationJobOptions {
8
8
  id: string;
@@ -54,7 +54,7 @@ export abstract class AbstractReplicationJob {
54
54
  * Safely stop the replication process
55
55
  */
56
56
  public async stop(): Promise<void> {
57
- this.logger.info(`Stopping replication job for sync rule iteration: ${this.storage.group_id}`);
57
+ this.logger.info(`Stopping replication job`);
58
58
  this.abortController.abort();
59
59
  await this.isReplicatingPromise;
60
60
  }
@@ -32,19 +32,19 @@ export interface AbstractReplicatorOptions {
32
32
  /**
33
33
  * A replicator manages the mechanics for replicating data from a data source to a storage bucket.
34
34
  * This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
35
- * It also handles any changes to the sync rules.
35
+ * It also handles any changes to the sync config.
36
36
  */
37
37
  export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
38
38
  protected logger: winston.Logger;
39
39
  private lockAlerted: boolean = false;
40
40
  /**
41
- * Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
42
- * transitioning to a new set of sync rules.
41
+ * Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
42
+ * transitioning to a new replication stream.
43
43
  */
44
44
  private replicationJobs = new Map<number, T>();
45
45
 
46
46
  /**
47
- * Map of sync rule ids to promises that are clearing the sync rule configuration.
47
+ * Map of replciation stream ids to promises that are clearing the replication stream.
48
48
  *
49
49
  * We primarily do this to keep track of what we're currently clearing, but don't currently
50
50
  * use the Promise value.
@@ -68,8 +68,8 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
68
68
  abstract createJob(options: CreateJobOptions): T;
69
69
 
70
70
  /**
71
- * Clean up any configuration or state for the specified sync rule on the datasource.
72
- * Should be a no-op if the configuration has already been cleared
71
+ * Clean up any configuration or state for the specified replication stream on the datasource.
72
+ * Should be a no-op if the replication stream has already been cleared
73
73
  */
74
74
  abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
75
75
 
@@ -100,7 +100,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
100
100
  public async start(): Promise<void> {
101
101
  this.abortController = new AbortController();
102
102
  this.runLoop().catch((e) => {
103
- this.logger.error('Data source fatal replication error', e);
103
+ this.logger.error('Fatal replication error', e);
104
104
  container.reporter.captureException(e);
105
105
  setTimeout(() => {
106
106
  process.exit(1);
@@ -135,9 +135,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
135
135
 
136
136
  let configuredLock: storage.ReplicationLock | undefined = undefined;
137
137
  if (syncRules != null) {
138
- this.logger.info('Loaded sync rules');
138
+ this.logger.info('Loaded sync config');
139
139
  try {
140
- // Configure new sync rules, if they have changed.
140
+ // Configure new sync config, if they have changed.
141
141
  // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
142
142
 
143
143
  const { lock } = await this.storage.configureSyncRules(
@@ -149,11 +149,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
149
149
  } catch (e) {
150
150
  // Log and re-raise to exit.
151
151
  // Should only reach this due to validation errors if exit_on_error is true.
152
- this.logger.error(`Failed to update sync rules from configuration`, e);
152
+ this.logger.error(`Failed to update sync config`, e);
153
153
  throw e;
154
154
  }
155
155
  } else {
156
- this.logger.info('No sync rules configured - configure via API');
156
+ this.logger.info('No sync streams or rules configured - configure via API');
157
157
  }
158
158
  while (!this.stopped) {
159
159
  await container.probes.touch();
@@ -206,7 +206,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
206
206
  // Remove from the list. Next refresh call will restart the job.
207
207
  existingJobs.delete(syncRules.id);
208
208
  } else {
209
- // New sync rules were found (or resume after restart)
209
+ // New sync config was found (or resume after restart)
210
210
  try {
211
211
  let lock: storage.ReplicationLock;
212
212
  if (configuredLock?.sync_rules_id == syncRules.id) {
@@ -229,15 +229,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
229
229
  } catch (e) {
230
230
  if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
231
231
  if (!this.lockAlerted) {
232
- this.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
232
+ syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
233
233
  this.lockAlerted = true;
234
234
  }
235
235
  } else {
236
- // Could be a sync rules parse error,
236
+ // Could be a sync config parse error,
237
237
  // for example from stricter validation that was added.
238
238
  // This will be retried every couple of seconds.
239
- // When new (valid) sync rules are deployed and processed, this one be disabled.
240
- this.logger.error('Failed to start replication for new sync rules', e);
239
+ // When new (valid) sync config is deployed and processed, this one be disabled.
240
+ syncRules.logger.error('Failed to start replication for new sync config', e);
241
241
  }
242
242
  }
243
243
  }
@@ -246,7 +246,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
246
246
  this.replicationJobs = newJobs;
247
247
  this.activeReplicationJob = activeJob;
248
248
 
249
- // Stop any orphaned jobs that no longer have sync rules.
249
+ // Stop any orphaned jobs that no longer have a replication stream.
250
250
  // Termination happens below
251
251
  for (let job of existingJobs.values()) {
252
252
  // Old - stop and clean up
@@ -254,11 +254,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
254
254
  await job.stop();
255
255
  } catch (e) {
256
256
  // This will be retried
257
- this.logger.warn('Failed to stop old replication job}', e);
257
+ job.storage.logger.warn('Failed to stop old replication job', e);
258
258
  }
259
259
  }
260
260
 
261
- // Sync rules stopped previously, including by a different process.
261
+ // Replication stream stopped previously, including by a different process.
262
262
  const stopped = await this.storage.getStoppedSyncRules();
263
263
  for (let syncRules of stopped) {
264
264
  if (this.clearingJobs.has(syncRules.id)) {
@@ -268,11 +268,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
268
268
 
269
269
  // We clear storage asynchronously.
270
270
  // It is important to be able to continue running the refresh loop, otherwise we cannot
271
- // retry locked sync rules, for example.
271
+ // retry locked replication stream, for example.
272
272
  const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
273
273
  const promise = this.terminateSyncRules(syncRuleStorage)
274
274
  .catch((e) => {
275
- this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
275
+ syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
276
276
  })
277
277
  .finally(() => {
278
278
  this.clearingJobs.delete(syncRules.id);
@@ -286,12 +286,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
286
286
  }
287
287
 
288
288
  protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
289
- this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
289
+ syncRuleStorage.logger.info(`Terminating replication stream...`);
290
290
  // This deletes postgres replication slots - should complete quickly.
291
291
  // It is safe to do before or after clearing the data in the storage.
292
292
  await this.cleanUp(syncRuleStorage);
293
293
  await syncRuleStorage.terminate({ signal: this.abortController?.signal, clearStorage: true });
294
- this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
294
+ syncRuleStorage.logger.info(`Successfully terminated replication stream`);
295
295
  }
296
296
 
297
297
  abstract testConnection(): Promise<ConnectionTestResult>;
@@ -1,8 +1,8 @@
1
1
  export * from './AbstractReplicationJob.js';
2
2
  export * from './AbstractReplicator.js';
3
3
  export * from './ErrorRateLimiter.js';
4
- export * from './ReplicationLagTracker.js';
4
+ export * from './RelationCache.js';
5
+ export * from './replication-metrics.js';
5
6
  export * from './ReplicationEngine.js';
7
+ export * from './ReplicationLagTracker.js';
6
8
  export * from './ReplicationModule.js';
7
- export * from './replication-metrics.js';
8
- export * from './RelationCache.js';
@@ -1,5 +1,5 @@
1
- import { MetricsEngine } from '../metrics/metrics-index.js';
2
1
  import { ReplicationMetric } from '@powersync/service-types';
2
+ import { MetricsEngine } from '../metrics/metrics-index.js';
3
3
 
4
4
  /**
5
5
  * Create and register the core replication metrics.
@@ -119,7 +119,7 @@ export const reprocess = routeDefinition({
119
119
  const apiHandler = service_context.routerEngine.getAPI();
120
120
  const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
121
121
  if (next != null) {
122
- throw new Error(`Busy processing sync rules - cannot reprocess`);
122
+ throw new Error(`Busy processing sync config - cannot reprocess`);
123
123
  }
124
124
 
125
125
  const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
@@ -127,13 +127,17 @@ export const reprocess = routeDefinition({
127
127
  throw new errors.ServiceError({
128
128
  status: 422,
129
129
  code: ErrorCode.PSYNC_S4104,
130
- description: 'No active sync rules'
130
+ description: 'No active sync config'
131
131
  });
132
132
  }
133
133
 
134
+ // There are some differences between this and using asUpdateOptions():
135
+ // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
136
+ // 2. If the source does not set the storage version, this will update it do the current version.
137
+ // We can consider tweaking this behavior in the future.
134
138
  const new_rules = await activeBucketStorage.updateSyncRules(
135
139
  storage.updateSyncRulesFromYaml(active.sync_rules.config.content, {
136
- // These sync rules already passed validation. But if the rules are not valid anymore due
140
+ // This sync config already passed validation. But if the config is not valid anymore due
137
141
  // to a service change, we do want to report the error here.
138
142
  validate: true
139
143
  })
@@ -33,7 +33,7 @@ export const writeCheckpoint = routeDefinition({
33
33
  const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
34
34
  const cp = await bucketStorage?.getCheckpoint();
35
35
  if (cp == null) {
36
- throw new Error('No sync rules available');
36
+ throw new Error('No sync config available');
37
37
  }
38
38
  if (cp.lsn && cp.lsn >= head) {
39
39
  logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);
@@ -77,7 +77,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
77
77
  new errors.ServiceError({
78
78
  status: 500,
79
79
  code: ErrorCode.PSYNC_S2302,
80
- description: 'No sync rules available'
80
+ description: 'No sync config available'
81
81
  })
82
82
  );
83
83
  responder.onComplete();