@powersync/service-core 1.20.3 → 1.20.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/api/RouteAPI.d.ts +14 -0
  3. package/dist/api/api-index.d.ts +1 -1
  4. package/dist/api/api-index.js +1 -1
  5. package/dist/api/api-index.js.map +1 -1
  6. package/dist/api/api-metrics.js.map +1 -1
  7. package/dist/api/diagnostics.js +16 -14
  8. package/dist/api/diagnostics.js.map +1 -1
  9. package/dist/auth/CachedKeyCollector.js +1 -1
  10. package/dist/auth/CachedKeyCollector.js.map +1 -1
  11. package/dist/auth/CompoundKeyCollector.js.map +1 -1
  12. package/dist/auth/KeyStore.js.map +1 -1
  13. package/dist/auth/RemoteJWKSCollector.js.map +1 -1
  14. package/dist/auth/StaticKeyCollector.d.ts +1 -1
  15. package/dist/auth/StaticKeyCollector.js.map +1 -1
  16. package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
  17. package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
  18. package/dist/entry/commands/teardown-action.js +1 -1
  19. package/dist/entry/commands/teardown-action.js.map +1 -1
  20. package/dist/entry/entry-index.d.ts +1 -1
  21. package/dist/entry/entry-index.js +1 -1
  22. package/dist/entry/entry-index.js.map +1 -1
  23. package/dist/events/EventsEngine.js +1 -1
  24. package/dist/events/EventsEngine.js.map +1 -1
  25. package/dist/metrics/MetricsEngine.d.ts +1 -1
  26. package/dist/metrics/RollingBucketMax.d.ts +28 -0
  27. package/dist/metrics/RollingBucketMax.js +80 -0
  28. package/dist/metrics/RollingBucketMax.js.map +1 -0
  29. package/dist/metrics/metrics-index.d.ts +3 -2
  30. package/dist/metrics/metrics-index.js +3 -2
  31. package/dist/metrics/metrics-index.js.map +1 -1
  32. package/dist/metrics/open-telemetry/util.js +1 -1
  33. package/dist/metrics/open-telemetry/util.js.map +1 -1
  34. package/dist/metrics/register-metrics.js +2 -2
  35. package/dist/metrics/register-metrics.js.map +1 -1
  36. package/dist/modules/AbstractModule.d.ts +1 -1
  37. package/dist/modules/AbstractModule.js.map +1 -1
  38. package/dist/modules/modules-index.d.ts +1 -1
  39. package/dist/modules/modules-index.js +1 -1
  40. package/dist/modules/modules-index.js.map +1 -1
  41. package/dist/replication/AbstractReplicationJob.d.ts +2 -2
  42. package/dist/replication/AbstractReplicator.d.ts +1 -1
  43. package/dist/replication/AbstractReplicator.js +10 -7
  44. package/dist/replication/AbstractReplicator.js.map +1 -1
  45. package/dist/replication/ReplicationLagTracker.d.ts +50 -0
  46. package/dist/replication/ReplicationLagTracker.js +78 -0
  47. package/dist/replication/ReplicationLagTracker.js.map +1 -0
  48. package/dist/replication/replication-index.d.ts +3 -2
  49. package/dist/replication/replication-index.js +3 -2
  50. package/dist/replication/replication-index.js.map +1 -1
  51. package/dist/replication/replication-metrics.js.map +1 -1
  52. package/dist/routes/configure-fastify.d.ts +59 -32
  53. package/dist/routes/endpoints/admin.d.ts +108 -54
  54. package/dist/routes/endpoints/sync-rules.js +3 -3
  55. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  56. package/dist/routes/endpoints/sync-stream.d.ts +10 -10
  57. package/dist/routes/hooks.js +1 -1
  58. package/dist/routes/hooks.js.map +1 -1
  59. package/dist/routes/route-register.js.map +1 -1
  60. package/dist/storage/BucketStorageBatch.d.ts +1 -1
  61. package/dist/storage/BucketStorageFactory.d.ts +5 -3
  62. package/dist/storage/BucketStorageFactory.js +10 -8
  63. package/dist/storage/BucketStorageFactory.js.map +1 -1
  64. package/dist/storage/ChecksumCache.js.map +1 -1
  65. package/dist/storage/PersistedSyncRulesContent.js +14 -2
  66. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  67. package/dist/storage/ReplicationEventPayload.d.ts +1 -1
  68. package/dist/storage/SourceTable.d.ts +1 -1
  69. package/dist/storage/SourceTable.js.map +1 -1
  70. package/dist/storage/storage-index.d.ts +8 -8
  71. package/dist/storage/storage-index.js +8 -8
  72. package/dist/storage/storage-index.js.map +1 -1
  73. package/dist/storage/storage-metrics.js.map +1 -1
  74. package/dist/streams/streams-index.d.ts +2 -2
  75. package/dist/streams/streams-index.js +2 -2
  76. package/dist/streams/streams-index.js.map +1 -1
  77. package/dist/sync/BucketChecksumState.js +4 -19
  78. package/dist/sync/BucketChecksumState.js.map +1 -1
  79. package/dist/sync/RequestTracker.js +1 -1
  80. package/dist/sync/RequestTracker.js.map +1 -1
  81. package/dist/sync/sync-index.d.ts +2 -2
  82. package/dist/sync/sync-index.js +2 -2
  83. package/dist/sync/sync-index.js.map +1 -1
  84. package/dist/sync/sync.js.map +1 -1
  85. package/dist/sync/util.js.map +1 -1
  86. package/dist/system/ServiceContext.d.ts +1 -1
  87. package/dist/system/ServiceContext.js +1 -1
  88. package/dist/system/ServiceContext.js.map +1 -1
  89. package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
  90. package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
  91. package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
  92. package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
  93. package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
  94. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  95. package/dist/util/config.js +1 -1
  96. package/dist/util/config.js.map +1 -1
  97. package/dist/util/errors.d.ts +3 -0
  98. package/dist/util/errors.js +15 -0
  99. package/dist/util/errors.js.map +1 -0
  100. package/dist/util/protocol-types.d.ts +2 -2
  101. package/dist/util/util-index.d.ts +1 -1
  102. package/dist/util/util-index.js +1 -1
  103. package/dist/util/util-index.js.map +1 -1
  104. package/dist/util/utils.d.ts +1 -1
  105. package/package.json +6 -6
  106. package/src/api/RouteAPI.ts +17 -0
  107. package/src/api/api-index.ts +1 -1
  108. package/src/api/api-metrics.ts +1 -1
  109. package/src/api/diagnostics.ts +18 -19
  110. package/src/auth/CachedKeyCollector.ts +2 -3
  111. package/src/auth/CompoundKeyCollector.ts +2 -3
  112. package/src/auth/KeyStore.ts +1 -1
  113. package/src/auth/RemoteJWKSCollector.ts +0 -1
  114. package/src/auth/StaticKeyCollector.ts +1 -1
  115. package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
  116. package/src/entry/commands/teardown-action.ts +1 -1
  117. package/src/entry/entry-index.ts +1 -1
  118. package/src/events/EventsEngine.ts +1 -1
  119. package/src/metrics/MetricsEngine.ts +1 -1
  120. package/src/metrics/RollingBucketMax.ts +109 -0
  121. package/src/metrics/metrics-index.ts +3 -2
  122. package/src/metrics/open-telemetry/util.ts +1 -1
  123. package/src/metrics/register-metrics.ts +3 -3
  124. package/src/modules/AbstractModule.ts +1 -1
  125. package/src/modules/modules-index.ts +1 -1
  126. package/src/replication/AbstractReplicationJob.ts +2 -2
  127. package/src/replication/AbstractReplicator.ts +9 -7
  128. package/src/replication/ReplicationLagTracker.ts +86 -0
  129. package/src/replication/replication-index.ts +3 -2
  130. package/src/replication/replication-metrics.ts +1 -1
  131. package/src/routes/endpoints/sync-rules.ts +3 -5
  132. package/src/routes/hooks.ts +2 -2
  133. package/src/routes/route-register.ts +2 -10
  134. package/src/storage/BucketStorageBatch.ts +1 -1
  135. package/src/storage/BucketStorageFactory.ts +18 -14
  136. package/src/storage/ChecksumCache.ts +1 -1
  137. package/src/storage/PersistedSyncRulesContent.ts +18 -2
  138. package/src/storage/ReplicationEventPayload.ts +1 -1
  139. package/src/storage/SourceTable.ts +1 -1
  140. package/src/storage/storage-index.ts +8 -8
  141. package/src/storage/storage-metrics.ts +2 -2
  142. package/src/streams/streams-index.ts +2 -2
  143. package/src/sync/BucketChecksumState.ts +5 -21
  144. package/src/sync/RequestTracker.ts +1 -1
  145. package/src/sync/sync-index.ts +2 -2
  146. package/src/sync/sync.ts +1 -7
  147. package/src/sync/util.ts +1 -1
  148. package/src/system/ServiceContext.ts +1 -1
  149. package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
  150. package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
  151. package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
  152. package/src/util/config.ts +1 -1
  153. package/src/util/errors.ts +21 -0
  154. package/src/util/util-index.ts +1 -1
  155. package/src/util/utils.ts +1 -1
  156. package/test/src/ReplicationLagTracker.test.ts +53 -0
  157. package/test/src/RollingBucketMax.test.ts +106 -0
  158. package/test/src/auth.test.ts +7 -7
  159. package/test/src/module-loader.test.ts +1 -1
  160. package/test/src/routes/mocks.ts +1 -1
  161. package/test/src/routes/stream.test.ts +1 -2
  162. package/test/src/sync/BucketChecksumState.test.ts +2 -2
  163. package/test/src/util/protocol_types.test.ts +1 -1
  164. package/tsconfig.tsbuildinfo +1 -1
  165. package/vitest.config.ts +1 -1
@@ -0,0 +1,53 @@
1
+ import { ReplicationLagTracker } from '@/replication/ReplicationLagTracker.js';
2
+ import { describe, expect, it } from 'vitest';
3
+
4
+ describe('ReplicationLagTracker', () => {
5
+ it('returns undefined before replication has started', () => {
6
+ const tracker = new ReplicationLagTracker();
7
+
8
+ expect(tracker.getLagMillis(0)).toBeUndefined();
9
+ });
10
+
11
+ it('tracks the oldest in-flight change and returns current lag', () => {
12
+ const tracker = new ReplicationLagTracker();
13
+
14
+ tracker.trackUncommittedChange(new Date(1_000));
15
+ tracker.trackUncommittedChange(new Date(2_000));
16
+
17
+ expect(tracker.oldestUncommittedChange?.getTime()).toBe(1_000);
18
+ expect(tracker.getCurrentLagMillis(4_000)).toBe(3_000);
19
+ expect(tracker.getLagMillis(4_000)).toBe(3_000);
20
+ });
21
+
22
+ it('records commit lag into the rolling window and clears in-flight state', () => {
23
+ const tracker = new ReplicationLagTracker();
24
+
25
+ tracker.trackUncommittedChange(new Date(0));
26
+ tracker.markCommitted(5_000);
27
+
28
+ expect(tracker.oldestUncommittedChange).toBeNull();
29
+ expect(tracker.isStartingReplication).toBe(false);
30
+ expect(tracker.getLagMillis(5_000)).toBe(5_000);
31
+ expect(tracker.getLagMillis(35_000)).toBe(0);
32
+ });
33
+
34
+ it('can clear in-flight state without changing started state', () => {
35
+ const tracker = new ReplicationLagTracker();
36
+
37
+ tracker.trackUncommittedChange(new Date(0));
38
+ tracker.clearUncommittedChange();
39
+
40
+ expect(tracker.oldestUncommittedChange).toBeNull();
41
+ expect(tracker.isStartingReplication).toBe(true);
42
+ expect(tracker.getLagMillis(5_000)).toBeUndefined();
43
+ });
44
+
45
+ it('can mark replication as started without a committed transaction', () => {
46
+ const tracker = new ReplicationLagTracker();
47
+
48
+ tracker.markStarted();
49
+
50
+ expect(tracker.isStartingReplication).toBe(false);
51
+ expect(tracker.getLagMillis(0)).toBe(0);
52
+ });
53
+ });
@@ -0,0 +1,106 @@
1
+ import { RollingBucketMax } from '@/metrics/RollingBucketMax.js';
2
+ import { describe, expect, it } from 'vitest';
3
+
4
+ describe('RollingBucketMax', () => {
5
+ it('returns undefined before any values are reported', () => {
6
+ const tracker = new RollingBucketMax();
7
+
8
+ expect(tracker.getRollingMax(0)).toBeUndefined();
9
+ });
10
+
11
+ it('tracks the maximum value within a single bucket', () => {
12
+ const tracker = new RollingBucketMax();
13
+
14
+ tracker.report(3, 100);
15
+ tracker.report(9, 1_000);
16
+ tracker.report(5, 4_999);
17
+
18
+ expect(tracker.getRollingMax(4_999)).toBe(9);
19
+ });
20
+
21
+ it('keeps the rolling max across the last six 5s buckets', () => {
22
+ const tracker = new RollingBucketMax();
23
+
24
+ tracker.report(20, 0);
25
+ tracker.report(11, 5_000);
26
+ tracker.report(12, 10_000);
27
+ tracker.report(13, 15_000);
28
+ tracker.report(14, 20_000);
29
+ tracker.report(15, 25_000);
30
+
31
+ expect(tracker.getRollingMax(29_999)).toBe(20);
32
+ expect(tracker.getRollingMax(30_000)).toBe(15);
33
+ });
34
+
35
+ it('slides the rolling max forward as older buckets age out', () => {
36
+ const tracker = new RollingBucketMax();
37
+
38
+ tracker.report(20, 0);
39
+ expect(tracker.getRollingMax(0)).toBe(20);
40
+
41
+ tracker.report(18, 5_000);
42
+ expect(tracker.getRollingMax(5_000)).toBe(20);
43
+
44
+ tracker.report(16, 10_000);
45
+ expect(tracker.getRollingMax(10_000)).toBe(20);
46
+
47
+ tracker.report(14, 15_000);
48
+ expect(tracker.getRollingMax(15_000)).toBe(20);
49
+
50
+ tracker.report(12, 20_000);
51
+ expect(tracker.getRollingMax(20_000)).toBe(20);
52
+
53
+ tracker.report(10, 25_000);
54
+ expect(tracker.getRollingMax(29_999)).toBe(20);
55
+
56
+ tracker.report(8, 30_000);
57
+ expect(tracker.getRollingMax(30_000)).toBe(18);
58
+
59
+ tracker.report(6, 35_000);
60
+ expect(tracker.getRollingMax(35_000)).toBe(16);
61
+
62
+ tracker.report(4, 40_000);
63
+ expect(tracker.getRollingMax(40_000)).toBe(14);
64
+ });
65
+
66
+ it('keeps newer buckets in the rolling window while older peaks fall out', () => {
67
+ const tracker = new RollingBucketMax();
68
+
69
+ tracker.report(50, 0);
70
+ tracker.report(11, 5_000);
71
+ tracker.report(12, 10_000);
72
+ tracker.report(13, 15_000);
73
+ tracker.report(14, 20_000);
74
+ tracker.report(15, 25_000);
75
+
76
+ expect(tracker.getRollingMax(29_999)).toBe(50);
77
+
78
+ tracker.report(40, 30_000);
79
+ expect(tracker.getRollingMax(30_000)).toBe(40);
80
+ expect(tracker.getRollingMax(34_999)).toBe(40);
81
+ });
82
+
83
+ it('expires values after the rolling window passes with no new reports', () => {
84
+ const tracker = new RollingBucketMax();
85
+
86
+ tracker.report(7, 0);
87
+
88
+ expect(tracker.getRollingMax(29_999)).toBe(7);
89
+ expect(tracker.getRollingMax(30_000)).toBeUndefined();
90
+ });
91
+
92
+ it('supports custom bucket and window sizes', () => {
93
+ const tracker = new RollingBucketMax({
94
+ bucketSizeMs: 1_000,
95
+ windowSizeMs: 3_000
96
+ });
97
+
98
+ tracker.report(4, 0);
99
+ tracker.report(6, 1_000);
100
+ tracker.report(5, 2_000);
101
+
102
+ expect(tracker.getRollingMax(2_999)).toBe(6);
103
+ expect(tracker.getRollingMax(3_000)).toBe(6);
104
+ expect(tracker.getRollingMax(4_000)).toBe(5);
105
+ });
106
+ });
@@ -1,13 +1,13 @@
1
- import { describe, expect, test } from 'vitest';
2
- import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
1
+ import { StaticSupabaseKeyCollector } from '@/index.js';
3
2
  import * as jose from 'jose';
4
- import { KeyStore } from '../../src/auth/KeyStore.js';
3
+ import { describe, expect, test } from 'vitest';
4
+ import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
5
+ import { KeyResult } from '../../src/auth/KeyCollector.js';
5
6
  import { KeySpec } from '../../src/auth/KeySpec.js';
7
+ import { KeyStore } from '../../src/auth/KeyStore.js';
6
8
  import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
7
- import { KeyResult } from '../../src/auth/KeyCollector.js';
8
- import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
9
- import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
10
- import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
9
+ import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
10
+ import { getSupabaseJwksUrl } from '../../src/auth/utils.js';
11
11
 
12
12
  const publicKeyRSA: jose.JWK = {
13
13
  use: 'sig',
@@ -1,5 +1,5 @@
1
1
  import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js';
2
- import { describe, expect, it, vi } from 'vitest';
2
+ import { describe, expect, it } from 'vitest';
3
3
 
4
4
  interface MockConfig {
5
5
  connections?: { type: string }[];
@@ -1,3 +1,4 @@
1
+ import { EventsEngine } from '@/events/EventsEngine.js';
1
2
  import {
2
3
  BucketStorageFactory,
3
4
  createCoreAPIMetrics,
@@ -11,7 +12,6 @@ import {
11
12
  SyncRulesBucketStorage
12
13
  } from '@/index.js';
13
14
  import { MeterProvider } from '@opentelemetry/sdk-metrics';
14
- import { EventsEngine } from '@/events/EventsEngine.js';
15
15
 
16
16
  export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
17
17
  // This is very incomplete - just enough to get the current tests passing.
@@ -1,6 +1,6 @@
1
1
  import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
2
2
  import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
3
- import { SqlSyncRules } from '@powersync/service-sync-rules';
3
+ import { DEFAULT_HYDRATION_STATE, SqlSyncRules } from '@powersync/service-sync-rules';
4
4
  import { Readable, Writable } from 'stream';
5
5
  import { pipeline } from 'stream/promises';
6
6
  import { describe, expect, it } from 'vitest';
@@ -8,7 +8,6 @@ import winston from 'winston';
8
8
  import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
9
9
  import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
10
10
  import { mockServiceContext } from './mocks.js';
11
- import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
12
11
 
13
12
  describe('Stream Route', () => {
14
13
  describe('compressed stream', () => {
@@ -15,13 +15,12 @@ import {
15
15
  import { JSONBig } from '@powersync/service-jsonbig';
16
16
  import {
17
17
  ParameterIndexLookupCreator,
18
- RequestJwtPayload,
19
18
  ScopedParameterLookup,
19
+ SourceTableInterface,
20
20
  SqliteJsonRow,
21
21
  SqliteRow,
22
22
  SqlSyncRules,
23
23
  TablePattern,
24
- SourceTableInterface,
25
24
  versionedHydrationState
26
25
  } from '@powersync/service-sync-rules';
27
26
  import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
@@ -986,6 +985,7 @@ bucket_definitions:
986
985
  expect(errorMessages[0]).toContain('tasks: 20');
987
986
  expect(errorMessages[0]).toContain('comments: 10');
988
987
 
988
+ expect(errorData[0].checkpoint).toEqual(1n);
989
989
  expect(errorData[0].parameter_query_results).toBe(60);
990
990
  expect(errorData[0].parameter_query_results_by_definition).toEqual({
991
991
  projects: 30,
@@ -1,6 +1,6 @@
1
1
  import { StreamingSyncRequest } from '@/index.js';
2
2
  import { schema } from '@powersync/lib-services-framework';
3
- import { describe, test, expect, it } from 'vitest';
3
+ import { describe, expect, test } from 'vitest';
4
4
 
5
5
  describe('protocol types', () => {
6
6
  describe('StreamingSyncRequest', () => {