@powersync/service-core 1.20.3 → 1.20.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/api/RouteAPI.d.ts +14 -0
- package/dist/api/api-index.d.ts +1 -1
- package/dist/api/api-index.js +1 -1
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.js +16 -14
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/CachedKeyCollector.js +1 -1
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.d.ts +1 -1
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +1 -1
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -1
- package/dist/entry/entry-index.js +1 -1
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/events/EventsEngine.js +1 -1
- package/dist/events/EventsEngine.js.map +1 -1
- package/dist/metrics/MetricsEngine.d.ts +1 -1
- package/dist/metrics/RollingBucketMax.d.ts +28 -0
- package/dist/metrics/RollingBucketMax.js +80 -0
- package/dist/metrics/RollingBucketMax.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +3 -2
- package/dist/metrics/metrics-index.js +3 -2
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +1 -1
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/metrics/register-metrics.js +2 -2
- package/dist/metrics/register-metrics.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +1 -1
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/modules-index.d.ts +1 -1
- package/dist/modules/modules-index.js +1 -1
- package/dist/modules/modules-index.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -2
- package/dist/replication/AbstractReplicator.d.ts +1 -1
- package/dist/replication/AbstractReplicator.js +10 -7
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationLagTracker.d.ts +50 -0
- package/dist/replication/ReplicationLagTracker.js +78 -0
- package/dist/replication/ReplicationLagTracker.js.map +1 -0
- package/dist/replication/replication-index.d.ts +3 -2
- package/dist/replication/replication-index.js +3 -2
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/replication/replication-metrics.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +59 -32
- package/dist/routes/endpoints/admin.d.ts +108 -54
- package/dist/routes/endpoints/sync-rules.js +3 -3
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -10
- package/dist/routes/hooks.js +1 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +5 -3
- package/dist/storage/BucketStorageFactory.js +10 -8
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.js +14 -2
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/SourceTable.d.ts +1 -1
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/storage-index.d.ts +8 -8
- package/dist/storage/storage-index.js +8 -8
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/streams/streams-index.d.ts +2 -2
- package/dist/streams/streams-index.js +2 -2
- package/dist/streams/streams-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.js +4 -19
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.js +1 -1
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +2 -2
- package/dist/sync/sync-index.js +2 -2
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +1 -1
- package/dist/system/ServiceContext.js +1 -1
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config.js +1 -1
- package/dist/util/config.js.map +1 -1
- package/dist/util/errors.d.ts +3 -0
- package/dist/util/errors.js +15 -0
- package/dist/util/errors.js.map +1 -0
- package/dist/util/protocol-types.d.ts +2 -2
- package/dist/util/util-index.d.ts +1 -1
- package/dist/util/util-index.js +1 -1
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/package.json +6 -6
- package/src/api/RouteAPI.ts +17 -0
- package/src/api/api-index.ts +1 -1
- package/src/api/api-metrics.ts +1 -1
- package/src/api/diagnostics.ts +18 -19
- package/src/auth/CachedKeyCollector.ts +2 -3
- package/src/auth/CompoundKeyCollector.ts +2 -3
- package/src/auth/KeyStore.ts +1 -1
- package/src/auth/RemoteJWKSCollector.ts +0 -1
- package/src/auth/StaticKeyCollector.ts +1 -1
- package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
- package/src/entry/commands/teardown-action.ts +1 -1
- package/src/entry/entry-index.ts +1 -1
- package/src/events/EventsEngine.ts +1 -1
- package/src/metrics/MetricsEngine.ts +1 -1
- package/src/metrics/RollingBucketMax.ts +109 -0
- package/src/metrics/metrics-index.ts +3 -2
- package/src/metrics/open-telemetry/util.ts +1 -1
- package/src/metrics/register-metrics.ts +3 -3
- package/src/modules/AbstractModule.ts +1 -1
- package/src/modules/modules-index.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +2 -2
- package/src/replication/AbstractReplicator.ts +9 -7
- package/src/replication/ReplicationLagTracker.ts +86 -0
- package/src/replication/replication-index.ts +3 -2
- package/src/replication/replication-metrics.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +3 -5
- package/src/routes/hooks.ts +2 -2
- package/src/routes/route-register.ts +2 -10
- package/src/storage/BucketStorageBatch.ts +1 -1
- package/src/storage/BucketStorageFactory.ts +18 -14
- package/src/storage/ChecksumCache.ts +1 -1
- package/src/storage/PersistedSyncRulesContent.ts +18 -2
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/SourceTable.ts +1 -1
- package/src/storage/storage-index.ts +8 -8
- package/src/storage/storage-metrics.ts +2 -2
- package/src/streams/streams-index.ts +2 -2
- package/src/sync/BucketChecksumState.ts +5 -21
- package/src/sync/RequestTracker.ts +1 -1
- package/src/sync/sync-index.ts +2 -2
- package/src/sync/sync.ts +1 -7
- package/src/sync/util.ts +1 -1
- package/src/system/ServiceContext.ts +1 -1
- package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
- package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
- package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
- package/src/util/config.ts +1 -1
- package/src/util/errors.ts +21 -0
- package/src/util/util-index.ts +1 -1
- package/src/util/utils.ts +1 -1
- package/test/src/ReplicationLagTracker.test.ts +53 -0
- package/test/src/RollingBucketMax.test.ts +106 -0
- package/test/src/auth.test.ts +7 -7
- package/test/src/module-loader.test.ts +1 -1
- package/test/src/routes/mocks.ts +1 -1
- package/test/src/routes/stream.test.ts +1 -2
- package/test/src/sync/BucketChecksumState.test.ts +2 -2
- package/test/src/util/protocol_types.test.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +1 -1
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { ReplicationLagTracker } from '@/replication/ReplicationLagTracker.js';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
|
|
4
|
+
describe('ReplicationLagTracker', () => {
|
|
5
|
+
it('returns undefined before replication has started', () => {
|
|
6
|
+
const tracker = new ReplicationLagTracker();
|
|
7
|
+
|
|
8
|
+
expect(tracker.getLagMillis(0)).toBeUndefined();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it('tracks the oldest in-flight change and returns current lag', () => {
|
|
12
|
+
const tracker = new ReplicationLagTracker();
|
|
13
|
+
|
|
14
|
+
tracker.trackUncommittedChange(new Date(1_000));
|
|
15
|
+
tracker.trackUncommittedChange(new Date(2_000));
|
|
16
|
+
|
|
17
|
+
expect(tracker.oldestUncommittedChange?.getTime()).toBe(1_000);
|
|
18
|
+
expect(tracker.getCurrentLagMillis(4_000)).toBe(3_000);
|
|
19
|
+
expect(tracker.getLagMillis(4_000)).toBe(3_000);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('records commit lag into the rolling window and clears in-flight state', () => {
|
|
23
|
+
const tracker = new ReplicationLagTracker();
|
|
24
|
+
|
|
25
|
+
tracker.trackUncommittedChange(new Date(0));
|
|
26
|
+
tracker.markCommitted(5_000);
|
|
27
|
+
|
|
28
|
+
expect(tracker.oldestUncommittedChange).toBeNull();
|
|
29
|
+
expect(tracker.isStartingReplication).toBe(false);
|
|
30
|
+
expect(tracker.getLagMillis(5_000)).toBe(5_000);
|
|
31
|
+
expect(tracker.getLagMillis(35_000)).toBe(0);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('can clear in-flight state without changing started state', () => {
|
|
35
|
+
const tracker = new ReplicationLagTracker();
|
|
36
|
+
|
|
37
|
+
tracker.trackUncommittedChange(new Date(0));
|
|
38
|
+
tracker.clearUncommittedChange();
|
|
39
|
+
|
|
40
|
+
expect(tracker.oldestUncommittedChange).toBeNull();
|
|
41
|
+
expect(tracker.isStartingReplication).toBe(true);
|
|
42
|
+
expect(tracker.getLagMillis(5_000)).toBeUndefined();
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('can mark replication as started without a committed transaction', () => {
|
|
46
|
+
const tracker = new ReplicationLagTracker();
|
|
47
|
+
|
|
48
|
+
tracker.markStarted();
|
|
49
|
+
|
|
50
|
+
expect(tracker.isStartingReplication).toBe(false);
|
|
51
|
+
expect(tracker.getLagMillis(0)).toBe(0);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { RollingBucketMax } from '@/metrics/RollingBucketMax.js';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
|
|
4
|
+
describe('RollingBucketMax', () => {
|
|
5
|
+
it('returns undefined before any values are reported', () => {
|
|
6
|
+
const tracker = new RollingBucketMax();
|
|
7
|
+
|
|
8
|
+
expect(tracker.getRollingMax(0)).toBeUndefined();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it('tracks the maximum value within a single bucket', () => {
|
|
12
|
+
const tracker = new RollingBucketMax();
|
|
13
|
+
|
|
14
|
+
tracker.report(3, 100);
|
|
15
|
+
tracker.report(9, 1_000);
|
|
16
|
+
tracker.report(5, 4_999);
|
|
17
|
+
|
|
18
|
+
expect(tracker.getRollingMax(4_999)).toBe(9);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('keeps the rolling max across the last six 5s buckets', () => {
|
|
22
|
+
const tracker = new RollingBucketMax();
|
|
23
|
+
|
|
24
|
+
tracker.report(20, 0);
|
|
25
|
+
tracker.report(11, 5_000);
|
|
26
|
+
tracker.report(12, 10_000);
|
|
27
|
+
tracker.report(13, 15_000);
|
|
28
|
+
tracker.report(14, 20_000);
|
|
29
|
+
tracker.report(15, 25_000);
|
|
30
|
+
|
|
31
|
+
expect(tracker.getRollingMax(29_999)).toBe(20);
|
|
32
|
+
expect(tracker.getRollingMax(30_000)).toBe(15);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('slides the rolling max forward as older buckets age out', () => {
|
|
36
|
+
const tracker = new RollingBucketMax();
|
|
37
|
+
|
|
38
|
+
tracker.report(20, 0);
|
|
39
|
+
expect(tracker.getRollingMax(0)).toBe(20);
|
|
40
|
+
|
|
41
|
+
tracker.report(18, 5_000);
|
|
42
|
+
expect(tracker.getRollingMax(5_000)).toBe(20);
|
|
43
|
+
|
|
44
|
+
tracker.report(16, 10_000);
|
|
45
|
+
expect(tracker.getRollingMax(10_000)).toBe(20);
|
|
46
|
+
|
|
47
|
+
tracker.report(14, 15_000);
|
|
48
|
+
expect(tracker.getRollingMax(15_000)).toBe(20);
|
|
49
|
+
|
|
50
|
+
tracker.report(12, 20_000);
|
|
51
|
+
expect(tracker.getRollingMax(20_000)).toBe(20);
|
|
52
|
+
|
|
53
|
+
tracker.report(10, 25_000);
|
|
54
|
+
expect(tracker.getRollingMax(29_999)).toBe(20);
|
|
55
|
+
|
|
56
|
+
tracker.report(8, 30_000);
|
|
57
|
+
expect(tracker.getRollingMax(30_000)).toBe(18);
|
|
58
|
+
|
|
59
|
+
tracker.report(6, 35_000);
|
|
60
|
+
expect(tracker.getRollingMax(35_000)).toBe(16);
|
|
61
|
+
|
|
62
|
+
tracker.report(4, 40_000);
|
|
63
|
+
expect(tracker.getRollingMax(40_000)).toBe(14);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('keeps newer buckets in the rolling window while older peaks fall out', () => {
|
|
67
|
+
const tracker = new RollingBucketMax();
|
|
68
|
+
|
|
69
|
+
tracker.report(50, 0);
|
|
70
|
+
tracker.report(11, 5_000);
|
|
71
|
+
tracker.report(12, 10_000);
|
|
72
|
+
tracker.report(13, 15_000);
|
|
73
|
+
tracker.report(14, 20_000);
|
|
74
|
+
tracker.report(15, 25_000);
|
|
75
|
+
|
|
76
|
+
expect(tracker.getRollingMax(29_999)).toBe(50);
|
|
77
|
+
|
|
78
|
+
tracker.report(40, 30_000);
|
|
79
|
+
expect(tracker.getRollingMax(30_000)).toBe(40);
|
|
80
|
+
expect(tracker.getRollingMax(34_999)).toBe(40);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('expires values after the rolling window passes with no new reports', () => {
|
|
84
|
+
const tracker = new RollingBucketMax();
|
|
85
|
+
|
|
86
|
+
tracker.report(7, 0);
|
|
87
|
+
|
|
88
|
+
expect(tracker.getRollingMax(29_999)).toBe(7);
|
|
89
|
+
expect(tracker.getRollingMax(30_000)).toBeUndefined();
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('supports custom bucket and window sizes', () => {
|
|
93
|
+
const tracker = new RollingBucketMax({
|
|
94
|
+
bucketSizeMs: 1_000,
|
|
95
|
+
windowSizeMs: 3_000
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
tracker.report(4, 0);
|
|
99
|
+
tracker.report(6, 1_000);
|
|
100
|
+
tracker.report(5, 2_000);
|
|
101
|
+
|
|
102
|
+
expect(tracker.getRollingMax(2_999)).toBe(6);
|
|
103
|
+
expect(tracker.getRollingMax(3_000)).toBe(6);
|
|
104
|
+
expect(tracker.getRollingMax(4_000)).toBe(5);
|
|
105
|
+
});
|
|
106
|
+
});
|
package/test/src/auth.test.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
|
|
1
|
+
import { StaticSupabaseKeyCollector } from '@/index.js';
|
|
3
2
|
import * as jose from 'jose';
|
|
4
|
-
import {
|
|
3
|
+
import { describe, expect, test } from 'vitest';
|
|
4
|
+
import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
|
|
5
|
+
import { KeyResult } from '../../src/auth/KeyCollector.js';
|
|
5
6
|
import { KeySpec } from '../../src/auth/KeySpec.js';
|
|
7
|
+
import { KeyStore } from '../../src/auth/KeyStore.js';
|
|
6
8
|
import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
|
|
10
|
-
import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
|
|
9
|
+
import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
|
|
10
|
+
import { getSupabaseJwksUrl } from '../../src/auth/utils.js';
|
|
11
11
|
|
|
12
12
|
const publicKeyRSA: jose.JWK = {
|
|
13
13
|
use: 'sig',
|
package/test/src/routes/mocks.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EventsEngine } from '@/events/EventsEngine.js';
|
|
1
2
|
import {
|
|
2
3
|
BucketStorageFactory,
|
|
3
4
|
createCoreAPIMetrics,
|
|
@@ -11,7 +12,6 @@ import {
|
|
|
11
12
|
SyncRulesBucketStorage
|
|
12
13
|
} from '@/index.js';
|
|
13
14
|
import { MeterProvider } from '@opentelemetry/sdk-metrics';
|
|
14
|
-
import { EventsEngine } from '@/events/EventsEngine.js';
|
|
15
15
|
|
|
16
16
|
export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
|
|
17
17
|
// This is very incomplete - just enough to get the current tests passing.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
|
|
2
2
|
import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
|
|
3
|
-
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
3
|
+
import { DEFAULT_HYDRATION_STATE, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
4
4
|
import { Readable, Writable } from 'stream';
|
|
5
5
|
import { pipeline } from 'stream/promises';
|
|
6
6
|
import { describe, expect, it } from 'vitest';
|
|
@@ -8,7 +8,6 @@ import winston from 'winston';
|
|
|
8
8
|
import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
|
|
9
9
|
import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
|
|
10
10
|
import { mockServiceContext } from './mocks.js';
|
|
11
|
-
import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
|
|
12
11
|
|
|
13
12
|
describe('Stream Route', () => {
|
|
14
13
|
describe('compressed stream', () => {
|
|
@@ -15,13 +15,12 @@ import {
|
|
|
15
15
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
16
16
|
import {
|
|
17
17
|
ParameterIndexLookupCreator,
|
|
18
|
-
RequestJwtPayload,
|
|
19
18
|
ScopedParameterLookup,
|
|
19
|
+
SourceTableInterface,
|
|
20
20
|
SqliteJsonRow,
|
|
21
21
|
SqliteRow,
|
|
22
22
|
SqlSyncRules,
|
|
23
23
|
TablePattern,
|
|
24
|
-
SourceTableInterface,
|
|
25
24
|
versionedHydrationState
|
|
26
25
|
} from '@powersync/service-sync-rules';
|
|
27
26
|
import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
|
|
@@ -986,6 +985,7 @@ bucket_definitions:
|
|
|
986
985
|
expect(errorMessages[0]).toContain('tasks: 20');
|
|
987
986
|
expect(errorMessages[0]).toContain('comments: 10');
|
|
988
987
|
|
|
988
|
+
expect(errorData[0].checkpoint).toEqual(1n);
|
|
989
989
|
expect(errorData[0].parameter_query_results).toBe(60);
|
|
990
990
|
expect(errorData[0].parameter_query_results_by_definition).toEqual({
|
|
991
991
|
projects: 30,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { StreamingSyncRequest } from '@/index.js';
|
|
2
2
|
import { schema } from '@powersync/lib-services-framework';
|
|
3
|
-
import { describe,
|
|
3
|
+
import { describe, expect, test } from 'vitest';
|
|
4
4
|
|
|
5
5
|
describe('protocol types', () => {
|
|
6
6
|
describe('StreamingSyncRequest', () => {
|