@powersync/service-core 0.12.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/auth/KeySpec.d.ts +1 -0
- package/dist/auth/KeySpec.js +5 -2
- package/dist/auth/KeySpec.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js +1 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/entry/commands/compact-action.js +14 -14
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/migrate-action.js +15 -4
- package/dist/entry/commands/migrate-action.js.map +1 -1
- package/dist/index.d.ts +1 -3
- package/dist/index.js +1 -3
- package/dist/index.js.map +1 -1
- package/dist/migrations/PowerSyncMigrationManager.d.ts +17 -0
- package/dist/migrations/PowerSyncMigrationManager.js +21 -0
- package/dist/migrations/PowerSyncMigrationManager.js.map +1 -0
- package/dist/migrations/ensure-automatic-migrations.d.ts +4 -0
- package/dist/migrations/ensure-automatic-migrations.js +14 -0
- package/dist/migrations/ensure-automatic-migrations.js.map +1 -0
- package/dist/migrations/migrations-index.d.ts +2 -3
- package/dist/migrations/migrations-index.js +2 -3
- package/dist/migrations/migrations-index.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +12 -12
- package/dist/routes/endpoints/admin.d.ts +24 -24
- package/dist/routes/endpoints/probes.d.ts +1 -1
- package/dist/routes/endpoints/probes.js +5 -5
- package/dist/routes/endpoints/probes.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +49 -1
- package/dist/storage/BucketStorage.js +26 -0
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/SourceTable.d.ts +4 -0
- package/dist/storage/SourceTable.js +4 -0
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/bson.d.ts +24 -0
- package/dist/storage/bson.js +73 -0
- package/dist/storage/bson.js.map +1 -0
- package/dist/storage/storage-index.d.ts +3 -14
- package/dist/storage/storage-index.js +3 -14
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/sync.js +3 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +3 -0
- package/dist/system/ServiceContext.js +11 -3
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/util/config/types.d.ts +2 -2
- package/dist/util/utils.d.ts +13 -1
- package/dist/util/utils.js +20 -1
- package/dist/util/utils.js.map +1 -1
- package/package.json +7 -8
- package/src/auth/KeySpec.ts +5 -3
- package/src/auth/RemoteJWKSCollector.ts +1 -1
- package/src/entry/commands/compact-action.ts +19 -14
- package/src/entry/commands/migrate-action.ts +17 -4
- package/src/index.ts +1 -4
- package/src/migrations/PowerSyncMigrationManager.ts +42 -0
- package/src/migrations/ensure-automatic-migrations.ts +15 -0
- package/src/migrations/migrations-index.ts +2 -3
- package/src/routes/endpoints/probes.ts +6 -6
- package/src/storage/BucketStorage.ts +53 -1
- package/src/storage/SourceTable.ts +4 -0
- package/src/storage/bson.ts +78 -0
- package/src/storage/storage-index.ts +3 -15
- package/src/sync/sync.ts +3 -1
- package/src/system/ServiceContext.ts +17 -4
- package/src/util/config/types.ts +2 -2
- package/src/util/utils.ts +21 -1
- package/test/src/auth.test.ts +33 -0
- package/test/src/env.ts +0 -1
- package/test/src/routes/probes.test.ts +6 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/db/db-index.d.ts +0 -1
- package/dist/db/db-index.js +0 -2
- package/dist/db/db-index.js.map +0 -1
- package/dist/db/mongo.d.ts +0 -35
- package/dist/db/mongo.js +0 -73
- package/dist/db/mongo.js.map +0 -1
- package/dist/locks/LockManager.d.ts +0 -10
- package/dist/locks/LockManager.js +0 -7
- package/dist/locks/LockManager.js.map +0 -1
- package/dist/locks/MongoLocks.d.ts +0 -36
- package/dist/locks/MongoLocks.js +0 -81
- package/dist/locks/MongoLocks.js.map +0 -1
- package/dist/locks/locks-index.d.ts +0 -2
- package/dist/locks/locks-index.js +0 -3
- package/dist/locks/locks-index.js.map +0 -1
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +0 -3
- package/dist/migrations/db/migrations/1684951997326-init.js +0 -33
- package/dist/migrations/db/migrations/1684951997326-init.js.map +0 -1
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.d.ts +0 -2
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +0 -5
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +0 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +0 -3
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +0 -56
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +0 -1
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +0 -3
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +0 -29
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +0 -1
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts +0 -3
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js +0 -31
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map +0 -1
- package/dist/migrations/definitions.d.ts +0 -18
- package/dist/migrations/definitions.js +0 -6
- package/dist/migrations/definitions.js.map +0 -1
- package/dist/migrations/executor.d.ts +0 -16
- package/dist/migrations/executor.js +0 -64
- package/dist/migrations/executor.js.map +0 -1
- package/dist/migrations/migrations.d.ts +0 -18
- package/dist/migrations/migrations.js +0 -110
- package/dist/migrations/migrations.js.map +0 -1
- package/dist/migrations/store/migration-store.d.ts +0 -11
- package/dist/migrations/store/migration-store.js +0 -46
- package/dist/migrations/store/migration-store.js.map +0 -1
- package/dist/storage/MongoBucketStorage.d.ts +0 -48
- package/dist/storage/MongoBucketStorage.js +0 -427
- package/dist/storage/MongoBucketStorage.js.map +0 -1
- package/dist/storage/mongo/MongoBucketBatch.d.ts +0 -74
- package/dist/storage/mongo/MongoBucketBatch.js +0 -683
- package/dist/storage/mongo/MongoBucketBatch.js.map +0 -1
- package/dist/storage/mongo/MongoCompactor.d.ts +0 -40
- package/dist/storage/mongo/MongoCompactor.js +0 -310
- package/dist/storage/mongo/MongoCompactor.js.map +0 -1
- package/dist/storage/mongo/MongoIdSequence.d.ts +0 -12
- package/dist/storage/mongo/MongoIdSequence.js +0 -21
- package/dist/storage/mongo/MongoIdSequence.js.map +0 -1
- package/dist/storage/mongo/MongoPersistedSyncRules.d.ts +0 -9
- package/dist/storage/mongo/MongoPersistedSyncRules.js +0 -9
- package/dist/storage/mongo/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +0 -20
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +0 -26
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +0 -5
- package/dist/storage/mongo/MongoStorageProvider.js +0 -26
- package/dist/storage/mongo/MongoStorageProvider.js.map +0 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +0 -38
- package/dist/storage/mongo/MongoSyncBucketStorage.js +0 -534
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +0 -1
- package/dist/storage/mongo/MongoSyncRulesLock.d.ts +0 -16
- package/dist/storage/mongo/MongoSyncRulesLock.js +0 -65
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +0 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +0 -20
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +0 -104
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +0 -1
- package/dist/storage/mongo/OperationBatch.d.ts +0 -35
- package/dist/storage/mongo/OperationBatch.js +0 -119
- package/dist/storage/mongo/OperationBatch.js.map +0 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +0 -46
- package/dist/storage/mongo/PersistedBatch.js +0 -223
- package/dist/storage/mongo/PersistedBatch.js.map +0 -1
- package/dist/storage/mongo/config.d.ts +0 -19
- package/dist/storage/mongo/config.js +0 -26
- package/dist/storage/mongo/config.js.map +0 -1
- package/dist/storage/mongo/db.d.ts +0 -36
- package/dist/storage/mongo/db.js +0 -47
- package/dist/storage/mongo/db.js.map +0 -1
- package/dist/storage/mongo/models.d.ts +0 -163
- package/dist/storage/mongo/models.js +0 -27
- package/dist/storage/mongo/models.js.map +0 -1
- package/dist/storage/mongo/util.d.ts +0 -54
- package/dist/storage/mongo/util.js +0 -190
- package/dist/storage/mongo/util.js.map +0 -1
- package/src/db/db-index.ts +0 -1
- package/src/db/mongo.ts +0 -81
- package/src/locks/LockManager.ts +0 -16
- package/src/locks/MongoLocks.ts +0 -142
- package/src/locks/locks-index.ts +0 -2
- package/src/migrations/db/migrations/1684951997326-init.ts +0 -38
- package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +0 -5
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +0 -102
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +0 -34
- package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +0 -37
- package/src/migrations/definitions.ts +0 -21
- package/src/migrations/executor.ts +0 -87
- package/src/migrations/migrations.ts +0 -142
- package/src/migrations/store/migration-store.ts +0 -63
- package/src/storage/MongoBucketStorage.ts +0 -541
- package/src/storage/mongo/MongoBucketBatch.ts +0 -900
- package/src/storage/mongo/MongoCompactor.ts +0 -393
- package/src/storage/mongo/MongoIdSequence.ts +0 -24
- package/src/storage/mongo/MongoPersistedSyncRules.ts +0 -16
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +0 -50
- package/src/storage/mongo/MongoStorageProvider.ts +0 -31
- package/src/storage/mongo/MongoSyncBucketStorage.ts +0 -640
- package/src/storage/mongo/MongoSyncRulesLock.ts +0 -85
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +0 -154
- package/src/storage/mongo/OperationBatch.ts +0 -131
- package/src/storage/mongo/PersistedBatch.ts +0 -285
- package/src/storage/mongo/config.ts +0 -40
- package/src/storage/mongo/db.ts +0 -88
- package/src/storage/mongo/models.ts +0 -187
- package/src/storage/mongo/util.ts +0 -203
- package/test/src/__snapshots__/sync.test.ts.snap +0 -332
- package/test/src/bucket_validation.test.ts +0 -143
- package/test/src/bucket_validation.ts +0 -60
- package/test/src/compacting.test.ts +0 -295
- package/test/src/data_storage.test.ts +0 -1569
- package/test/src/stream_utils.ts +0 -42
- package/test/src/sync.test.ts +0 -511
- package/test/src/util.ts +0 -150
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
|
|
1
|
+
import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
|
|
2
2
|
|
|
3
|
+
import { framework } from '../index.js';
|
|
3
4
|
import * as metrics from '../metrics/Metrics.js';
|
|
5
|
+
import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
|
|
4
6
|
import * as replication from '../replication/replication-index.js';
|
|
5
7
|
import * as routes from '../routes/routes-index.js';
|
|
6
8
|
import * as storage from '../storage/storage-index.js';
|
|
@@ -13,6 +15,7 @@ export interface ServiceContext {
|
|
|
13
15
|
replicationEngine: replication.ReplicationEngine | null;
|
|
14
16
|
routerEngine: routes.RouterEngine | null;
|
|
15
17
|
storageEngine: storage.StorageEngine;
|
|
18
|
+
migrations: PowerSyncMigrationManager;
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
/**
|
|
@@ -30,13 +33,19 @@ export class ServiceContextContainer implements ServiceContext {
|
|
|
30
33
|
this.storageEngine = new storage.StorageEngine({
|
|
31
34
|
configuration
|
|
32
35
|
});
|
|
36
|
+
|
|
37
|
+
const migrationManager = new MigrationManager();
|
|
38
|
+
container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
|
|
39
|
+
|
|
40
|
+
this.lifeCycleEngine.withLifecycle(migrationManager, {
|
|
41
|
+
// Migrations should be executed before the system starts
|
|
42
|
+
start: () => migrationManager[Symbol.asyncDispose]()
|
|
43
|
+
});
|
|
44
|
+
|
|
33
45
|
this.lifeCycleEngine.withLifecycle(this.storageEngine, {
|
|
34
46
|
start: (storageEngine) => storageEngine.start(),
|
|
35
47
|
stop: (storageEngine) => storageEngine.shutDown()
|
|
36
48
|
});
|
|
37
|
-
|
|
38
|
-
// Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
|
|
39
|
-
this.storageEngine.registerProvider(new storage.MongoStorageProvider());
|
|
40
49
|
}
|
|
41
50
|
|
|
42
51
|
get replicationEngine(): replication.ReplicationEngine | null {
|
|
@@ -51,6 +60,10 @@ export class ServiceContextContainer implements ServiceContext {
|
|
|
51
60
|
return container.getOptional(metrics.Metrics);
|
|
52
61
|
}
|
|
53
62
|
|
|
63
|
+
get migrations(): PowerSyncMigrationManager {
|
|
64
|
+
return container.getImplementation(framework.ContainerImplementation.MIGRATION_MANAGER);
|
|
65
|
+
}
|
|
66
|
+
|
|
54
67
|
/**
|
|
55
68
|
* Allows for registering core and generic implementations of services/helpers.
|
|
56
69
|
* This uses the framework container under the hood.
|
package/src/util/config/types.ts
CHANGED
|
@@ -30,8 +30,8 @@ export type SyncRulesConfig = {
|
|
|
30
30
|
|
|
31
31
|
export type ResolvedPowerSyncConfig = {
|
|
32
32
|
base_config: PowerSyncConfig;
|
|
33
|
-
connections?: configFile.
|
|
34
|
-
storage: configFile.
|
|
33
|
+
connections?: configFile.GenericDataSourceConfig[];
|
|
34
|
+
storage: configFile.GenericStorageConfig;
|
|
35
35
|
dev: {
|
|
36
36
|
demo_auth: boolean;
|
|
37
37
|
demo_password?: string;
|
package/src/util/utils.ts
CHANGED
|
@@ -131,7 +131,16 @@ export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
|
|
|
131
131
|
return false;
|
|
132
132
|
}
|
|
133
133
|
|
|
134
|
-
|
|
134
|
+
/**
|
|
135
|
+
* Returns true if we have a complete row.
|
|
136
|
+
*
|
|
137
|
+
* If we don't store data, we assume we always have a complete row.
|
|
138
|
+
*/
|
|
139
|
+
export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
|
|
140
|
+
if (!storeData) {
|
|
141
|
+
// Assume the row is complete - no need to check
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
135
144
|
return !hasToastedValues(row);
|
|
136
145
|
}
|
|
137
146
|
|
|
@@ -199,6 +208,17 @@ export function reduceBucket(operations: OplogEntry[]) {
|
|
|
199
208
|
return finalState;
|
|
200
209
|
}
|
|
201
210
|
|
|
211
|
+
/**
|
|
212
|
+
* Flattens string to reduce memory usage (around 320 bytes -> 120 bytes),
|
|
213
|
+
* at the cost of some upfront CPU usage.
|
|
214
|
+
*
|
|
215
|
+
* From: https://github.com/davidmarkclements/flatstr/issues/8
|
|
216
|
+
*/
|
|
217
|
+
export function flatstr(s: string) {
|
|
218
|
+
s.match(/\n/g);
|
|
219
|
+
return s;
|
|
220
|
+
}
|
|
221
|
+
|
|
202
222
|
function rowKey(entry: OplogEntry) {
|
|
203
223
|
return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
|
|
204
224
|
}
|
package/test/src/auth.test.ts
CHANGED
|
@@ -40,6 +40,17 @@ const privateKeyEdDSA: jose.JWK = {
|
|
|
40
40
|
alg: 'EdDSA'
|
|
41
41
|
};
|
|
42
42
|
|
|
43
|
+
const privateKeyECDSA: jose.JWK = {
|
|
44
|
+
use: 'sig',
|
|
45
|
+
kty: 'EC',
|
|
46
|
+
crv: 'P-256',
|
|
47
|
+
kid: 'k3',
|
|
48
|
+
x: 'Y37HQjG1YvlQZ16CzO7UQxgkY_us-NfPxMPcHUDN-PE',
|
|
49
|
+
y: 'W3Jqs5_qlIh2UH79l8L3ApqNu14aFetM5oc9oCjAEaw',
|
|
50
|
+
d: 'p2HQaJApdgaAemVuVsL1hscCFOTd0r9uGxRnzvAelFU',
|
|
51
|
+
alg: 'ES256'
|
|
52
|
+
};
|
|
53
|
+
|
|
43
54
|
describe('JWT Auth', () => {
|
|
44
55
|
test('KeyStore basics', async () => {
|
|
45
56
|
const keys = await StaticKeyCollector.importKeys([sharedKey]);
|
|
@@ -372,4 +383,26 @@ describe('JWT Auth', () => {
|
|
|
372
383
|
|
|
373
384
|
expect(verified.claim).toEqual('test-claim');
|
|
374
385
|
});
|
|
386
|
+
|
|
387
|
+
test('signing with ECDSA', async () => {
|
|
388
|
+
const keys = await StaticKeyCollector.importKeys([privateKeyECDSA]);
|
|
389
|
+
const store = new KeyStore(keys);
|
|
390
|
+
const signKey = (await jose.importJWK(privateKeyECDSA)) as jose.KeyLike;
|
|
391
|
+
|
|
392
|
+
const signedJwt = await new jose.SignJWT({ claim: 'test-claim-2' })
|
|
393
|
+
.setProtectedHeader({ alg: 'ES256', kid: 'k3' })
|
|
394
|
+
.setSubject('f1')
|
|
395
|
+
.setIssuedAt()
|
|
396
|
+
.setIssuer('tester')
|
|
397
|
+
.setAudience('tests')
|
|
398
|
+
.setExpirationTime('5m')
|
|
399
|
+
.sign(signKey);
|
|
400
|
+
|
|
401
|
+
const verified = (await store.verifyJwt(signedJwt, {
|
|
402
|
+
defaultAudiences: ['tests'],
|
|
403
|
+
maxAge: '6m'
|
|
404
|
+
})) as JwtPayload & { claim: string };
|
|
405
|
+
|
|
406
|
+
expect(verified.claim).toEqual('test-claim-2');
|
|
407
|
+
});
|
|
375
408
|
});
|
package/test/src/env.ts
CHANGED
|
@@ -20,7 +20,12 @@ vi.mock('@powersync/lib-services-framework', () => ({
|
|
|
20
20
|
afterSend: () => Promise<void>;
|
|
21
21
|
__micro_router_response = true;
|
|
22
22
|
|
|
23
|
-
constructor({
|
|
23
|
+
constructor({
|
|
24
|
+
status,
|
|
25
|
+
data,
|
|
26
|
+
headers,
|
|
27
|
+
afterSend
|
|
28
|
+
}: {
|
|
24
29
|
status?: number;
|
|
25
30
|
data: any;
|
|
26
31
|
headers?: Record<string, string>;
|