@powersync/service-core 0.0.0-dev-20241119082750 → 0.0.0-dev-20241219091224

package/src/storage/storage-index.ts

@@ -1,21 +1,9 @@
 export * from './BucketStorage.js';
-export * from './MongoBucketStorage.js';
+export * from './ChecksumCache.js';
 export * from './ReplicationEventPayload.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
-
-export * from './mongo/config.js';
-export * from './mongo/db.js';
-export * from './mongo/models.js';
-export * from './mongo/MongoBucketBatch.js';
-export * from './mongo/MongoIdSequence.js';
-export * from './mongo/MongoPersistedSyncRules.js';
-export * from './mongo/MongoPersistedSyncRulesContent.js';
-export * from './mongo/MongoStorageProvider.js';
-export * from './mongo/MongoSyncBucketStorage.js';
-export * from './mongo/MongoSyncRulesLock.js';
-export * from './mongo/OperationBatch.js';
-export * from './mongo/PersistedBatch.js';
-export * from './mongo/util.js';
+export * from './StorageProvider.js';
 export * from './WriteCheckpointAPI.js';
+

package/src/sync/sync.ts

@@ -318,9 +318,18 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         // Send the object as is, will most likely be encoded as a BSON document
         send_data = { data: r };
       } else if (raw_data) {
-        // Data is a raw string - we can use the more efficient JSON.stringify.
+        /**
+         * Data is a raw string - we can use the more efficient JSON.stringify.
+         * FIXME: ensure that the data is actually a string
+         */
         const response: util.StreamingSyncData = {
-          data: r
+          data: {
+            ...r,
+            data: r.data.map((entry) => ({
+              ...entry,
+              data: typeof entry.data == 'string' ? entry.data : JSONBig.stringify(entry.data)
+            }))
+          }
         };
         send_data = JSON.stringify(response);
       } else {
@@ -375,7 +384,7 @@ function transformLegacyResponse(bucketData: util.SyncBucketData): any {
     data: bucketData.data.map((entry) => {
       return {
         ...entry,
-        data: entry.data == null ? null : new JsonContainer(entry.data as string),
+        data: entry.data == null ? null : typeof entry.data == 'string' ? new JsonContainer(entry.data) : entry.data,
         checksum: BigInt(entry.checksum)
       };
     })

package/src/system/ServiceContext.ts

@@ -1,6 +1,8 @@
-import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
+import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
 
+import { framework } from '../index.js';
 import * as metrics from '../metrics/Metrics.js';
+import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
 import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
@@ -13,6 +15,7 @@ export interface ServiceContext {
   replicationEngine: replication.ReplicationEngine | null;
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
+  migrations: PowerSyncMigrationManager;
 }
 
 /**
@@ -30,13 +33,19 @@ export class ServiceContextContainer implements ServiceContext {
     this.storageEngine = new storage.StorageEngine({
       configuration
     });
+
+    const migrationManager = new MigrationManager();
+    container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
+
+    this.lifeCycleEngine.withLifecycle(migrationManager, {
+      // Migrations should be executed before the system starts
+      start: () => migrationManager[Symbol.asyncDispose]()
+    });
+
     this.lifeCycleEngine.withLifecycle(this.storageEngine, {
       start: (storageEngine) => storageEngine.start(),
       stop: (storageEngine) => storageEngine.shutDown()
     });
-
-    // Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
-    this.storageEngine.registerProvider(new storage.MongoStorageProvider());
   }
 
   get replicationEngine(): replication.ReplicationEngine | null {
@@ -51,6 +60,10 @@ export class ServiceContextContainer implements ServiceContext {
     return container.getOptional(metrics.Metrics);
   }
 
+  get migrations(): PowerSyncMigrationManager {
+    return container.getImplementation(framework.ContainerImplementation.MIGRATION_MANAGER);
+  }
+
   /**
    * Allows for registering core and generic implementations of services/helpers.
    * This uses the framework container under the hood.

package/src/util/config/compound-config-collector.ts

@@ -65,9 +65,27 @@ export class CompoundConfigCollector {
 
     const inputKeys = baseConfig.client_auth?.jwks?.keys ?? [];
     const staticCollector = await auth.StaticKeyCollector.importKeys(inputKeys);
-
     collectors.add(staticCollector);
 
+    if (baseConfig.client_auth?.supabase && baseConfig.client_auth?.supabase_jwt_secret != null) {
+      // This replaces the old SupabaseKeyCollector, with a statically-configured key.
+      // You can get the same effect with manual HS256 key configuration, but this
+      // makes the config simpler.
+      // We also a custom audience ("authenticated"), increased max lifetime (1 week),
+      // and auto base64-url-encode the key.
+      collectors.add(
+        await auth.StaticSupabaseKeyCollector.importKeys([
+          {
+            kty: 'oct',
+            alg: 'HS256',
+            // In this case, the key is not base64-encoded yet.
+            k: Buffer.from(baseConfig.client_auth.supabase_jwt_secret, 'utf8').toString('base64url'),
+            kid: undefined // Wildcard kid - any kid can match
+          }
+        ])
+      );
+    }
+
     let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
     if (typeof jwks_uris == 'string') {
       jwks_uris = [jwks_uris];

package/src/util/config/types.ts

@@ -30,8 +30,8 @@ export type SyncRulesConfig = {
 
 export type ResolvedPowerSyncConfig = {
   base_config: PowerSyncConfig;
-  connections?: configFile.DataSourceConfig[];
-  storage: configFile.StorageConfig;
+  connections?: configFile.GenericDataSourceConfig[];
+  storage: configFile.GenericStorageConfig;
   dev: {
     demo_auth: boolean;
     demo_password?: string;

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, OpId } from './protocol-types.js';
+import { BucketChecksum, OpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
@@ -144,3 +144,61 @@ export function checkpointUserId(user_id: string | undefined, client_id: string
   }
   return `${user_id}/${client_id}`;
 }
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ *
+ * Used for tests.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}

package/test/src/auth.test.ts

@@ -1,13 +1,14 @@
-import { CachedKeyCollector } from '@/auth/CachedKeyCollector.js';
-import { KeyResult } from '@/auth/KeyCollector.js';
-import { KeySpec } from '@/auth/KeySpec.js';
-import { KeyStore } from '@/auth/KeyStore.js';
-import { RemoteJWKSCollector } from '@/auth/RemoteJWKSCollector.js';
-import { StaticKeyCollector } from '@/auth/StaticKeyCollector.js';
-import * as jose from 'jose';
 import { describe, expect, test } from 'vitest';
-
-const publicKey: jose.JWK = {
+import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import * as jose from 'jose';
+import { KeyStore } from '../../src/auth/KeyStore.js';
+import { KeySpec } from '../../src/auth/KeySpec.js';
+import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
+import { KeyResult } from '../../src/auth/KeyCollector.js';
+import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
+import { JwtPayload } from '@/index.js';
+
+const publicKeyRSA: jose.JWK = {
   use: 'sig',
   kty: 'RSA',
   e: 'AQAB',
@@ -29,6 +30,16 @@ const sharedKey2: jose.JWK = {
   k: Buffer.from('mysecret2', 'utf-8').toString('base64url')
 };
 
+const privateKeyEdDSA: jose.JWK = {
+  use: 'sig',
+  kty: 'OKP',
+  crv: 'Ed25519',
+  kid: 'k2',
+  x: 'nfaqgxakPaiiEdAtRGrubgh_SQ1mr6gAUx3--N-ehvo',
+  d: 'wweBqMbTrME6oChSEMYAOyYzxsGisQb-C1t0XMjb_Ng',
+  alg: 'EdDSA'
+};
+
 describe('JWT Auth', () => {
   test('KeyStore basics', async () => {
     const keys = await StaticKeyCollector.importKeys([sharedKey]);
@@ -86,20 +97,20 @@ describe('JWT Auth', () => {
   });
 
   test('Algorithm validation', async () => {
-    const keys = await StaticKeyCollector.importKeys([publicKey]);
+    const keys = await StaticKeyCollector.importKeys([publicKeyRSA]);
     const store = new KeyStore(keys);
 
     // Bad attempt at signing token with rsa public key
     const spoofedKey: jose.JWK = {
       kty: 'oct',
-      kid: publicKey.kid!,
+      kid: publicKeyRSA.kid!,
       alg: 'HS256',
-      k: publicKey.n!
+      k: publicKeyRSA.n!
     };
     const signKey = (await jose.importJWK(spoofedKey)) as jose.KeyLike;
 
     const signedJwt = await new jose.SignJWT({})
-      .setProtectedHeader({ alg: 'HS256', kid: publicKey.kid! })
+      .setProtectedHeader({ alg: 'HS256', kid: publicKeyRSA.kid! })
       .setSubject('f1')
       .setIssuedAt()
       .setIssuer('tester')
@@ -116,7 +127,7 @@ describe('JWT Auth', () => {
   });
 
   test('key selection for key with kid', async () => {
-    const keys = await StaticKeyCollector.importKeys([publicKey, sharedKey, sharedKey2]);
+    const keys = await StaticKeyCollector.importKeys([publicKeyRSA, sharedKey, sharedKey2]);
     const store = new KeyStore(keys);
     const signKey = (await jose.importJWK(sharedKey)) as jose.KeyLike;
     const signKey2 = (await jose.importJWK(sharedKey2)) as jose.KeyLike;
@@ -296,30 +307,30 @@ describe('JWT Auth', () => {
 
     currentResponse = Promise.resolve({
       errors: [],
-      keys: [await KeySpec.importKey(publicKey)]
+      keys: [await KeySpec.importKey(publicKeyRSA)]
     });
 
     let key = (await cached.getKeys()).keys[0];
-    expect(key.kid).toEqual(publicKey.kid!);
+    expect(key.kid).toEqual(publicKeyRSA.kid!);
 
     currentResponse = undefined as any;
 
     key = (await cached.getKeys()).keys[0];
-    expect(key.kid).toEqual(publicKey.kid!);
+    expect(key.kid).toEqual(publicKeyRSA.kid!);
 
     cached.addTimeForTests(301_000);
     currentResponse = Promise.reject('refresh failed');
 
     // Uses the promise, refreshes in the background
     let response = await cached.getKeys();
-    expect(response.keys[0].kid).toEqual(publicKey.kid!);
+    expect(response.keys[0].kid).toEqual(publicKeyRSA.kid!);
     expect(response.errors).toEqual([]);
 
     // Wait for refresh to finish
     await cached.addTimeForTests(0);
     response = await cached.getKeys();
     // Still have the cached key, but also have the error
-    expect(response.keys[0].kid).toEqual(publicKey.kid!);
+    expect(response.keys[0].kid).toEqual(publicKeyRSA.kid!);
     expect(response.errors[0].message).toMatch('Failed to fetch');
 
     await cached.addTimeForTests(3601_000);
@@ -331,12 +342,34 @@ describe('JWT Auth', () => {
 
     currentResponse = Promise.resolve({
       errors: [],
-      keys: [await KeySpec.importKey(publicKey)]
+      keys: [await KeySpec.importKey(publicKeyRSA)]
     });
 
     // After a delay, we can refresh again
     await cached.addTimeForTests(30_000);
     key = (await cached.getKeys()).keys[0];
-    expect(key.kid).toEqual(publicKey.kid!);
+    expect(key.kid).toEqual(publicKeyRSA.kid!);
+  });
+
+  test('signing with EdDSA', async () => {
+    const keys = await StaticKeyCollector.importKeys([privateKeyEdDSA]);
+    const store = new KeyStore(keys);
+    const signKey = (await jose.importJWK(privateKeyEdDSA)) as jose.KeyLike;
+
+    const signedJwt = await new jose.SignJWT({ claim: 'test-claim' })
+      .setProtectedHeader({ alg: 'EdDSA', kid: 'k2' })
+      .setSubject('f1')
+      .setIssuedAt()
+      .setIssuer('tester')
+      .setAudience('tests')
+      .setExpirationTime('5m')
+      .sign(signKey);
+
+    const verified = (await store.verifyJwt(signedJwt, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    })) as JwtPayload & { claim: string };
+
+    expect(verified.claim).toEqual('test-claim');
   });
 });

package/test/src/env.ts

@@ -1,6 +1,5 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
-  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   CI: utils.type.boolean.default('false')
 });