@powersync/service-core 0.12.1 → 0.13.0

package/src/storage/MongoBucketStorage.ts

@@ -347,9 +347,9 @@ export class MongoBucketStorage
       .catch(ignoreNotExiting);
 
     return {
-      operations_size_bytes: operations_aggregate[0].storageStats.size,
-      parameters_size_bytes: parameters_aggregate[0].storageStats.size,
-      replication_size_bytes: replication_aggregate[0].storageStats.size
+      operations_size_bytes: Number(operations_aggregate[0].storageStats.size),
+      parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size),
+      replication_size_bytes: Number(replication_aggregate[0].storageStats.size)
     };
   }
 

package/src/storage/SourceTable.ts

@@ -48,6 +48,8 @@ export class SourceTable {
   }
 
   /**
+   * Use for postgres only.
+   *
    * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]})
    */
   get qualifiedName() {
@@ -55,6 +57,8 @@ export class SourceTable {
   }
 
   /**
+   * Use for postgres and logs only.
+   *
    * Usage: db.query(`SELECT * FROM ${table.escapedIdentifier}`)
    */
   get escapedIdentifier() {

package/src/storage/mongo/MongoBucketBatch.ts

@@ -609,6 +609,8 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     super[Symbol.dispose]();
   }
 
+  private lastWaitingLogThottled = 0;
+
   async commit(lsn: string): Promise<boolean> {
     await this.flush();
 
@@ -619,9 +621,12 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
       return false;
     }
     if (lsn < this.no_checkpoint_before_lsn) {
-      logger.info(
-        `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`
-      );
+      if (Date.now() - this.lastWaitingLogThottled > 5_000) {
+        logger.info(
+          `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`
+        );
+        this.lastWaitingLogThottled = Date.now();
+      }
 
       // Edge case: During initial replication, we have a no_checkpoint_before_lsn set,
       // and don't actually commit the snapshot.

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -265,7 +265,7 @@ export class MongoSyncBucketStorage
         },
         {
           $group: {
-            _id: '$key',
+            _id: { key: '$key', lookup: '$lookup' },
             bucket_parameters: {
               $first: '$bucket_parameters'
             }
@@ -522,11 +522,13 @@ export class MongoSyncBucketStorage
     while (true) {
       try {
         await this.clearIteration();
+
+        logger.info(`${this.slot_name} Done clearing data`);
         return;
       } catch (e: unknown) {
         if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') {
           logger.info(
-            `Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.`
+            `${this.slot_name} Cleared batch of data in ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`
           );
           await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
           continue;

package/test/src/auth.test.ts

@@ -40,6 +40,17 @@ const privateKeyEdDSA: jose.JWK = {
   alg: 'EdDSA'
 };
 
+const privateKeyECDSA: jose.JWK = {
+  use: 'sig',
+  kty: 'EC',
+  crv: 'P-256',
+  kid: 'k3',
+  x: 'Y37HQjG1YvlQZ16CzO7UQxgkY_us-NfPxMPcHUDN-PE',
+  y: 'W3Jqs5_qlIh2UH79l8L3ApqNu14aFetM5oc9oCjAEaw',
+  d: 'p2HQaJApdgaAemVuVsL1hscCFOTd0r9uGxRnzvAelFU',
+  alg: 'ES256'
+};
+
 describe('JWT Auth', () => {
   test('KeyStore basics', async () => {
     const keys = await StaticKeyCollector.importKeys([sharedKey]);
@@ -372,4 +383,26 @@ describe('JWT Auth', () => {
 
     expect(verified.claim).toEqual('test-claim');
   });
+
+  test('signing with ECDSA', async () => {
+    const keys = await StaticKeyCollector.importKeys([privateKeyECDSA]);
+    const store = new KeyStore(keys);
+    const signKey = (await jose.importJWK(privateKeyECDSA)) as jose.KeyLike;
+
+    const signedJwt = await new jose.SignJWT({ claim: 'test-claim-2' })
+      .setProtectedHeader({ alg: 'ES256', kid: 'k3' })
+      .setSubject('f1')
+      .setIssuedAt()
+      .setIssuer('tester')
+      .setAudience('tests')
+      .setExpirationTime('5m')
+      .sign(signKey);
+
+    const verified = (await store.verifyJwt(signedJwt, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    })) as JwtPayload & { claim: string };
+
+    expect(verified.claim).toEqual('test-claim-2');
+  });
 });

package/test/src/data_storage.test.ts

@@ -119,6 +119,76 @@ bucket_definitions:
     ]);
   });
 
+  test('it should use the latest version after updates', async () => {
+    const sync_rules = testRules(
+      `
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT id AS todo_id
+        FROM todos
+        WHERE list_id IN token_parameters.list_id
+    data: [] 
+    `
+    );
+
+    const storage = (await factory()).getInstance(sync_rules);
+
+    const table = makeTestTable('todos', ['id', 'list_id']);
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // Create two todos which initially belong to different lists
+      await batch.save({
+        sourceTable: table,
+        tag: SaveOperationTag.INSERT,
+        after: {
+          id: 'todo1',
+          list_id: 'list1'
+        },
+        afterReplicaId: rid('todo1')
+      });
+      await batch.save({
+        sourceTable: table,
+        tag: SaveOperationTag.INSERT,
+        after: {
+          id: 'todo2',
+          list_id: 'list2'
+        },
+        afterReplicaId: rid('todo2')
+      });
+    });
+
+    const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // Update the second todo item to now belong to list 1
+      await batch.save({
+        sourceTable: table,
+        tag: SaveOperationTag.UPDATE,
+        after: {
+          id: 'todo2',
+          list_id: 'list1'
+        },
+        afterReplicaId: rid('todo2')
+      });
+    });
+
+    // We specifically request the todo_ids for both lists.
+    // There removal operation for the association of `list2`::`todo2` should not interfere with the new
+    // association of `list1`::`todo2`
+    const parameters = await storage.getParameterSets(BigInt(result2!.flushed_op).toString(), [
+      ['mybucket', '1', 'list1'],
+      ['mybucket', '1', 'list2']
+    ]);
+
+    expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([
+      {
+        todo_id: 'todo1'
+      },
+      {
+        todo_id: 'todo2'
+      }
+    ]);
+  });
+
   test('save and load parameters with different number types', async () => {
     const sync_rules = testRules(
       `

package/test/src/routes/probes.test.ts

@@ -20,7 +20,12 @@ vi.mock('@powersync/lib-services-framework', () => ({
       afterSend: () => Promise<void>;
       __micro_router_response = true;
 
-      constructor({ status, data, headers, afterSend }: {
+      constructor({
+        status,
+        data,
+        headers,
+        afterSend
+      }: {
         status?: number;
         data: any;
         headers?: Record<string, string>;