@powersync/service-core 0.12.0 → 0.12.2

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -265,7 +265,7 @@ export class MongoSyncBucketStorage
         },
         {
           $group: {
-            _id: '$key',
+            _id: { key: '$key', lookup: '$lookup' },
             bucket_parameters: {
               $first: '$bucket_parameters'
             }
@@ -522,11 +522,13 @@ export class MongoSyncBucketStorage
     while (true) {
       try {
         await this.clearIteration();
+
+        logger.info(`${this.slot_name} Done clearing data`);
         return;
       } catch (e: unknown) {
         if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') {
           logger.info(
-            `Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.`
+            `${this.slot_name} Cleared batch of data in ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`
           );
           await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
           continue;

package/src/storage/mongo/MongoWriteCheckpointAPI.ts

@@ -9,6 +9,7 @@ import {
   WriteCheckpointMode
 } from '../WriteCheckpointAPI.js';
 import { PowerSyncMongo } from './db.js';
+import { safeBulkWrite } from './util.js';
 
 export type MongoCheckpointAPIOptions = {
   db: PowerSyncMongo;
@@ -134,7 +135,8 @@ export async function batchCreateCustomWriteCheckpoints(
     return;
   }
 
-  await db.custom_write_checkpoints.bulkWrite(
+  await safeBulkWrite(
+    db.custom_write_checkpoints,
     checkpoints.map((checkpointOptions) => ({
       updateOne: {
         filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
@@ -146,6 +148,7 @@ export async function batchCreateCustomWriteCheckpoints(
         },
         upsert: true
       }
-    }))
+    })),
+    {}
   );
 }

package/src/storage/mongo/PersistedBatch.ts

@@ -16,7 +16,7 @@ import {
   SourceKey,
   ReplicaId
 } from './models.js';
-import { replicaIdToSubkey, serializeLookup } from './util.js';
+import { replicaIdToSubkey, safeBulkWrite, serializeLookup } from './util.js';
 import { logger } from '@powersync/lib-services-framework';
 
 /**
@@ -33,6 +33,13 @@ import { logger } from '@powersync/lib-services-framework';
  */
 const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
 
+/**
+ * Limit number of documents to write in a single transaction.
+ *
+ * This has an effect on error message size in some cases.
+ */
+const MAX_TRANSACTION_DOC_COUNT = 2_000;
+
 /**
  * Keeps track of bulkwrite operations within a transaction.
  *
@@ -231,26 +238,32 @@ export class PersistedBatch {
   }
 
   shouldFlushTransaction() {
-    return this.currentSize >= MAX_TRANSACTION_BATCH_SIZE;
+    return (
+      this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
+      this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT ||
+      this.currentData.length >= MAX_TRANSACTION_DOC_COUNT ||
+      this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
+    );
   }
 
   async flush(db: PowerSyncMongo, session: mongo.ClientSession) {
     if (this.bucketData.length > 0) {
-      await db.bucket_data.bulkWrite(this.bucketData, {
+      // calculate total size
+      await safeBulkWrite(db.bucket_data, this.bucketData, {
         session,
         // inserts only - order doesn't matter
         ordered: false
       });
     }
     if (this.bucketParameters.length > 0) {
-      await db.bucket_parameters.bulkWrite(this.bucketParameters, {
+      await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
         session,
         // inserts only - order doesn't matter
         ordered: false
       });
     }
     if (this.currentData.length > 0) {
-      await db.current_data.bulkWrite(this.currentData, {
+      await safeBulkWrite(db.current_data, this.currentData, {
         session,
         // may update and delete data within the same batch - order matters
         ordered: true

package/src/storage/mongo/util.ts

@@ -156,3 +156,48 @@ export function isUUID(value: any): value is bson.UUID {
   const uuid = value as bson.UUID;
   return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
 }
+
+/**
+ * MongoDB bulkWrite internally splits the operations into batches
+ * so that no batch exceeds 16MB. However, there are cases where
+ * the batch size is very close to 16MB, where additional metadata
+ * on the server pushes it over the limit, resulting in this error
+ * from the server:
+ *
+ * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
+ *
+ * We work around the issue by doing our own batching, limiting the
+ * batch size to 15MB. This does add additional overhead with
+ * BSON.calculateObjectSize.
+ */
+export async function safeBulkWrite<T extends mongo.Document>(
+  collection: mongo.Collection<T>,
+  operations: mongo.AnyBulkWriteOperation<T>[],
+  options: mongo.BulkWriteOptions
+) {
+  // Must be below 16MB.
+  // We could probably go a little closer, but 15MB is a safe threshold.
+  const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
+
+  let batch: mongo.AnyBulkWriteOperation<T>[] = [];
+  let currentSize = 0;
+  // Estimated overhead per operation, should be smaller in reality.
+  const keySize = 8;
+  for (let op of operations) {
+    const bsonSize =
+      mongo.BSON.calculateObjectSize(op, {
+        checkKeys: false,
+        ignoreUndefined: true
+      } as any) + keySize;
+    if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
+      await collection.bulkWrite(batch, options);
+      currentSize = 0;
+      batch = [];
+    }
+    batch.push(op);
+    currentSize += bsonSize;
+  }
+  if (batch.length > 0) {
+    await collection.bulkWrite(batch, options);
+  }
+}

package/test/src/data_storage.test.ts

@@ -119,6 +119,76 @@ bucket_definitions:
     ]);
   });
 
+  test('it should use the latest version after updates', async () => {
+    const sync_rules = testRules(
+      `
+bucket_definitions:
+  mybucket:
+    parameters:
+      - SELECT id AS todo_id
+        FROM todos
+        WHERE list_id IN token_parameters.list_id
+    data: [] 
+    `
+    );
+
+    const storage = (await factory()).getInstance(sync_rules);
+
+    const table = makeTestTable('todos', ['id', 'list_id']);
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // Create two todos which initially belong to different lists
+      await batch.save({
+        sourceTable: table,
+        tag: SaveOperationTag.INSERT,
+        after: {
+          id: 'todo1',
+          list_id: 'list1'
+        },
+        afterReplicaId: rid('todo1')
+      });
+      await batch.save({
+        sourceTable: table,
+        tag: SaveOperationTag.INSERT,
+        after: {
+          id: 'todo2',
+          list_id: 'list2'
+        },
+        afterReplicaId: rid('todo2')
+      });
+    });
+
+    const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // Update the second todo item to now belong to list 1
+      await batch.save({
+        sourceTable: table,
+        tag: SaveOperationTag.UPDATE,
+        after: {
+          id: 'todo2',
+          list_id: 'list1'
+        },
+        afterReplicaId: rid('todo2')
+      });
+    });
+
+    // We specifically request the todo_ids for both lists.
+    // There removal operation for the association of `list2`::`todo2` should not interfere with the new
+    // association of `list1`::`todo2`
+    const parameters = await storage.getParameterSets(BigInt(result2!.flushed_op).toString(), [
+      ['mybucket', '1', 'list1'],
+      ['mybucket', '1', 'list2']
+    ]);
+
+    expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([
+      {
+        todo_id: 'todo1'
+      },
+      {
+        todo_id: 'todo2'
+      }
+    ]);
+  });
+
   test('save and load parameters with different number types', async () => {
     const sync_rules = testRules(
       `