@powersync/service-core 0.10.1 → 0.11.0

package/src/storage/BucketStorage.ts

@@ -211,6 +211,13 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
    * database, for example from MongoDB.
    */
   storeCurrentData: boolean;
+
+  /**
+   * Set to true for initial replication.
+   *
+   * This will avoid creating new operations for rows previously replicated.
+   */
+  skipExistingRows?: boolean;
 }
 
 export interface SyncRulesBucketStorageListener extends DisposableListener {

package/src/storage/MongoBucketStorage.ts

@@ -199,6 +199,7 @@ export class MongoBucketStorage
         last_checkpoint: null,
         last_checkpoint_lsn: null,
         no_checkpoint_before: null,
+        keepalive_op: null,
         snapshot_done: false,
         state: SyncRuleState.PROCESSING,
         slot_name: slot_name,

package/src/storage/mongo/MongoBucketBatch.ts

@@ -9,6 +9,7 @@ import {
   BucketStorageBatch,
   FlushedResult,
   mergeToast,
+  SaveOperationTag,
   SaveOptions
 } from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
@@ -20,6 +21,7 @@ import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
+import * as timers from 'node:timers/promises';
 
 /**
  * 15MB
@@ -39,8 +41,13 @@ export interface MongoBucketBatchOptions {
   groupId: number;
   slotName: string;
   lastCheckpointLsn: string | null;
+  keepaliveOp: string | null;
   noCheckpointBeforeLsn: string;
   storeCurrentData: boolean;
+  /**
+   * Set to true for initial replication.
+   */
+  skipExistingRows: boolean;
 }
 
 export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListener> implements BucketStorageBatch {
@@ -53,6 +60,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
 
   private readonly slot_name: string;
   private readonly storeCurrentData: boolean;
+  private readonly skipExistingRows: boolean;
 
   private batch: OperationBatch | null = null;
   private write_checkpoint_batch: CustomWriteCheckpointOptions[] = [];
@@ -86,7 +94,12 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     this.slot_name = options.slotName;
     this.sync_rules = options.syncRules;
     this.storeCurrentData = options.storeCurrentData;
+    this.skipExistingRows = options.skipExistingRows;
     this.batch = new OperationBatch();
+
+    if (options.keepaliveOp) {
+      this.persisted_op = BigInt(options.keepaliveOp);
+    }
   }
 
   addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void {
@@ -148,10 +161,13 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     op_seq: MongoIdSequence
   ): Promise<OperationBatch | null> {
     let sizes: Map<string, number> | undefined = undefined;
-    if (this.storeCurrentData) {
+    if (this.storeCurrentData && !this.skipExistingRows) {
       // We skip this step if we don't store current_data, since the sizes will
       // always be small in that case.
 
+      // With skipExistingRows, we don't load the full documents into memory,
+      // so we can also skip the size lookup step.
+
       // Find sizes of current_data documents, to assist in intelligent batching without
       // exceeding memory limits.
       //
@@ -204,11 +220,13 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
         return { g: this.group_id, t: r.record.sourceTable.id, k: r.beforeId };
       });
       let current_data_lookup = new Map<string, CurrentDataDocument>();
+      // With skipExistingRows, we only need to know whether or not the row exists.
+      const projection = this.skipExistingRows ? { _id: 1 } : undefined;
       const cursor = this.db.current_data.find(
         {
           _id: { $in: lookups }
         },
-        { session }
+        { session, projection }
       );
       for await (let doc of cursor.stream()) {
         current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc);
@@ -273,7 +291,21 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
 
     const before_key: SourceKey = { g: this.group_id, t: record.sourceTable.id, k: beforeId };
 
-    if (record.tag == 'update') {
+    if (this.skipExistingRows) {
+      if (record.tag == SaveOperationTag.INSERT) {
+        if (current_data != null) {
+          // Initial replication, and we already have the record.
+          // This may be a different version of the record, but streaming replication
+          // will take care of that.
+          // Skip the insert here.
+          return null;
+        }
+      } else {
+        throw new Error(`${record.tag} not supported with skipExistingRows: true`);
+      }
+    }
+
+    if (record.tag == SaveOperationTag.UPDATE) {
       const result = current_data;
       if (result == null) {
         // Not an error if we re-apply a transaction
@@ -293,7 +325,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
           after = mergeToast(after!, data);
         }
       }
-    } else if (record.tag == 'delete') {
+    } else if (record.tag == SaveOperationTag.DELETE) {
       const result = current_data;
       if (result == null) {
         // Not an error if we re-apply a transaction
@@ -494,7 +526,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
             } else {
               logger.warn('Transaction error', e as Error);
             }
-            await new Promise((resolve) => setTimeout(resolve, Math.random() * 50));
+            await timers.setTimeout(Math.random() * 50);
             throw e;
           }
         },
@@ -587,7 +619,28 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
       return false;
     }
     if (lsn < this.no_checkpoint_before_lsn) {
-      logger.info(`Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}`);
+      logger.info(
+        `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`
+      );
+
+      // Edge case: During initial replication, we have a no_checkpoint_before_lsn set,
+      // and don't actually commit the snapshot.
+      // The first commit can happen from an implicit keepalive message.
+      // That needs the persisted_op to get an accurate checkpoint, so
+      // we persist that in keepalive_op.
+
+      await this.db.sync_rules.updateOne(
+        {
+          _id: this.group_id
+        },
+        {
+          $set: {
+            keepalive_op: this.persisted_op == null ? null : String(this.persisted_op)
+          }
+        },
+        { session: this.session }
+      );
+
       return false;
     }
 
@@ -597,7 +650,8 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
       last_checkpoint_ts: now,
       last_keepalive_ts: now,
       snapshot_done: true,
-      last_fatal_error: null
+      last_fatal_error: null,
+      keepalive_op: null
     };
 
     if (this.persisted_op != null) {
@@ -631,6 +685,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     if (this.persisted_op != null) {
       // The commit may have been skipped due to "no_checkpoint_before_lsn".
       // Apply it now if relevant
+      logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`);
       return await this.commit(lsn);
     }
 
@@ -684,9 +739,8 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
 
     if (this.batch.shouldFlush()) {
       const r = await this.flush();
-      // HACK: Give other streams a chance to also flush
-      const t = 150;
-      await new Promise((resolve) => setTimeout(resolve, t));
+      // HACK: Give other streams a  chance to also flush
+      await timers.setTimeout(5);
       return r;
     }
     return null;

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -137,7 +137,7 @@ export class MongoSyncBucketStorage
       {
         _id: this.group_id
       },
-      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1 } }
+      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1 } }
     );
     const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null;
 
@@ -148,7 +148,9 @@ export class MongoSyncBucketStorage
       slotName: this.slot_name,
       lastCheckpointLsn: checkpoint_lsn,
       noCheckpointBeforeLsn: doc?.no_checkpoint_before ?? options.zeroLSN,
-      storeCurrentData: options.storeCurrentData
+      keepaliveOp: doc?.keepalive_op ?? null,
+      storeCurrentData: options.storeCurrentData,
+      skipExistingRows: options.skipExistingRows ?? false
     });
     this.iterateListeners((cb) => cb.batchStarted?.(batch));
 

package/src/storage/mongo/models.ts

@@ -135,6 +135,14 @@ export interface SyncRuleDocument {
    */
   no_checkpoint_before: string | null;
 
+  /**
+   * Goes together with no_checkpoint_before.
+   *
+   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+   * then the checkpoint must be equal to this keepalive_op.
+   */
+  keepalive_op: string | null;
+
   slot_name: string | null;
 
   /**

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, OpId } from './protocol-types.js';
+import { BucketChecksum, OpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
@@ -144,3 +144,61 @@ export function checkpointUserId(user_id: string | undefined, client_id: string
   }
   return `${user_id}/${client_id}`;
 }
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ *
+ * Used for tests.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}

package/test/src/bucket_validation.test.ts

@@ -1,6 +1,7 @@
 import { OplogEntry } from '@/util/protocol-types.js';
 import { describe, expect, test } from 'vitest';
-import { reduceBucket, validateBucket } from './bucket_validation.js';
+import { validateBucket } from './bucket_validation.js';
+import { reduceBucket } from '@/index.js';
 
 // This tests the reduceBucket function.
 // While this function is not used directly in the service implementation,

package/test/src/bucket_validation.ts

@@ -1,63 +1,7 @@
 import { OplogEntry } from '@/util/protocol-types.js';
-import { addChecksums } from '@/util/utils.js';
+import { reduceBucket } from '@/util/utils.js';
 import { expect } from 'vitest';
 
-/**
- * Reduce a bucket to the final state as stored on the client.
- *
- * This keeps the final state for each row as a PUT operation.
- *
- * All other operations are replaced with a single CLEAR operation,
- * summing their checksums, and using a 0 as an op_id.
- *
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
- */
-export function reduceBucket(operations: OplogEntry[]) {
-  let rowState = new Map<string, OplogEntry>();
-  let otherChecksum = 0;
-
-  for (let op of operations) {
-    const key = rowKey(op);
-    if (op.op == 'PUT') {
-      const existing = rowState.get(key);
-      if (existing) {
-        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
-      }
-      rowState.set(key, op);
-    } else if (op.op == 'REMOVE') {
-      const existing = rowState.get(key);
-      if (existing) {
-        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
-      }
-      rowState.delete(key);
-      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
-    } else if (op.op == 'CLEAR') {
-      rowState.clear();
-      otherChecksum = op.checksum as number;
-    } else if (op.op == 'MOVE') {
-      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
-    } else {
-      throw new Error(`Unknown operation ${op.op}`);
-    }
-  }
-
-  const puts = [...rowState.values()].sort((a, b) => {
-    return Number(BigInt(a.op_id) - BigInt(b.op_id));
-  });
-
-  let finalState: OplogEntry[] = [
-    // Special operation to indiciate the checksum remainder
-    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
-    ...puts
-  ];
-
-  return finalState;
-}
-
-function rowKey(entry: OplogEntry) {
-  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
-}
-
 /**
  * Validate this property, as described in /docs/bucket-properties.md:
  *

package/test/src/util.ts

@@ -30,6 +30,8 @@ export interface StorageOptions {
    * Setting this to true will drop the collections completely.
    */
   dropAll?: boolean;
+
+  doNotClear?: boolean;
 }
 export type StorageFactory = (options?: StorageOptions) => Promise<BucketStorageFactory>;
 
@@ -37,7 +39,7 @@ export const MONGO_STORAGE_FACTORY: StorageFactory = async (options?: StorageOpt
   const db = await connectMongo();
   if (options?.dropAll) {
     await db.drop();
-  } else {
+  } else if (!options?.doNotClear) {
     await db.clear();
   }
   return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });