@powersync/service-module-mongodb 0.0.0-dev-20241219091224 → 0.0.0-dev-20241219145106

package/src/storage/implementation/MongoBucketBatch.ts

@@ -11,7 +11,7 @@ import { MongoIdSequence } from './MongoIdSequence.js';
 import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
-import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
+import { idPrefixFilter } from './util.js';
 
 /**
  * 15MB
@@ -314,7 +314,10 @@ export class MongoBucketBatch
         existing_buckets = result.buckets;
         existing_lookups = result.lookups;
         if (this.storeCurrentData) {
-          const data = bson.deserialize((result.data as mongo.Binary).buffer, BSON_DESERIALIZE_OPTIONS) as SqliteRow;
+          const data = bson.deserialize(
+            (result.data as mongo.Binary).buffer,
+            storage.BSON_DESERIALIZE_OPTIONS
+          ) as SqliteRow;
           after = storage.mergeToast(after!, data);
         }
       }
@@ -370,7 +373,7 @@ export class MongoBucketBatch
     }
 
     // 2. Save bucket data
-    if (beforeId != null && (afterId == null || !replicaIdEquals(beforeId, afterId))) {
+    if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) {
       // Source ID updated
       if (sourceTable.syncData) {
         // Delete old record
@@ -476,7 +479,7 @@ export class MongoBucketBatch
           existing_lookups
         });
         new_lookups = paramEvaluated.map((p) => {
-          return serializeLookup(p.lookup);
+          return storage.serializeLookup(p.lookup);
         });
       }
     }
@@ -500,7 +503,7 @@ export class MongoBucketBatch
       };
     }
 
-    if (afterId == null || !replicaIdEquals(beforeId, afterId)) {
+    if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) {
       // Either a delete (afterId == null), or replaced the old replication id
       batch.deleteCurrentData(before_key);
     }

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -12,7 +12,7 @@ import { BucketDataDocument, BucketDataKey, SourceKey } from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
-import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
+import { idPrefixFilter, mapOpEntry, readSingleBatch } from './util.js';
 
 export class MongoSyncBucketStorage
   extends DisposableObserver<storage.SyncRulesBucketStorageListener>
@@ -219,7 +219,7 @@ export class MongoSyncBucketStorage
 
   async getParameterSets(checkpoint: utils.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]> {
     const lookupFilter = lookups.map((lookup) => {
-      return serializeLookup(lookup);
+      return storage.serializeLookup(lookup);
     });
     const rows = await this.db.bucket_parameters
       .aggregate([
@@ -325,7 +325,7 @@ export class MongoSyncBucketStorage
 
     // Ordered by _id, meaning buckets are grouped together
     for (let rawData of data) {
-      const row = bson.deserialize(rawData, BSON_DESERIALIZE_OPTIONS) as BucketDataDocument;
+      const row = bson.deserialize(rawData, storage.BSON_DESERIALIZE_OPTIONS) as BucketDataDocument;
       const bucket = row._id.b;
 
       if (currentBatch == null || currentBatch.bucket != bucket || batchSize >= sizeLimit) {

package/src/storage/implementation/OperationBatch.ts

@@ -2,7 +2,6 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
 import { storage } from '@powersync/service-core';
-import { isUUID } from './util.js';
 
 /**
  * Maximum number of operations in a batch.
@@ -94,7 +93,7 @@ export class RecordOperation {
  * In-memory cache key - must not be persisted.
  */
 export function cacheKey(table: bson.ObjectId, id: storage.ReplicaId) {
-  if (isUUID(id)) {
+  if (storage.isUUID(id)) {
     return `${table.toHexString()}.${id.toHexString()}`;
   } else if (typeof id == 'string') {
     return `${table.toHexString()}.${id}`;

package/src/storage/implementation/PersistedBatch.ts

@@ -15,7 +15,7 @@ import {
   CurrentDataDocument,
   SourceKey
 } from './models.js';
-import { replicaIdToSubkey, safeBulkWrite, serializeLookup } from './util.js';
+import { replicaIdToSubkey, safeBulkWrite } from './util.js';
 
 /**
  * Maximum size of operations we write in a single transaction.
@@ -164,7 +164,7 @@ export class PersistedBatch {
 
     // 1. Insert new entries
     for (let result of evaluated) {
-      const binLookup = serializeLookup(result.lookup);
+      const binLookup = storage.serializeLookup(result.lookup);
       const hex = binLookup.toString('base64');
       remaining_lookups.delete(hex);
 

package/src/storage/implementation/db.ts

@@ -1,6 +1,6 @@
+import { storage } from '@powersync/service-core';
 import { configFile } from '@powersync/service-types';
 import * as mongo from 'mongodb';
-
 import * as db from '../../db/db-index.js';
 import { Lock } from '../../locks/MonogLocks.js';
 import {
@@ -14,7 +14,6 @@ import {
   SyncRuleDocument,
   WriteCheckpointDocument
 } from './models.js';
-import { BSON_DESERIALIZE_OPTIONS } from './util.js';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -46,7 +45,7 @@ export class PowerSyncMongo {
     this.client = client;
 
     const db = client.db(options?.database, {
-      ...BSON_DESERIALIZE_OPTIONS
+      ...storage.BSON_DESERIALIZE_OPTIONS
     });
     this.db = db;
 

package/src/storage/implementation/models.ts

@@ -20,6 +20,15 @@ export interface SourceKey {
   k: ReplicaId;
 }
 
+export interface BucketDataKey {
+  /** group_id */
+  g: number;
+  /** bucket name */
+  b: string;
+  /** op_id */
+  o: bigint;
+}
+
 export interface CurrentDataDocument {
   _id: SourceKey;
   data: bson.Binary;
@@ -40,19 +49,11 @@ export interface BucketParameterDocument {
   bucket_parameters: Record<string, SqliteJsonValue>[];
 }
 
-export interface BucketDataKey {
-  /** group_id */
-  g: number;
-  /** bucket name */
-  b: string;
-  /** op_id */
-  o: bigint;
-}
 export interface BucketDataDocument {
   _id: BucketDataKey;
   op: OpType;
   source_table?: bson.ObjectId;
-  source_key?: storage.ReplicaId;
+  source_key?: ReplicaId;
   table?: string;
   row_id?: string;
   checksum: number;

package/src/storage/implementation/util.ts

@@ -1,27 +1,10 @@
 import { storage, utils } from '@powersync/service-core';
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import * as crypto from 'crypto';
 import * as mongo from 'mongodb';
 import * as uuid from 'uuid';
 import { BucketDataDocument } from './models.js';
 
-/**
- * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
- * @param lookup
- */
-
-export function serializeLookup(lookup: SqliteJsonValue[]) {
-  const normalized = lookup.map((value) => {
-    if (typeof value == 'number' && Number.isInteger(value)) {
-      return BigInt(value);
-    } else {
-      return value;
-    }
-  });
-  return new bson.Binary(bson.serialize({ l: normalized }));
-}
-
 export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
   let filter = {
     $gte: {
@@ -85,11 +68,6 @@ export async function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{
   }
 }
 
-export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
-  // use bigint instead of Long
-  useBigInt64: true
-};
-
 export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
   if (row.op == 'PUT' || row.op == 'REMOVE') {
     return {
@@ -112,28 +90,8 @@ export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
   }
 }
 
-/**
- * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
- */
-export function replicaIdEquals(a: storage.ReplicaId, b: storage.ReplicaId) {
-  if (a === b) {
-    return true;
-  } else if (typeof a == 'string' && typeof b == 'string') {
-    return a == b;
-  } else if (isUUID(a) && isUUID(b)) {
-    return a.equals(b);
-  } else if (a == null && b == null) {
-    return true;
-  } else if ((b == null && a != null) || (a == null && b != null)) {
-    return false;
-  } else {
-    // There are many possible primitive values, this covers them all
-    return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
-  }
-}
-
 export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string {
-  if (isUUID(id)) {
+  if (storage.isUUID(id)) {
     // Special case for UUID for backwards-compatiblity
     return `${table.toHexString()}/${id.toHexString()}`;
   } else {
@@ -143,19 +101,6 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
-/**
- * True if this is a bson.UUID.
- *
- * Works even with multiple copies of the bson package.
- */
-export function isUUID(value: any): value is bson.UUID {
-  if (value == null || typeof value != 'object') {
-    return false;
-  }
-  const uuid = value as bson.UUID;
-  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
-}
-
 /**
  * Helper function for creating a MongoDB client from consumers of this package
  */