@powersync/service-module-mongodb 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219110735

package/src/storage/implementation/models.ts

@@ -0,0 +1,160 @@
+import { storage } from '@powersync/service-core';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = bson.UUID | bson.Document | any;
+
+export interface SourceKey {
+  /** group_id */
+  g: number;
+  /** source table id */
+  t: bson.ObjectId;
+  /** source key */
+  k: ReplicaId;
+}
+
+export interface CurrentDataDocument {
+  _id: SourceKey;
+  data: bson.Binary;
+  buckets: CurrentBucket[];
+  lookups: bson.Binary[];
+}
+
+export interface CurrentBucket {
+  bucket: string;
+  table: string;
+  id: string;
+}
+
+export interface BucketParameterDocument {
+  _id: bigint;
+  key: SourceKey;
+  lookup: bson.Binary;
+  bucket_parameters: Record<string, SqliteJsonValue>[];
+}
+
+export interface BucketDataKey {
+  /** group_id */
+  g: number;
+  /** bucket name */
+  b: string;
+  /** op_id */
+  o: bigint;
+}
+export interface BucketDataDocument {
+  _id: BucketDataKey;
+  op: OpType;
+  source_table?: bson.ObjectId;
+  source_key?: storage.ReplicaId;
+  table?: string;
+  row_id?: string;
+  checksum: number;
+  data: string | null;
+  target_op?: bigint | null;
+}
+
+export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+
+export interface SourceTableDocument {
+  _id: bson.ObjectId;
+  group_id: number;
+  connection_id: number;
+  relation_id: number | string | undefined;
+  schema_name: string;
+  table_name: string;
+  replica_id_columns: string[] | null;
+  replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
+  snapshot_done: boolean | undefined;
+}
+
+export interface IdSequenceDocument {
+  _id: string;
+  op_id: bigint;
+}
+
+export interface SyncRuleDocument {
+  _id: number;
+
+  state: storage.SyncRuleState;
+
+  /**
+   * True if initial snapshot has been replicated.
+   *
+   * Can only be false if state == PROCESSING.
+   */
+  snapshot_done: boolean;
+
+  /**
+   * The last consistent checkpoint.
+   *
+   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
+   */
+  last_checkpoint: bigint | null;
+
+  /**
+   * The LSN associated with the last consistent checkpoint.
+   */
+  last_checkpoint_lsn: string | null;
+
+  /**
+   * If set, no new checkpoints may be created < this value.
+   */
+  no_checkpoint_before: string | null;
+
+  /**
+   * Goes together with no_checkpoint_before.
+   *
+   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+   * then the checkpoint must be equal to this keepalive_op.
+   */
+  keepalive_op: string | null;
+
+  slot_name: string | null;
+
+  /**
+   * Last time we persisted a checkpoint.
+   *
+   * This may be old if no data is incoming.
+   */
+  last_checkpoint_ts: Date | null;
+
+  /**
+   * Last time we persisted a checkpoint or keepalive.
+   *
+   * This should stay fairly current while replicating.
+   */
+  last_keepalive_ts: Date | null;
+
+  /**
+   * If an error is stopping replication, it will be stored here.
+   */
+  last_fatal_error: string | null;
+
+  content: string;
+}
+
+export interface CustomWriteCheckpointDocument {
+  _id: bson.ObjectId;
+  user_id: string;
+  checkpoint: bigint;
+  sync_rules_id: number;
+}
+
+export interface WriteCheckpointDocument {
+  _id: bson.ObjectId;
+  user_id: string;
+  lsns: Record<string, string>;
+  client_id: bigint;
+}
+
+export interface InstanceDocument {
+  // The instance UUID
+  _id: string;
+}

package/src/storage/implementation/util.ts

@@ -0,0 +1,209 @@
+import { storage, utils } from '@powersync/service-core';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import * as crypto from 'crypto';
+import * as mongo from 'mongodb';
+import * as uuid from 'uuid';
+import { BucketDataDocument } from './models.js';
+
+/**
+ * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
+ * @param lookup
+ */
+
+export function serializeLookup(lookup: SqliteJsonValue[]) {
+  const normalized = lookup.map((value) => {
+    if (typeof value == 'number' && Number.isInteger(value)) {
+      return BigInt(value);
+    } else {
+      return value;
+    }
+  });
+  return new bson.Binary(bson.serialize({ l: normalized }));
+}
+
+export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
+  let filter = {
+    $gte: {
+      ...prefix
+    } as any,
+    $lt: {
+      ...prefix
+    } as any
+  };
+
+  for (let key of rest) {
+    filter.$gte[key] = new bson.MinKey();
+    filter.$lt[key] = new bson.MaxKey();
+  }
+
+  return filter;
+}
+
+export function generateSlotName(prefix: string, sync_rules_id: number) {
+  const slot_suffix = crypto.randomBytes(2).toString('hex');
+  return `${prefix}${sync_rules_id}_${slot_suffix}`;
+}
+
+/**
+ * Read a single batch of data from a cursor, then close it.
+ *
+ * We do our best to avoid MongoDB fetching any more data than this single batch.
+ *
+ * This is similar to using `singleBatch: true` in find options.
+ * However, that makes `has_more` detection very difficult, since the cursor is always closed
+ * after the first batch. Instead, we do a workaround to only fetch a single batch below.
+ *
+ * For this to be effective, set batchSize = limit in the find command.
+ */
+export async function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
+  try {
+    let data: T[];
+    let hasMore = true;
+    // Let MongoDB load the first batch of data
+    const hasAny = await cursor.hasNext();
+    // Now it's in memory, and we can read it
+    data = cursor.readBufferedDocuments();
+    if (!hasAny || cursor.id?.isZero()) {
+      // A zero id means the cursor is exhaused.
+      // No results (hasAny == false) means even this batch doesn't have data.
+      // This should similar results as `await cursor.hasNext()`, but without
+      // actually fetching the next batch.
+      //
+      // Note that it is safe (but slightly inefficient) to return `hasMore: true`
+      // without there being more data, as long as the next batch
+      // will return `hasMore: false`.
+      hasMore = false;
+    }
+    return { data, hasMore };
+  } finally {
+    // Match the from the cursor iterator logic here:
+    // https://github.com/mongodb/node-mongodb-native/blob/e02534e7d1c627bf50b85ca39f5995dbf165ad44/src/cursor/abstract_cursor.ts#L327-L331
+    if (!cursor.closed) {
+      await cursor.close();
+    }
+  }
+}
+
+export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
+  // use bigint instead of Long
+  useBigInt64: true
+};
+
+export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
+  if (row.op == 'PUT' || row.op == 'REMOVE') {
+    return {
+      op_id: utils.timestampToOpId(row._id.o),
+      op: row.op,
+      object_type: row.table,
+      object_id: row.row_id,
+      checksum: Number(row.checksum),
+      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
+      data: row.data
+    };
+  } else {
+    // MOVE, CLEAR
+
+    return {
+      op_id: utils.timestampToOpId(row._id.o),
+      op: row.op,
+      checksum: Number(row.checksum)
+    };
+  }
+}
+
+/**
+ * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
+ */
+export function replicaIdEquals(a: storage.ReplicaId, b: storage.ReplicaId) {
+  if (a === b) {
+    return true;
+  } else if (typeof a == 'string' && typeof b == 'string') {
+    return a == b;
+  } else if (isUUID(a) && isUUID(b)) {
+    return a.equals(b);
+  } else if (a == null && b == null) {
+    return true;
+  } else if ((b == null && a != null) || (a == null && b != null)) {
+    return false;
+  } else {
+    // There are many possible primitive values, this covers them all
+    return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
+  }
+}
+
+export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string {
+  if (isUUID(id)) {
+    // Special case for UUID for backwards-compatiblity
+    return `${table.toHexString()}/${id.toHexString()}`;
+  } else {
+    // Hashed UUID from the table and id
+    const repr = bson.serialize({ table, id });
+    return uuid.v5(repr, utils.ID_NAMESPACE);
+  }
+}
+
+/**
+ * True if this is a bson.UUID.
+ *
+ * Works even with multiple copies of the bson package.
+ */
+export function isUUID(value: any): value is bson.UUID {
+  if (value == null || typeof value != 'object') {
+    return false;
+  }
+  const uuid = value as bson.UUID;
+  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
+}
+
+/**
+ * Helper function for creating a MongoDB client from consumers of this package
+ */
+export const createMongoClient = (url: string, options?: mongo.MongoClientOptions) => {
+  return new mongo.MongoClient(url, options);
+};
+
+/**
+ * MongoDB bulkWrite internally splits the operations into batches
+ * so that no batch exceeds 16MB. However, there are cases where
+ * the batch size is very close to 16MB, where additional metadata
+ * on the server pushes it over the limit, resulting in this error
+ * from the server:
+ *
+ * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
+ *
+ * We work around the issue by doing our own batching, limiting the
+ * batch size to 15MB. This does add additional overhead with
+ * BSON.calculateObjectSize.
+ */
+export async function safeBulkWrite<T extends mongo.Document>(
+  collection: mongo.Collection<T>,
+  operations: mongo.AnyBulkWriteOperation<T>[],
+  options: mongo.BulkWriteOptions
+) {
+  // Must be below 16MB.
+  // We could probably go a little closer, but 15MB is a safe threshold.
+  const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
+
+  let batch: mongo.AnyBulkWriteOperation<T>[] = [];
+  let currentSize = 0;
+  // Estimated overhead per operation, should be smaller in reality.
+  const keySize = 8;
+  for (let op of operations) {
+    const bsonSize =
+      mongo.BSON.calculateObjectSize(op, {
+        checkKeys: false,
+        ignoreUndefined: true
+      } as any) + keySize;
+    if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
+      await collection.bulkWrite(batch, options);
+      currentSize = 0;
+      batch = [];
+    }
+    batch.push(op);
+    currentSize += bsonSize;
+  }
+  if (batch.length > 0) {
+    await collection.bulkWrite(batch, options);
+  }
+}

package/src/storage/storage-index.ts

@@ -0,0 +1,14 @@
+export * from './implementation/config.js';
+export * from './implementation/db.js';
+export * from './implementation/models.js';
+export * from './implementation/MongoBucketBatch.js';
+export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
+export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoStorageProvider.js';
+export * from './implementation/MongoSyncBucketStorage.js';
+export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/OperationBatch.js';
+export * from './implementation/PersistedBatch.js';
+export * from './implementation/util.js';
+export * from './MongoBucketStorage.js';

package/src/types/types.ts

@@ -1,6 +1,7 @@
-import { normalizeMongoConfig } from '@powersync/service-core';
 import * as service_types from '@powersync/service-types';
+import { MongoStorageConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
 import * as t from 'ts-codec';
+import { MONGO_STORAGE_TYPE, normalizeMongoConfig } from '../storage/storage-index.js';
 
 export const MONGO_CONNECTION_TYPE = 'mongodb' as const;
 
@@ -103,3 +104,9 @@ export function normalizeConnectionConfig(options: MongoConnectionConfig): Norma
 export function baseUri(options: NormalizedMongoConnectionConfig) {
   return options.uri;
 }
+
+export function isMongoStorageConfig(
+  config: service_types.configFile.GenericStorageConfig
+): config is MongoStorageConfig {
+  return config.type == MONGO_STORAGE_TYPE;
+}