@powersync/service-module-mongodb 0.0.0-dev-20250102111825 → 0.0.0-dev-20250108073049

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -1,146 +0,0 @@
-import * as framework from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
-import { PowerSyncMongo } from './db.js';
-import { safeBulkWrite } from './util.js';
-
-export type MongoCheckpointAPIOptions = {
-  db: PowerSyncMongo;
-  mode: storage.WriteCheckpointMode;
-};
-
-export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
-  readonly db: PowerSyncMongo;
-  private _mode: storage.WriteCheckpointMode;
-
-  constructor(options: MongoCheckpointAPIOptions) {
-    this.db = options.db;
-    this._mode = options.mode;
-  }
-
-  get writeCheckpointMode() {
-    return this._mode;
-  }
-
-  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
-    this._mode = mode;
-  }
-
-  async batchCreateCustomWriteCheckpoints(checkpoints: storage.CustomWriteCheckpointOptions[]): Promise<void> {
-    return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
-  }
-
-  async createCustomWriteCheckpoint(options: storage.CustomWriteCheckpointOptions): Promise<bigint> {
-    if (this.writeCheckpointMode !== storage.WriteCheckpointMode.CUSTOM) {
-      throw new framework.errors.ValidationError(
-        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
-      );
-    }
-
-    const { checkpoint, user_id, sync_rules_id } = options;
-    const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id,
-        sync_rules_id
-      },
-      {
-        $set: {
-          checkpoint
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
-    );
-    return doc!.checkpoint;
-  }
-
-  async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
-    if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
-      throw new framework.errors.ValidationError(
-        `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
-      );
-    }
-
-    const { user_id, heads: lsns } = checkpoint;
-    const doc = await this.db.write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id
-      },
-      {
-        $set: {
-          lsns
-        },
-        $inc: {
-          client_id: 1n
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
-    );
-    return doc!.client_id;
-  }
-
-  async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
-    switch (this.writeCheckpointMode) {
-      case storage.WriteCheckpointMode.CUSTOM:
-        if (false == 'sync_rules_id' in filters) {
-          throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
-        }
-        return this.lastCustomWriteCheckpoint(filters);
-      case storage.WriteCheckpointMode.MANAGED:
-        if (false == 'heads' in filters) {
-          throw new framework.errors.ValidationError(
-            `Replication HEAD is required for managed Write Checkpoint filtering`
-          );
-        }
-        return this.lastManagedWriteCheckpoint(filters);
-    }
-  }
-
-  protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) {
-    const { user_id, sync_rules_id } = filters;
-    const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
-      user_id,
-      sync_rules_id
-    });
-    return lastWriteCheckpoint?.checkpoint ?? null;
-  }
-
-  protected async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters) {
-    const { user_id, heads } = filters;
-    // TODO: support multiple heads when we need to support multiple connections
-    const lsn = heads['1'];
-    if (lsn == null) {
-      // Can happen if we haven't replicated anything yet.
-      return null;
-    }
-    const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
-      user_id: user_id,
-      'lsns.1': { $lte: lsn }
-    });
-    return lastWriteCheckpoint?.client_id ?? null;
-  }
-}
-
-export async function batchCreateCustomWriteCheckpoints(
-  db: PowerSyncMongo,
-  checkpoints: storage.CustomWriteCheckpointOptions[]
-): Promise<void> {
-  if (!checkpoints.length) {
-    return;
-  }
-
-  await safeBulkWrite(
-    db.custom_write_checkpoints,
-    checkpoints.map((checkpointOptions) => ({
-      updateOne: {
-        filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
-        update: {
-          $set: {
-            checkpoint: checkpointOptions.checkpoint,
-            sync_rules_id: checkpointOptions.sync_rules_id
-          }
-        },
-        upsert: true
-      }
-    })),
-    {}
-  );
-}

package/src/storage/implementation/OperationBatch.ts

@@ -1,129 +0,0 @@
-import { ToastableSqliteRow } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-
-import { storage } from '@powersync/service-core';
-
-/**
- * Maximum number of operations in a batch.
- */
-const MAX_BATCH_COUNT = 2000;
-
-/**
- * Maximum size of operations in the batch (estimated).
- */
-const MAX_RECORD_BATCH_SIZE = 5_000_000;
-
-/**
- * Maximum size of size of current_data documents we lookup at a time.
- */
-const MAX_CURRENT_DATA_BATCH_SIZE = 16_000_000;
-
-/**
- * Batch of input operations.
- *
- * We accumulate operations up to MAX_RECORD_BATCH_SIZE,
- * then further split into sub-batches if MAX_CURRENT_DATA_BATCH_SIZE is exceeded.
- */
-export class OperationBatch {
-  batch: RecordOperation[] = [];
-  currentSize: number = 0;
-
-  get length() {
-    return this.batch.length;
-  }
-
-  push(op: RecordOperation) {
-    this.batch.push(op);
-    this.currentSize += op.estimatedSize;
-  }
-
-  shouldFlush() {
-    return this.batch.length >= MAX_BATCH_COUNT || this.currentSize > MAX_RECORD_BATCH_SIZE;
-  }
-
-  /**
-   *
-   * @param sizes Map of source key to estimated size of the current_data document, or undefined if current_data is not persisted.
-   *
-   */
-  *batched(sizes: Map<string, number> | undefined): Generator<RecordOperation[]> {
-    if (sizes == null) {
-      yield this.batch;
-      return;
-    }
-    let currentBatch: RecordOperation[] = [];
-    let currentBatchSize = 0;
-    for (let op of this.batch) {
-      const key = op.internalBeforeKey;
-      const size = sizes.get(key) ?? 0;
-      if (currentBatchSize + size > MAX_CURRENT_DATA_BATCH_SIZE && currentBatch.length > 0) {
-        yield currentBatch;
-        currentBatch = [];
-        currentBatchSize = 0;
-      }
-      currentBatchSize += size;
-      currentBatch.push(op);
-    }
-    if (currentBatch.length > 0) {
-      yield currentBatch;
-    }
-  }
-}
-
-export class RecordOperation {
-  public readonly afterId: storage.ReplicaId | null;
-  public readonly beforeId: storage.ReplicaId;
-  public readonly internalBeforeKey: string;
-  public readonly internalAfterKey: string | null;
-  public readonly estimatedSize: number;
-
-  constructor(public readonly record: storage.SaveOptions) {
-    const afterId = record.afterReplicaId ?? null;
-    const beforeId = record.beforeReplicaId ?? record.afterReplicaId;
-    this.afterId = afterId;
-    this.beforeId = beforeId;
-    this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId);
-    this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null;
-
-    this.estimatedSize = estimateRowSize(record.before) + estimateRowSize(record.after);
-  }
-}
-
-/**
- * In-memory cache key - must not be persisted.
- */
-export function cacheKey(table: bson.ObjectId, id: storage.ReplicaId) {
-  if (storage.isUUID(id)) {
-    return `${table.toHexString()}.${id.toHexString()}`;
-  } else if (typeof id == 'string') {
-    return `${table.toHexString()}.${id}`;
-  } else {
-    return `${table.toHexString()}.${(bson.serialize({ id: id }) as Buffer).toString('base64')}`;
-  }
-}
-
-/**
- * Estimate in-memory size of row.
- */
-function estimateRowSize(record: ToastableSqliteRow | undefined) {
-  if (record == null) {
-    return 12;
-  }
-  let size = 0;
-  for (let [key, value] of Object.entries(record)) {
-    size += 12 + key.length;
-    // number | string | null | bigint | Uint8Array
-    if (value == null) {
-      size += 4;
-    } else if (typeof value == 'number') {
-      size += 8;
-    } else if (typeof value == 'bigint') {
-      size += 8;
-    } else if (typeof value == 'string') {
-      size += value.length;
-    } else if (value instanceof Uint8Array) {
-      size += value.byteLength;
-    }
-  }
-  return size;
-}

package/src/storage/implementation/PersistedBatch.ts

@@ -1,283 +0,0 @@
-import { JSONBig } from '@powersync/service-jsonbig';
-import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-import * as mongo from 'mongodb';
-
-import { logger } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
-import { currentBucketKey } from './MongoBucketBatch.js';
-import { MongoIdSequence } from './MongoIdSequence.js';
-import { PowerSyncMongo } from './db.js';
-import {
-  BucketDataDocument,
-  BucketParameterDocument,
-  CurrentBucket,
-  CurrentDataDocument,
-  SourceKey
-} from './models.js';
-import { replicaIdToSubkey, safeBulkWrite } from './util.js';
-
-/**
- * Maximum size of operations we write in a single transaction.
- *
- * It's tricky to find the exact limit, but from experience, over 100MB
- * can cause an error:
- * > transaction is too large and will not fit in the storage engine cache
- *
- * Additionally, unbounded size here can balloon our memory usage in some edge
- * cases.
- *
- * When we reach this threshold, we commit the transaction and start a new one.
- */
-const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
-
-/**
- * Limit number of documents to write in a single transaction.
- *
- * This has an effect on error message size in some cases.
- */
-const MAX_TRANSACTION_DOC_COUNT = 2_000;
-
-/**
- * Keeps track of bulkwrite operations within a transaction.
- *
- * There may be multiple of these batches per transaction, but it may not span
- * multiple transactions.
- */
-export class PersistedBatch {
-  bucketData: mongo.AnyBulkWriteOperation<BucketDataDocument>[] = [];
-  bucketParameters: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
-  currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
-
-  /**
-   * For debug logging only.
-   */
-  debugLastOpId: bigint | null = null;
-
-  /**
-   * Very rough estimate of transaction size.
-   */
-  currentSize = 0;
-
-  constructor(
-    private group_id: number,
-    writtenSize: number
-  ) {
-    this.currentSize = writtenSize;
-  }
-
-  saveBucketData(options: {
-    op_seq: MongoIdSequence;
-    sourceKey: storage.ReplicaId;
-    table: storage.SourceTable;
-    evaluated: EvaluatedRow[];
-    before_buckets: CurrentBucket[];
-  }) {
-    const remaining_buckets = new Map<string, CurrentBucket>();
-    for (let b of options.before_buckets) {
-      const key = currentBucketKey(b);
-      remaining_buckets.set(key, b);
-    }
-
-    const dchecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey));
-
-    for (const k of options.evaluated) {
-      const key = currentBucketKey(k);
-      remaining_buckets.delete(key);
-
-      // INSERT
-      const recordData = JSONBig.stringify(k.data);
-      const checksum = utils.hashData(k.table, k.id, recordData);
-      this.currentSize += recordData.length + 200;
-
-      const op_id = options.op_seq.next();
-      this.debugLastOpId = op_id;
-
-      this.bucketData.push({
-        insertOne: {
-          document: {
-            _id: {
-              g: this.group_id,
-              b: k.bucket,
-              o: op_id
-            },
-            op: 'PUT',
-            source_table: options.table.id,
-            source_key: options.sourceKey,
-            table: k.table,
-            row_id: k.id,
-            checksum: checksum,
-            data: recordData
-          }
-        }
-      });
-    }
-
-    for (let bd of remaining_buckets.values()) {
-      // REMOVE
-
-      const op_id = options.op_seq.next();
-      this.debugLastOpId = op_id;
-
-      this.bucketData.push({
-        insertOne: {
-          document: {
-            _id: {
-              g: this.group_id,
-              b: bd.bucket,
-              o: op_id
-            },
-            op: 'REMOVE',
-            source_table: options.table.id,
-            source_key: options.sourceKey,
-            table: bd.table,
-            row_id: bd.id,
-            checksum: dchecksum,
-            data: null
-          }
-        }
-      });
-      this.currentSize += 200;
-    }
-  }
-
-  saveParameterData(data: {
-    op_seq: MongoIdSequence;
-    sourceKey: storage.ReplicaId;
-    sourceTable: storage.SourceTable;
-    evaluated: EvaluatedParameters[];
-    existing_lookups: bson.Binary[];
-  }) {
-    // This is similar to saving bucket data.
-    // A key difference is that we don't need to keep the history intact.
-    // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint.
-    // Instead of storing per bucket id, we store per "lookup".
-    // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is
-    // either persisted or removed.
-    // We also don't need to keep history intact.
-    const { sourceTable, sourceKey, evaluated } = data;
-
-    const remaining_lookups = new Map<string, bson.Binary>();
-    for (let l of data.existing_lookups) {
-      remaining_lookups.set(l.toString('base64'), l);
-    }
-
-    // 1. Insert new entries
-    for (let result of evaluated) {
-      const binLookup = storage.serializeLookup(result.lookup);
-      const hex = binLookup.toString('base64');
-      remaining_lookups.delete(hex);
-
-      const op_id = data.op_seq.next();
-      this.debugLastOpId = op_id;
-      this.bucketParameters.push({
-        insertOne: {
-          document: {
-            _id: op_id,
-            key: {
-              g: this.group_id,
-              t: sourceTable.id,
-              k: sourceKey
-            },
-            lookup: binLookup,
-            bucket_parameters: result.bucket_parameters
-          }
-        }
-      });
-
-      this.currentSize += 200;
-    }
-
-    // 2. "REMOVE" entries for any lookup not touched.
-    for (let lookup of remaining_lookups.values()) {
-      const op_id = data.op_seq.next();
-      this.debugLastOpId = op_id;
-      this.bucketParameters.push({
-        insertOne: {
-          document: {
-            _id: op_id,
-            key: {
-              g: this.group_id,
-              t: sourceTable.id,
-              k: sourceKey
-            },
-            lookup: lookup,
-            bucket_parameters: []
-          }
-        }
-      });
-
-      this.currentSize += 200;
-    }
-  }
-
-  deleteCurrentData(id: SourceKey) {
-    const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
-      deleteOne: {
-        filter: { _id: id }
-      }
-    };
-    this.currentData.push(op);
-    this.currentSize += 50;
-  }
-
-  upsertCurrentData(id: SourceKey, values: Partial<CurrentDataDocument>) {
-    const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
-      updateOne: {
-        filter: { _id: id },
-        update: {
-          $set: values
-        },
-        upsert: true
-      }
-    };
-    this.currentData.push(op);
-    this.currentSize += (values.data?.length() ?? 0) + 100;
-  }
-
-  shouldFlushTransaction() {
-    return (
-      this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
-      this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT ||
-      this.currentData.length >= MAX_TRANSACTION_DOC_COUNT ||
-      this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
-    );
-  }
-
-  async flush(db: PowerSyncMongo, session: mongo.ClientSession) {
-    if (this.bucketData.length > 0) {
-      // calculate total size
-      await safeBulkWrite(db.bucket_data, this.bucketData, {
-        session,
-        // inserts only - order doesn't matter
-        ordered: false
-      });
-    }
-    if (this.bucketParameters.length > 0) {
-      await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
-        session,
-        // inserts only - order doesn't matter
-        ordered: false
-      });
-    }
-    if (this.currentData.length > 0) {
-      await safeBulkWrite(db.current_data, this.currentData, {
-        session,
-        // may update and delete data within the same batch - order matters
-        ordered: true
-      });
-    }
-
-    logger.info(
-      `powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${
-        this.currentData.length
-      } updates, ${Math.round(this.currentSize / 1024)}kb. Last op_id: ${this.debugLastOpId}`
-    );
-
-    this.bucketData = [];
-    this.bucketParameters = [];
-    this.currentData = [];
-    this.currentSize = 0;
-    this.debugLastOpId = null;
-  }
-}

package/src/storage/implementation/config.ts

@@ -1,40 +0,0 @@
-import * as urijs from 'uri-js';
-
-export interface MongoConnectionConfig {
-  uri: string;
-  username?: string;
-  password?: string;
-  database?: string;
-}
-
-/**
- * Validate and normalize connection options.
- *
- * Returns destructured options.
- *
- * For use by both storage and mongo module.
- */
-export function normalizeMongoConfig(options: MongoConnectionConfig) {
-  let uri = urijs.parse(options.uri);
-
-  const database = options.database ?? uri.path?.substring(1) ?? '';
-
-  const userInfo = uri.userinfo?.split(':');
-
-  const username = options.username ?? userInfo?.[0];
-  const password = options.password ?? userInfo?.[1];
-
-  if (database == '') {
-    throw new Error(`database required`);
-  }
-
-  delete uri.userinfo;
-
-  return {
-    uri: urijs.serialize(uri),
-    database,
-
-    username,
-    password
-  };
-}

package/src/storage/implementation/db.ts

@@ -1,87 +0,0 @@
-import { storage } from '@powersync/service-core';
-import { configFile } from '@powersync/service-types';
-import * as mongo from 'mongodb';
-import * as db from '../../db/db-index.js';
-import { Lock } from '../../locks/MongoLockManager.js';
-import {
-  BucketDataDocument,
-  BucketParameterDocument,
-  CurrentDataDocument,
-  CustomWriteCheckpointDocument,
-  IdSequenceDocument,
-  InstanceDocument,
-  SourceTableDocument,
-  SyncRuleDocument,
-  WriteCheckpointDocument
-} from './models.js';
-
-export interface PowerSyncMongoOptions {
-  /**
-   * Optional - uses the database from the MongoClient connection URI if not specified.
-   */
-  database?: string;
-}
-
-export function createPowerSyncMongo(config: configFile.MongoStorageConfig) {
-  return new PowerSyncMongo(db.mongo.createMongoClient(config), { database: config.database });
-}
-
-export class PowerSyncMongo {
-  readonly current_data: mongo.Collection<CurrentDataDocument>;
-  readonly bucket_data: mongo.Collection<BucketDataDocument>;
-  readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
-  readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
-  readonly sync_rules: mongo.Collection<SyncRuleDocument>;
-  readonly source_tables: mongo.Collection<SourceTableDocument>;
-  readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
-  readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
-  readonly instance: mongo.Collection<InstanceDocument>;
-  readonly locks: mongo.Collection<Lock>;
-
-  readonly client: mongo.MongoClient;
-  readonly db: mongo.Db;
-
-  constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions) {
-    this.client = client;
-
-    const db = client.db(options?.database, {
-      ...storage.BSON_DESERIALIZE_OPTIONS
-    });
-    this.db = db;
-
-    this.current_data = db.collection<CurrentDataDocument>('current_data');
-    this.bucket_data = db.collection('bucket_data');
-    this.bucket_parameters = db.collection('bucket_parameters');
-    this.op_id_sequence = db.collection('op_id_sequence');
-    this.sync_rules = db.collection('sync_rules');
-    this.source_tables = db.collection('source_tables');
-    this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
-    this.write_checkpoints = db.collection('write_checkpoints');
-    this.instance = db.collection('instance');
-    this.locks = this.db.collection('locks');
-  }
-
-  /**
-   * Clear all collections.
-   */
-  async clear() {
-    await this.current_data.deleteMany({});
-    await this.bucket_data.deleteMany({});
-    await this.bucket_parameters.deleteMany({});
-    await this.op_id_sequence.deleteMany({});
-    await this.sync_rules.deleteMany({});
-    await this.source_tables.deleteMany({});
-    await this.write_checkpoints.deleteMany({});
-    await this.instance.deleteOne({});
-    await this.locks.deleteMany({});
-  }
-
-  /**
-   * Drop the entire database.
-   *
-   * Primarily for tests.
-   */
-  async drop() {
-    await this.db.dropDatabase();
-  }
-}