@powersync/service-module-mongodb 0.0.0-dev-20241219153510 → 0.0.0-dev-20250108073049

package/src/storage/implementation/MongoCompactor.ts

@@ -1,392 +0,0 @@
-import { logger } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
-import { AnyBulkWriteOperation, MaxKey, MinKey } from 'mongodb';
-import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey } from './models.js';
-import { cacheKey } from './OperationBatch.js';
-import { safeBulkWrite } from './util.js';
-
-interface CurrentBucketState {
-  /** Bucket name */
-  bucket: string;
-  /**
-   * Rows seen in the bucket, with the last op_id of each.
-   */
-  seen: Map<string, bigint>;
-  /**
-   * Estimated memory usage of the seen Map.
-   */
-  trackingSize: number;
-
-  /**
-   * Last (lowest) seen op_id that is not a PUT.
-   */
-  lastNotPut: bigint | null;
-
-  /**
-   * Number of REMOVE/MOVE operations seen since lastNotPut.
-   */
-  opsSincePut: number;
-}
-
-/**
- * Additional options, primarily for testing.
- */
-export interface MongoCompactOptions extends storage.CompactOptions {
-  /** Minimum of 2 */
-  clearBatchLimit?: number;
-  /** Minimum of 1 */
-  moveBatchLimit?: number;
-  /** Minimum of 1 */
-  moveBatchQueryLimit?: number;
-}
-
-const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
-const DEFAULT_MOVE_BATCH_LIMIT = 2000;
-const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
-
-/** This default is primarily for tests. */
-const DEFAULT_MEMORY_LIMIT_MB = 64;
-
-export class MongoCompactor {
-  private updates: AnyBulkWriteOperation<BucketDataDocument>[] = [];
-
-  private idLimitBytes: number;
-  private moveBatchLimit: number;
-  private moveBatchQueryLimit: number;
-  private clearBatchLimit: number;
-  private maxOpId: bigint | undefined;
-  private buckets: string[] | undefined;
-
-  constructor(
-    private db: PowerSyncMongo,
-    private group_id: number,
-    options?: MongoCompactOptions
-  ) {
-    this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
-    this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
-    this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
-    this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
-    this.maxOpId = options?.maxOpId;
-    this.buckets = options?.compactBuckets;
-  }
-
-  /**
-   * Compact buckets by converting operations into MOVE and/or CLEAR operations.
-   *
-   * See /docs/compacting-operations.md for details.
-   */
-  async compact() {
-    if (this.buckets) {
-      for (let bucket of this.buckets) {
-        // We can make this more efficient later on by iterating
-        // through the buckets in a single query.
-        // That makes batching more tricky, so we leave for later.
-        await this.compactInternal(bucket);
-      }
-    } else {
-      await this.compactInternal(undefined);
-    }
-  }
-
-  async compactInternal(bucket: string | undefined) {
-    const idLimitBytes = this.idLimitBytes;
-
-    let currentState: CurrentBucketState | null = null;
-
-    let bucketLower: string | MinKey;
-    let bucketUpper: string | MaxKey;
-
-    if (bucket == null) {
-      bucketLower = new MinKey();
-      bucketUpper = new MaxKey();
-    } else if (bucket.includes('[')) {
-      // Exact bucket name
-      bucketLower = bucket;
-      bucketUpper = bucket;
-    } else {
-      // Bucket definition name
-      bucketLower = `${bucket}[`;
-      bucketUpper = `${bucket}[\uFFFF`;
-    }
-
-    // Constant lower bound
-    const lowerBound: BucketDataKey = {
-      g: this.group_id,
-      b: bucketLower as string,
-      o: new MinKey() as any
-    };
-
-    // Upper bound is adjusted for each batch
-    let upperBound: BucketDataKey = {
-      g: this.group_id,
-      b: bucketUpper as string,
-      o: new MaxKey() as any
-    };
-
-    while (true) {
-      // Query one batch at a time, to avoid cursor timeouts
-      const batch = await this.db.bucket_data
-        .find(
-          {
-            _id: {
-              $gte: lowerBound,
-              $lt: upperBound
-            }
-          },
-          {
-            projection: {
-              _id: 1,
-              op: 1,
-              table: 1,
-              row_id: 1,
-              source_table: 1,
-              source_key: 1
-            },
-            limit: this.moveBatchQueryLimit,
-            sort: { _id: -1 },
-            singleBatch: true
-          }
-        )
-        .toArray();
-
-      if (batch.length == 0) {
-        // We've reached the end
-        break;
-      }
-
-      // Set upperBound for the next batch
-      upperBound = batch[batch.length - 1]._id;
-
-      for (let doc of batch) {
-        if (currentState == null || doc._id.b != currentState.bucket) {
-          if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
-            // Important to flush before clearBucket()
-            await this.flush();
-            logger.info(
-              `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
-            );
-
-            const bucket = currentState.bucket;
-            const clearOp = currentState.lastNotPut;
-            // Free memory before clearing bucket
-            currentState = null;
-            await this.clearBucket(bucket, clearOp);
-          }
-          currentState = {
-            bucket: doc._id.b,
-            seen: new Map(),
-            trackingSize: 0,
-            lastNotPut: null,
-            opsSincePut: 0
-          };
-        }
-
-        if (this.maxOpId != null && doc._id.o > this.maxOpId) {
-          continue;
-        }
-
-        let isPersistentPut = doc.op == 'PUT';
-
-        if (doc.op == 'REMOVE' || doc.op == 'PUT') {
-          const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`;
-          const targetOp = currentState.seen.get(key);
-          if (targetOp) {
-            // Will convert to MOVE, so don't count as PUT
-            isPersistentPut = false;
-
-            this.updates.push({
-              updateOne: {
-                filter: {
-                  _id: doc._id
-                },
-                update: {
-                  $set: {
-                    op: 'MOVE',
-                    target_op: targetOp
-                  },
-                  $unset: {
-                    source_table: 1,
-                    source_key: 1,
-                    table: 1,
-                    row_id: 1,
-                    data: 1
-                  }
-                }
-              }
-            });
-          } else {
-            if (currentState.trackingSize >= idLimitBytes) {
-              // Reached memory limit.
-              // Keep the highest seen values in this case.
-            } else {
-              // flatstr reduces the memory usage by flattening the string
-              currentState.seen.set(flatstr(key), doc._id.o);
-              // length + 16 for the string
-              // 24 for the bigint
-              // 50 for map overhead
-              // 50 for additional overhead
-              currentState.trackingSize += key.length + 140;
-            }
-          }
-        }
-
-        if (isPersistentPut) {
-          currentState.lastNotPut = null;
-          currentState.opsSincePut = 0;
-        } else if (doc.op != 'CLEAR') {
-          if (currentState.lastNotPut == null) {
-            currentState.lastNotPut = doc._id.o;
-          }
-          currentState.opsSincePut += 1;
-        }
-
-        if (this.updates.length >= this.moveBatchLimit) {
-          await this.flush();
-        }
-      }
-    }
-
-    await this.flush();
-    currentState?.seen.clear();
-    if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
-      logger.info(
-        `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
-      );
-      const bucket = currentState.bucket;
-      const clearOp = currentState.lastNotPut;
-      // Free memory before clearing bucket
-      currentState = null;
-      await this.clearBucket(bucket, clearOp);
-    }
-  }
-
-  private async flush() {
-    if (this.updates.length > 0) {
-      logger.info(`Compacting ${this.updates.length} ops`);
-      await safeBulkWrite(this.db.bucket_data, this.updates, {
-        // Order is not important.
-        // Since checksums are not affected, these operations can happen in any order,
-        // and it's fine if the operations are partially applied.
-        // Each individual operation is atomic.
-        ordered: false
-      });
-      this.updates = [];
-    }
-  }
-
-  /**
-   * Perform a CLEAR compact for a bucket.
-   *
-   * @param bucket bucket name
-   * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
-   */
-  private async clearBucket(bucket: string, op: bigint) {
-    const opFilter = {
-      _id: {
-        $gte: {
-          g: this.group_id,
-          b: bucket,
-          o: new MinKey() as any
-        },
-        $lte: {
-          g: this.group_id,
-          b: bucket,
-          o: op
-        }
-      }
-    };
-
-    const session = this.db.client.startSession();
-    try {
-      let done = false;
-      while (!done) {
-        // Do the CLEAR operation in batches, with each batch a separate transaction.
-        // The state after each batch is fully consistent.
-        // We need a transaction per batch to make sure checksums stay consistent.
-        await session.withTransaction(
-          async () => {
-            const query = this.db.bucket_data.find(opFilter, {
-              session,
-              sort: { _id: 1 },
-              projection: {
-                _id: 1,
-                op: 1,
-                checksum: 1,
-                target_op: 1
-              },
-              limit: this.clearBatchLimit
-            });
-            let checksum = 0;
-            let lastOpId: BucketDataKey | null = null;
-            let targetOp: bigint | null = null;
-            let gotAnOp = false;
-            for await (let op of query.stream()) {
-              if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
-                checksum = utils.addChecksums(checksum, op.checksum);
-                lastOpId = op._id;
-                if (op.op != 'CLEAR') {
-                  gotAnOp = true;
-                }
-                if (op.target_op != null) {
-                  if (targetOp == null || op.target_op > targetOp) {
-                    targetOp = op.target_op;
-                  }
-                }
-              } else {
-                throw new Error(`Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`);
-              }
-            }
-            if (!gotAnOp) {
-              done = true;
-              return;
-            }
-
-            logger.info(`Flushing CLEAR at ${lastOpId?.o}`);
-            await this.db.bucket_data.deleteMany(
-              {
-                _id: {
-                  $gte: {
-                    g: this.group_id,
-                    b: bucket,
-                    o: new MinKey() as any
-                  },
-                  $lte: lastOpId!
-                }
-              },
-              { session }
-            );
-
-            await this.db.bucket_data.insertOne(
-              {
-                _id: lastOpId!,
-                op: 'CLEAR',
-                checksum: checksum,
-                data: null,
-                target_op: targetOp
-              },
-              { session }
-            );
-          },
-          {
-            writeConcern: { w: 'majority' },
-            readConcern: { level: 'snapshot' }
-          }
-        );
-      }
-    } finally {
-      await session.endSession();
-    }
-  }
-}
-
-/**
- * Flattens string to reduce memory usage (around 320 bytes -> 120 bytes),
- * at the cost of some upfront CPU usage.
- *
- * From: https://github.com/davidmarkclements/flatstr/issues/8
- */
-function flatstr(s: string) {
-  s.match(/\n/g);
-  return s;
-}

package/src/storage/implementation/MongoIdSequence.ts

@@ -1,24 +0,0 @@
-/**
- * Manages op_id or similar sequence in memory.
- *
- * This is typically used within a transaction, with the last value persisted
- * at the end of the transaction.
- */
-export class MongoIdSequence {
-  private _last: bigint;
-
-  constructor(last: bigint) {
-    if (typeof last != 'bigint') {
-      throw new Error(`BigInt required, got ${last} ${typeof last}`);
-    }
-    this._last = last;
-  }
-
-  next() {
-    return ++this._last;
-  }
-
-  last() {
-    return this._last;
-  }
-}

package/src/storage/implementation/MongoPersistedSyncRules.ts

@@ -1,16 +0,0 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-
-import { storage } from '@powersync/service-core';
-
-export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
-  public readonly slot_name: string;
-
-  constructor(
-    public readonly id: number,
-    public readonly sync_rules: SqlSyncRules,
-    public readonly checkpoint_lsn: string | null,
-    slot_name: string | null
-  ) {
-    this.slot_name = slot_name ?? `powersync_${id}`;
-  }
-}

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,49 +0,0 @@
-import { storage } from '@powersync/service-core';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import * as mongo from 'mongodb';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
-import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
-import { PowerSyncMongo } from './db.js';
-import { SyncRuleDocument } from './models.js';
-
-export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-
-  public current_lock: MongoSyncRulesLock | null = null;
-
-  constructor(
-    private db: PowerSyncMongo,
-    doc: mongo.WithId<SyncRuleDocument>
-  ) {
-    this.id = doc._id;
-    this.sync_rules_content = doc.content;
-    this.last_checkpoint_lsn = doc.last_checkpoint_lsn;
-    // Handle legacy values
-    this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
-    this.last_fatal_error = doc.last_fatal_error;
-    this.last_checkpoint_ts = doc.last_checkpoint_ts;
-    this.last_keepalive_ts = doc.last_keepalive_ts;
-  }
-
-  parsed(options: storage.ParseSyncRulesOptions) {
-    return new MongoPersistedSyncRules(
-      this.id,
-      SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      this.last_checkpoint_lsn,
-      this.slot_name
-    );
-  }
-
-  async lock() {
-    const lock = await MongoSyncRulesLock.createLock(this.db, this);
-    this.current_lock = lock;
-    return lock;
-  }
-}

package/src/storage/implementation/MongoStorageProvider.ts

@@ -1,42 +0,0 @@
-import { logger } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
-import { configFile } from '@powersync/service-types';
-import * as db from '../../db/db-index.js';
-import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { PowerSyncMongo } from './db.js';
-
-export const MONGO_STORAGE_TYPE = 'mongodb';
-export class MongoStorageProvider implements storage.BucketStorageProvider {
-  get type() {
-    return MONGO_STORAGE_TYPE;
-  }
-
-  async getStorage(options: storage.GetStorageOptions): Promise<storage.ActiveStorage> {
-    const { resolvedConfig } = options;
-
-    const { storage } = resolvedConfig;
-    if (storage.type != MONGO_STORAGE_TYPE) {
-      // This should not be reached since the generation should be managed externally.
-      throw new Error(
-        `Cannot create MongoDB bucket storage with provided config ${storage.type} !== ${MONGO_STORAGE_TYPE}`
-      );
-    }
-
-    const decodedConfig = configFile.MongoStorageConfig.decode(storage as any);
-    const client = db.mongo.createMongoClient(decodedConfig);
-
-    const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
-
-    return {
-      storage: new MongoBucketStorage(database, {
-        // TODO currently need the entire resolved config due to this
-        slot_name_prefix: resolvedConfig.slot_name_prefix
-      }),
-      shutDown: () => client.close(),
-      tearDown: () => {
-        logger.info(`Tearing down storage: ${database.db.namespace}...`);
-        return database.db.dropDatabase();
-      }
-    } satisfies storage.ActiveStorage;
-  }
-}