@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -1,12 +1,12 @@
 import crypto from 'crypto';
 
-import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
-import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
+import { VersionedPowerSyncMongo } from './db.js';
 
 /**
- * Manages a lock on a sync rules document, so that only one process
- * replicates those sync rules at a time.
+ * Manages a lock on a replication stream document, so that only one process
+ * processes that replication stream at a time.
  */
 export class MongoSyncRulesLock implements storage.ReplicationLock {
   private readonly refreshInterval: NodeJS.Timeout;
@@ -38,29 +38,27 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
       if (heldLock?.lock?.expires_at) {
         throw new ServiceError(
           ErrorCode.PSYNC_S1003,
-          `Sync rules ${sync_rules.id} have been locked by another process for replication, expiring at ${heldLock.lock.expires_at.toISOString()}.`
+          `Replication stream is locked by another process, standing by. Lock expiring at ${heldLock.lock.expires_at.toISOString()}.`
         );
       } else {
-        throw new ServiceError(
-          ErrorCode.PSYNC_S1003,
-          `Sync rules ${sync_rules.id} have been locked by another process for replication.`
-        );
+        throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
       }
     }
-    logger.info(`Sync rules ${sync_rules.id} has been locked for replication with lock ID ${lockId}.`);
-    return new MongoSyncRulesLock(db, sync_rules.id, lockId);
+    sync_rules.logger.info(`Locked replication stream for processing`);
+    return new MongoSyncRulesLock(db, sync_rules.id, lockId, sync_rules.logger);
   }
 
   constructor(
     private db: VersionedPowerSyncMongo,
     public sync_rules_id: number,
-    private lock_id: string
+    private lock_id: string,
+    private logger: Logger
   ) {
     this.refreshInterval = setInterval(async () => {
       try {
         await this.refresh();
       } catch (e) {
-        logger.error('Failed to refresh lock', e);
+        this.logger.error('Failed to refresh lock', e);
         clearInterval(this.refreshInterval);
       }
     }, 30_130);
@@ -79,7 +77,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     );
     if (result.modifiedCount == 0) {
       // Log and ignore
-      logger.warn(`Lock already released: ${this.sync_rules_id}/${this.lock_id}`);
+      this.logger.warn(`Lock already released: ${this.sync_rules_id}/${this.lock_id}`);
     }
   }
 

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -1,7 +1,7 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import * as framework from '@powersync/lib-services-framework';
 import { GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
-import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
+import { VersionedPowerSyncMongo } from './db.js';
 
 export type MongoCheckpointAPIOptions = {
   db: VersionedPowerSyncMongo;
@@ -56,7 +56,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     switch (this.writeCheckpointMode) {
       case storage.WriteCheckpointMode.CUSTOM:
         if (false == 'sync_rules_id' in filters) {
-          throw new framework.ServiceAssertionError(`Sync rules ID is required for custom Write Checkpoint filtering`);
+          throw new framework.ServiceAssertionError(
+            `Replication stream ID is required for custom Write Checkpoint filtering`
+          );
         }
         return this.lastCustomWriteCheckpoint(filters);
       case storage.WriteCheckpointMode.MANAGED:

package/src/storage/implementation/OperationBatch.ts

@@ -2,7 +2,7 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
 import { storage } from '@powersync/service-core';
-import { mongoTableId } from '../storage-index.js';
+import { mongoTableId } from '../../utils/util.js';
 
 /**
  * Maximum number of operations in a batch.

package/src/storage/implementation/common/BucketDataDoc.ts

@@ -0,0 +1,37 @@
+import { InternalOpId } from '@powersync/service-core';
+import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { BucketDataProperties } from '../models.js';
+
+/**
+ * Full context identifying a bucket.
+ */
+export interface BucketKey {
+  /**
+   * Also referred to as g / group_id.
+   */
+  replicationStreamId: number;
+  /**
+   * Bucket definition id, '0' for storage V1.
+   */
+  definitionId: BucketDefinitionId;
+  /**
+   * Bucket name.
+   */
+  bucket: string;
+}
+
+/**
+ * In-memory bucket data document.
+ *
+ * This is converted to/from BucketDataDocumentV1 / BucketDataDocumentV3 for storage.
+ */
+export interface BucketDataDoc extends BucketDataProperties {
+  /**
+   * Identifies the bucket for this document.
+   */
+  bucketKey: BucketKey;
+  /**
+   * op_id
+   */
+  o: InternalOpId;
+}

package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts

@@ -0,0 +1,15 @@
+import { InternalOpId } from '@powersync/service-core';
+import * as bson from 'bson';
+import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
+import type { VersionedPowerSyncMongo } from '../db.js';
+
+export interface MongoSyncBucketStorageContext<TDb extends VersionedPowerSyncMongo = VersionedPowerSyncMongo> {
+  db: TDb;
+  group_id: number;
+  mapping: BucketDefinitionMapping;
+}
+
+export interface MongoSyncBucketStorageCheckpoint {
+  checkpoint: InternalOpId;
+  snapshotTime: bson.Timestamp;
+}

package/src/storage/implementation/common/PersistedBatch.ts

@@ -0,0 +1,364 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { BucketDataSource, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+
+import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework';
+import { InternalOpId, storage, utils } from '@powersync/service-core';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
+import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
+import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
+import { MongoIdSequence } from '../MongoIdSequence.js';
+import type { VersionedPowerSyncMongo } from '../db.js';
+import { TaggedBucketParameterDocument } from '../models.js';
+import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
+import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js';
+
+/**
+ * Maximum size of operations we write in a single transaction.
+ *
+ * It's tricky to find the exact limit, but from experience, over 100MB
+ * can cause an error:
+ * > transaction is too large and will not fit in the storage engine cache
+ *
+ * Additionally, unbounded size here can balloon our memory usage in some edge
+ * cases.
+ *
+ * When we reach this threshold, we commit the transaction and start a new one.
+ */
+const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
+
+/**
+ * Limit number of documents to write in a single transaction.
+ *
+ * This has an effect on error message size in some cases.
+ */
+const MAX_TRANSACTION_DOC_COUNT = 2_000;
+
+export interface SaveBucketDataOptions {
+  op_seq: MongoIdSequence;
+  sourceKey: storage.ReplicaId;
+  table: storage.SourceTable;
+  evaluated: EvaluatedRow[];
+  before_buckets: SourceRecordBucketState[];
+}
+
+export interface SaveParameterDataOptions {
+  op_seq: MongoIdSequence;
+  sourceKey: storage.ReplicaId;
+  sourceTable: storage.SourceTable;
+  evaluated: EvaluatedParameters[];
+  existing_lookups: SourceRecordLookupState[];
+}
+
+export interface UpsertCurrentDataOptions {
+  sourceTableId: bson.ObjectId;
+  replicaId: storage.ReplicaId;
+  data: bson.Binary | null;
+  buckets: SourceRecordBucketState[];
+  lookups: SourceRecordLookupState[];
+}
+
+export interface PersistedBatchOptions {
+  logger?: Logger;
+}
+
+/**
+ * Keeps track of bulkwrite operations within a transaction.
+ *
+ * There may be multiple of these batches per transaction, but it may not span
+ * multiple transactions.
+ */
+export abstract class PersistedBatch {
+  logger: Logger;
+  bucketData: BucketDataDoc[] = [];
+  bucketParameters: TaggedBucketParameterDocument[] = [];
+  bucketStates: Map<string, BucketStateUpdate> = new Map();
+
+  /**
+   * For debug logging only.
+   */
+  debugLastOpId: InternalOpId | null = null;
+
+  /**
+   * Very rough estimate of transaction size.
+   */
+  currentSize = 0;
+
+  constructor(
+    protected readonly db: VersionedPowerSyncMongo,
+    protected readonly group_id: number,
+    protected readonly mapping: BucketDefinitionMapping,
+    writtenSize: number,
+    options?: PersistedBatchOptions
+  ) {
+    this.currentSize = writtenSize;
+    this.logger = options?.logger ?? defaultLogger;
+  }
+
+  saveBucketData(options: SaveBucketDataOptions) {
+    const remaining_buckets = new Map<string, SaveBucketDataOptions['before_buckets'][number]>();
+    for (let bucket of options.before_buckets) {
+      const mapped: SourceRecordBucketState = {
+        bucket: bucket.bucket,
+        definitionId: this.checkDefinitionId(bucket.definitionId),
+        id: bucket.id,
+        table: bucket.table
+      };
+      remaining_buckets.set(currentBucketKey(mapped), mapped);
+    }
+
+    const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)));
+
+    for (const evaluated of options.evaluated) {
+      const definitionId = this.getBucketDefinitionId(evaluated.source);
+      const key = currentBucketKey({
+        definitionId: definitionId,
+        bucket: evaluated.bucket,
+        table: evaluated.table,
+        id: evaluated.id
+      });
+
+      const recordData = JSONBig.stringify(evaluated.data);
+      const checksum = utils.hashData(evaluated.table, evaluated.id, recordData);
+      if (recordData.length > MAX_ROW_SIZE) {
+        this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`);
+        continue;
+      }
+
+      remaining_buckets.delete(key);
+      const byteEstimate = recordData.length + 200;
+      this.currentSize += byteEstimate;
+
+      const op_id = options.op_seq.next();
+      this.debugLastOpId = op_id;
+
+      this.addBucketDataPut({
+        bucketKey: {
+          bucket: evaluated.bucket,
+          definitionId: definitionId,
+          replicationStreamId: this.group_id
+        },
+        op_id,
+        bucket: evaluated.bucket,
+        sourceTableId: options.table.id,
+        sourceKey: options.sourceKey,
+        table: evaluated.table,
+        rowId: evaluated.id,
+        checksum: BigInt(checksum),
+        data: recordData
+      });
+      this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate);
+    }
+
+    for (let bucket of remaining_buckets.values()) {
+      const definitionId = bucket.definitionId!;
+      const op_id = options.op_seq.next();
+      this.debugLastOpId = op_id;
+
+      this.addBucketDataRemove({
+        bucketKey: {
+          replicationStreamId: this.group_id,
+          definitionId,
+          bucket: bucket.bucket
+        },
+        op_id,
+        sourceTableId: options.table.id,
+        sourceKey: options.sourceKey,
+        table: bucket.table,
+        rowId: bucket.id,
+        checksum: dchecksum
+      });
+      this.currentSize += 200;
+      this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
+    }
+  }
+
+  abstract saveParameterData(data: SaveParameterDataOptions): void;
+
+  abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void;
+
+  abstract softDeleteCurrentData(
+    sourceTableId: bson.ObjectId,
+    replicaId: storage.ReplicaId,
+    checkpointGreaterThan: bigint
+  ): void;
+
+  abstract upsertCurrentData(values: UpsertCurrentDataOptions): void;
+
+  protected abstract get currentDataCount(): number;
+
+  protected abstract flushBucketData(session: mongo.ClientSession): Promise<void>;
+
+  protected abstract flushBucketParameters(session: mongo.ClientSession): Promise<void>;
+
+  protected abstract flushCurrentData(session: mongo.ClientSession): Promise<void>;
+
+  protected abstract flushBucketStates(session: mongo.ClientSession): Promise<void>;
+
+  protected abstract resetCurrentData(): void;
+
+  protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
+  protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
+
+  protected get bucketDataCount(): number {
+    return this.bucketData.length;
+  }
+
+  protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number) {
+    const key = `${definitionId ?? ''}:${bucket}`;
+    let existingState = this.bucketStates.get(key);
+    if (existingState) {
+      existingState.lastOp = op_id;
+      existingState.incrementCount += 1;
+      existingState.incrementBytes += bytes;
+    } else {
+      this.bucketStates.set(key, {
+        definitionId,
+        bucket,
+        lastOp: op_id,
+        incrementCount: 1,
+        incrementBytes: bytes
+      });
+    }
+  }
+
+  protected addBucketDataPut(options: {
+    op_id: InternalOpId;
+    bucketKey: BucketKey;
+    bucket: string;
+    sourceTableId: storage.SourceTable['id'];
+    sourceKey: storage.ReplicaId;
+    table: string;
+    rowId: string;
+    checksum: bigint;
+    data: string;
+  }) {
+    this.bucketData.push({
+      bucketKey: options.bucketKey,
+      o: options.op_id,
+      op: 'PUT',
+      source_table: mongoTableId(options.sourceTableId),
+      source_key: options.sourceKey,
+      table: options.table,
+      row_id: options.rowId,
+      checksum: options.checksum,
+      data: options.data
+    });
+  }
+
+  protected addBucketDataRemove(options: {
+    op_id: InternalOpId;
+    bucketKey: BucketKey;
+    sourceTableId: storage.SourceTable['id'];
+    sourceKey: storage.ReplicaId;
+    table: string;
+    rowId: string;
+    checksum: bigint;
+  }) {
+    this.bucketData.push({
+      bucketKey: options.bucketKey,
+      o: options.op_id,
+      op: 'REMOVE',
+      source_table: mongoTableId(options.sourceTableId),
+      source_key: options.sourceKey,
+      table: options.table,
+      row_id: options.rowId,
+      checksum: options.checksum,
+      data: null
+    });
+  }
+
+  shouldFlushTransaction() {
+    return (
+      this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
+      this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT ||
+      this.currentDataCount >= MAX_TRANSACTION_DOC_COUNT ||
+      this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
+    );
+  }
+
+  async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
+    const startAt = performance.now();
+    let flushedSomething = false;
+    if (this.bucketDataCount > 0) {
+      flushedSomething = true;
+      await this.flushBucketData(session);
+    }
+    if (this.bucketParameters.length > 0) {
+      flushedSomething = true;
+      await this.flushBucketParameters(session);
+    }
+    if (this.currentDataCount > 0) {
+      flushedSomething = true;
+      await this.flushCurrentData(session);
+    }
+
+    if (this.bucketStates.size > 0) {
+      flushedSomething = true;
+      await this.flushBucketStates(session);
+    }
+
+    if (flushedSomething) {
+      const duration = Math.round(performance.now() - startAt);
+      if (options?.oldestUncommittedChange != null) {
+        const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
+
+        this.logger.info(
+          `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${
+            this.currentDataCount
+          } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
+          {
+            flushed: {
+              duration: duration,
+              size: this.currentSize,
+              bucket_data_count: this.bucketDataCount,
+              parameter_data_count: this.bucketParameters.length,
+              current_data_count: this.currentDataCount,
+              replication_lag_seconds: replicationLag
+            }
+          }
+        );
+      } else {
+        this.logger.info(
+          `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${
+            this.currentDataCount
+          } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
+          {
+            flushed: {
+              duration: duration,
+              size: this.currentSize,
+              bucket_data_count: this.bucketDataCount,
+              parameter_data_count: this.bucketParameters.length,
+              current_data_count: this.currentDataCount
+            }
+          }
+        );
+      }
+    }
+
+    const stats = {
+      bucketDataCount: this.bucketDataCount,
+      parameterDataCount: this.bucketParameters.length,
+      currentDataCount: this.currentDataCount,
+      flushedAny: flushedSomething
+    };
+
+    this.bucketData = [];
+    this.bucketParameters = [];
+    this.resetCurrentData();
+    this.bucketStates.clear();
+    this.currentSize = 0;
+    this.debugLastOpId = null;
+
+    return stats;
+  }
+}
+
+export interface BucketStateUpdate {
+  definitionId: BucketDefinitionId | null;
+  bucket: string;
+  lastOp: InternalOpId;
+  incrementCount: number;
+  incrementBytes: number;
+}

package/src/storage/implementation/common/SingleBucketStore.ts

@@ -0,0 +1,63 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { InternalOpId } from '@powersync/service-core';
+import { BucketDataProperties } from '../models.js';
+import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
+
+const GENERIC_ID = Symbol('BucketDataDocumentGenericId');
+export type BucketDataDocumentGenericId = {
+  b: string;
+  o: InternalOpId;
+  // Hack to ensure this can't be constructed directly
+  [GENERIC_ID]: true;
+};
+
+/**
+ * This document is never actually constructed - we use it as a "virtual" type.
+ *
+ * The actual implementations are BucketDataDocumentV1 or BucketDataDocumentV3.
+ * They don't fully satisfy this interface, but this works to share common implementations.
+ *
+ * The idea is that we can have a common implementation between V1 & V3, using this type,
+ * and operate on MongoDB collections.
+ *
+ * This interface serves two primary purposes:
+ * 1. Captures properties that exist on both V1 and V3 storage models.
+ * 2. Gives a common reference when querying or modifying collections.
+ *
+ * Generics would've been ideal, but they don't play well with MongoDB collections.
+ */
+export interface BucketDataDocumentGeneric extends BucketDataProperties {
+  _id: BucketDataDocumentGenericId;
+}
+
+/**
+ * Represent read/write access for a single bucket.
+ *
+ * This does not implement the actual collection operations, but supports the required conversions
+ * between in-memory BucketDataDoc and the specific storage formats.
+ */
+export interface SingleBucketStore {
+  readonly key: BucketKey;
+
+  readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
+  docId(o: InternalOpId): BucketDataDocumentGenericId;
+  readonly minId: BucketDataDocumentGenericId;
+  readonly maxId: BucketDataDocumentGenericId;
+
+  /**
+   * Convert in-memory document -> persisted document.
+   */
+  toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric;
+
+  /**
+   * Convert persisted document -> in-memory document.
+   */
+  fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc;
+
+  /**
+   * Convert partial persisted document -> partial in-memory document.
+   */
+  fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
+    doc: Pick<BucketDataDocumentGeneric, '_id' | T>
+  ): Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
+}

package/src/storage/implementation/common/SourceRecordStore.ts

@@ -0,0 +1,49 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
+
+export interface SourceRecordLookupEntry {
+  sourceTableId: bson.ObjectId;
+  replicaId: storage.ReplicaId;
+}
+
+export interface SourceRecordBucketState {
+  definitionId: BucketDefinitionId | null;
+  bucket: string;
+  table: string;
+  id: string;
+}
+
+export interface SourceRecordLookupState {
+  indexId: ParameterIndexId | null;
+  lookup: bson.Binary;
+}
+
+export interface LoadedSourceRecord {
+  sourceTableId: bson.ObjectId;
+  replicaId: storage.ReplicaId;
+  data: bson.Binary | null;
+  buckets: SourceRecordBucketState[];
+  lookups: SourceRecordLookupState[];
+  cacheKey: string;
+}
+
+export interface SourceRecordStore {
+  mapEvaluatedBuckets(evaluated: EvaluatedRow[]): SourceRecordBucketState[];
+  mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[];
+  loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise<Map<string, number>>;
+  loadDocuments(
+    session: mongo.ClientSession,
+    entries: SourceRecordLookupEntry[],
+    idsOnly: boolean
+  ): Promise<Map<string, LoadedSourceRecord>>;
+  loadTruncateBatch(
+    session: mongo.ClientSession,
+    sourceTableId: bson.ObjectId,
+    limit: number
+  ): Promise<LoadedSourceRecord[]>;
+  postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise<void>;
+}