@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts

@@ -0,0 +1,112 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
+import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
+import { CommonSourceTableDocument } from '../models.js';
+import {
+  BucketDataDocumentV3,
+  BucketParameterDocumentV3,
+  BucketStateDocumentV3,
+  CurrentDataDocumentV3,
+  SourceTableDocumentV3
+} from './models.js';
+
+export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
+  sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
+    const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
+    return this.db.collection<CurrentDataDocumentV3>(collectionName);
+  }
+
+  async listSourceRecordCollectionsV3(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
+    return this.listCollectionsByPrefix<CurrentDataDocumentV3>(`source_records_${replicationStreamId}_`);
+  }
+
+  async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
+    await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex(
+      {
+        pending_delete: 1
+      },
+      {
+        partialFilterExpression: { pending_delete: { $exists: true } },
+        name: 'pending_delete'
+      }
+    );
+  }
+
+  commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument> {
+    return this.sourceTablesV3(replicationStreamId) as mongo.Collection<CommonSourceTableDocument>;
+  }
+
+  bucketStateV3(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
+    return this.db.collection(`bucket_state_${replicationStreamId}`);
+  }
+
+  parameterIndexV3(
+    replicationStreamId: number,
+    indexId: ParameterIndexId
+  ): mongo.Collection<BucketParameterDocumentV3> {
+    return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
+  }
+
+  sourceTablesV3(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
+    return this.db.collection<SourceTableDocumentV3>(this.sourceTableCollectionName(replicationStreamId));
+  }
+
+  async initializeStreamStorage(replicationStreamId: number) {
+    const sourceTables = this.sourceTablesV3(replicationStreamId);
+    const bucketState = this.bucketStateV3(replicationStreamId);
+    await sourceTables.createIndex(
+      {
+        connection_id: 1,
+        schema_name: 1,
+        table_name: 1,
+        relation_id: 1
+      },
+      {
+        name: 'source_lookup'
+      }
+    );
+    await sourceTables.createIndex(
+      {
+        latest_pending_delete: 1
+      },
+      {
+        partialFilterExpression: { latest_pending_delete: { $exists: true } },
+        name: 'latest_pending_delete'
+      }
+    );
+    await bucketState.createIndex(
+      {
+        last_op: 1
+      },
+      { name: 'bucket_updates', unique: true }
+    );
+    await bucketState.createIndex(
+      {
+        'estimate_since_compact.count': -1
+      },
+      { name: 'dirty_count' }
+    );
+  }
+
+  bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) {
+    return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
+  }
+
+  listBucketDataCollectionsV3(replicationStreamId: number) {
+    return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
+  }
+
+  async listParameterIndexCollectionsV3(
+    replicationStreamId: number
+  ): Promise<{ collection: mongo.Collection<BucketParameterDocumentV3>; indexId: ParameterIndexId }[]> {
+    const prefix = `parameter_index_${replicationStreamId}_`;
+    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+
+    return collections
+      .filter((collection) => collection.name.startsWith(prefix))
+      .map((collection) => ({
+        collection: this.db.collection<BucketParameterDocumentV3>(collection.name),
+        indexId: collection.name.slice(prefix.length)
+      }));
+  }
+}

package/src/storage/implementation/v3/models.ts

@@ -0,0 +1,96 @@
+import { InternalOpId } from '@powersync/service-core';
+import * as bson from 'bson';
+import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
+import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
+import {
+  BucketDataDocumentBase,
+  BucketDataKey,
+  BucketParameterDocumentBase,
+  BucketStateDocumentBase,
+  CurrentBucket,
+  ReplicaId,
+  SourceTableDocument,
+  SourceTableKey,
+  TaggedBucketParameterDocument
+} from '../models.js';
+
+export interface CurrentBucketV3 extends CurrentBucket {
+  def: BucketDefinitionId;
+}
+
+export interface RecordedLookupV3 {
+  i: ParameterIndexId;
+  l: bson.Binary;
+}
+
+export interface CurrentDataDocumentV3 {
+  _id: ReplicaId;
+  data: bson.Binary | null;
+  buckets: CurrentBucketV3[];
+  lookups: RecordedLookupV3[];
+  /**
+   * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete.
+   *
+   * This must only be set if buckets = [], lookups = [].
+   */
+  pending_delete?: bigint;
+}
+
+export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<SourceTableKey> {}
+
+export type BucketDataKeyV3 = BucketDataKey;
+
+export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
+  _id: BucketDataKeyV3;
+}
+
+export function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3 {
+  const { bucketKey, o } = document;
+  return {
+    _id: {
+      b: bucketKey.bucket,
+      o: o
+    },
+    // List fields directly, so that we don't accidentally persist any unknown fields
+    op: document.op,
+    source_table: document.source_table,
+    source_key: document.source_key,
+    table: document.table,
+    row_id: document.row_id,
+    checksum: document.checksum,
+    data: document.data,
+    target_op: document.target_op
+  };
+}
+
+export function loadBucketDataDocumentV3(
+  context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
+  doc: BucketDataDocumentV3
+): BucketDataDoc {
+  const { _id, ...rest } = doc;
+  return {
+    bucketKey: {
+      ...context,
+      bucket: _id.b
+    },
+    o: _id.o,
+    ...rest
+  };
+}
+
+export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 {
+  const { index: _index, ...rest } = document;
+  return rest as BucketParameterDocumentV3;
+}
+
+export interface SourceTableDocumentV3 extends SourceTableDocument {
+  bucket_data_source_ids: BucketDefinitionId[];
+  parameter_lookup_source_ids: ParameterIndexId[];
+  latest_pending_delete?: InternalOpId | undefined;
+}
+
+export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
+  _id: BucketStateDocumentBase['_id'] & {
+    d: BucketDefinitionId;
+  };
+}

package/src/storage/storage-index.ts

@@ -1,14 +1,17 @@
+export * as test_utils from '../utils/test-utils.js';
+export * from '../utils/util.js';
+export * from './implementation/BucketDefinitionMapping.js';
+export * from './implementation/common/PersistedBatch.js';
+export * from './implementation/createMongoSyncBucketStorage.js';
 export * from './implementation/db.js';
 export * from './implementation/models.js';
-export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
-export * from './implementation/MongoSyncBucketStorage.js';
 export * from './implementation/MongoSyncRulesLock.js';
 export * from './implementation/OperationBatch.js';
-export * from './implementation/PersistedBatch.js';
-export * from '../utils/util.js';
+export * from './implementation/v1/models.js';
+export * from './implementation/v3/models.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';
-export * as test_utils from '../utils/test-utils.js';

package/src/utils/util.ts

@@ -1,10 +1,12 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { storage, utils } from '@powersync/service-core';
 import * as bson from 'bson';
 import * as crypto from 'crypto';
+import * as timers from 'node:timers/promises';
 import * as uuid from 'uuid';
-import { mongo } from '@powersync/lib-service-mongodb';
-import { storage, utils } from '@powersync/service-core';
-import { ServiceAssertionError } from '@powersync/lib-services-framework';
-import { BucketDataDocument } from '../storage/implementation/models.js';
+import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
 
 export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
   let filter = {
@@ -69,10 +71,10 @@ export async function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Promi
   }
 }
 
-export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
+export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry {
   if (row.op == 'PUT' || row.op == 'REMOVE') {
     return {
-      op_id: utils.internalToExternalOpId(row._id.o),
+      op_id: utils.internalToExternalOpId(row.o),
       op: row.op,
       object_type: row.table,
       object_id: row.row_id,
@@ -84,7 +86,7 @@ export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
     // MOVE, CLEAR
 
     return {
-      op_id: utils.internalToExternalOpId(row._id.o),
+      op_id: utils.internalToExternalOpId(row.o),
       op: row.op,
       checksum: Number(row.checksum)
     };
@@ -129,6 +131,33 @@ export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.
   }
 }
 
+export async function retryOnMongoMaxTimeMSExpired<T>(
+  operation: () => Promise<T>,
+  options: {
+    signal?: AbortSignal;
+    abortMessage?: string;
+    retryDelayMs: number;
+    onRetry?: (retryCount: number) => void;
+  }
+): Promise<T> {
+  let retryCount = 0;
+  while (true) {
+    if (options.signal?.aborted) {
+      throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
+    }
+    try {
+      return await operation();
+    } catch (e) {
+      if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
+        throw e;
+      }
+      retryCount += 1;
+      options.onRetry?.(retryCount);
+      await timers.setTimeout(options.retryDelayMs);
+    }
+  }
+}
+
 export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
   query: mongo.Filter<T>,
   collection: mongo.Collection<T>,

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -1,5 +1,99 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`sync - mongodb > storage v1 > can override priority when subscribing to stream 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#todos|0["a"]",
+          "checksum": -1712802421,
+          "count": 1,
+          "priority": 0,
+          "subscriptions": [
+            {
+              "sub": 0,
+            },
+            {
+              "sub": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#todos|0["b"]",
+          "checksum": -1291414318,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "sub": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": false,
+          "name": "todos",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["a"]",
+      "data": [
+        {
+          "checksum": 2582164875,
+          "data": "{"id":"a","description":"Test 1"}",
+          "object_id": "a",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 0,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["b"]",
+      "data": [
+        {
+          "checksum": 3003552978,
+          "data": "{"id":"b","description":"Test 2"}",
+          "object_id": "b",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > storage v1 > compacting data - invalidate checkpoint 1`] = `
 [
   {
@@ -1095,6 +1189,100 @@ exports[`sync - mongodb > storage v1 > sync updates to parameter query only 2`]
 ]
 `;
 
+exports[`sync - mongodb > storage v2 > can override priority when subscribing to stream 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#todos|0["a"]",
+          "checksum": -1712802421,
+          "count": 1,
+          "priority": 0,
+          "subscriptions": [
+            {
+              "sub": 0,
+            },
+            {
+              "sub": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#todos|0["b"]",
+          "checksum": -1291414318,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "sub": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": false,
+          "name": "todos",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["a"]",
+      "data": [
+        {
+          "checksum": 2582164875,
+          "data": "{"id":"a","description":"Test 1"}",
+          "object_id": "a",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 0,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["b"]",
+      "data": [
+        {
+          "checksum": 3003552978,
+          "data": "{"id":"b","description":"Test 2"}",
+          "object_id": "b",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > storage v2 > compacting data - invalidate checkpoint 1`] = `
 [
   {
@@ -2190,6 +2378,100 @@ exports[`sync - mongodb > storage v2 > sync updates to parameter query only 2`]
 ]
 `;
 
+exports[`sync - mongodb > storage v3 > can override priority when subscribing to stream 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#todos|0["a"]",
+          "checksum": -1712802421,
+          "count": 1,
+          "priority": 0,
+          "subscriptions": [
+            {
+              "sub": 0,
+            },
+            {
+              "sub": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#todos|0["b"]",
+          "checksum": -1291414318,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "sub": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": false,
+          "name": "todos",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["a"]",
+      "data": [
+        {
+          "checksum": 2582164875,
+          "data": "{"id":"a","description":"Test 1"}",
+          "object_id": "a",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 0,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["b"]",
+      "data": [
+        {
+          "checksum": 3003552978,
+          "data": "{"id":"b","description":"Test 2"}",
+          "object_id": "b",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > storage v3 > compacting data - invalidate checkpoint 1`] = `
 [
   {

package/test/src/connection-report-storage.test.ts

@@ -1,8 +1,8 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js';
+import { MongoReportStorage } from '@module/storage/MongoReportStorage.js';
 import { register, ReportUserData } from '@powersync/service-core-tests';
 import { event_types } from '@powersync/service-types';
-import { MongoReportStorage } from '@module/storage/MongoReportStorage.js';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js';
 
 const userData = register.REPORT_TEST_USERS;
 const dates = register.REPORT_TEST_DATES;

package/test/src/setup.ts

@@ -1,6 +1,6 @@
 import { container } from '@powersync/lib-services-framework';
-import { beforeAll, beforeEach } from 'vitest';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { beforeAll, beforeEach } from 'vitest';
 
 beforeAll(async () => {
   // Executes for every test file

package/test/src/storage.test.ts

@@ -1,8 +1,8 @@
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 import { env } from './env.js';
-import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 for (let storageVersion of TEST_STORAGE_VERSIONS) {
   describe(`Mongo Sync Bucket Storage - Parameters - v${storageVersion}`, () =>