@powersync/service-module-mongodb 0.0.0-dev-20241219153510 → 0.0.0-dev-20250108073049

package/src/storage/implementation/models.ts

@@ -1,161 +0,0 @@
-import { storage } from '@powersync/service-core';
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-
-/**
- * Replica id uniquely identifying a row on the source database.
- *
- * Can be any value serializable to BSON.
- *
- * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
- */
-export type ReplicaId = bson.UUID | bson.Document | any;
-
-export interface SourceKey {
-  /** group_id */
-  g: number;
-  /** source table id */
-  t: bson.ObjectId;
-  /** source key */
-  k: ReplicaId;
-}
-
-export interface BucketDataKey {
-  /** group_id */
-  g: number;
-  /** bucket name */
-  b: string;
-  /** op_id */
-  o: bigint;
-}
-
-export interface CurrentDataDocument {
-  _id: SourceKey;
-  data: bson.Binary;
-  buckets: CurrentBucket[];
-  lookups: bson.Binary[];
-}
-
-export interface CurrentBucket {
-  bucket: string;
-  table: string;
-  id: string;
-}
-
-export interface BucketParameterDocument {
-  _id: bigint;
-  key: SourceKey;
-  lookup: bson.Binary;
-  bucket_parameters: Record<string, SqliteJsonValue>[];
-}
-
-export interface BucketDataDocument {
-  _id: BucketDataKey;
-  op: OpType;
-  source_table?: bson.ObjectId;
-  source_key?: ReplicaId;
-  table?: string;
-  row_id?: string;
-  checksum: number;
-  data: string | null;
-  target_op?: bigint | null;
-}
-
-export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
-
-export interface SourceTableDocument {
-  _id: bson.ObjectId;
-  group_id: number;
-  connection_id: number;
-  relation_id: number | string | undefined;
-  schema_name: string;
-  table_name: string;
-  replica_id_columns: string[] | null;
-  replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
-  snapshot_done: boolean | undefined;
-}
-
-export interface IdSequenceDocument {
-  _id: string;
-  op_id: bigint;
-}
-
-export interface SyncRuleDocument {
-  _id: number;
-
-  state: storage.SyncRuleState;
-
-  /**
-   * True if initial snapshot has been replicated.
-   *
-   * Can only be false if state == PROCESSING.
-   */
-  snapshot_done: boolean;
-
-  /**
-   * The last consistent checkpoint.
-   *
-   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
-   */
-  last_checkpoint: bigint | null;
-
-  /**
-   * The LSN associated with the last consistent checkpoint.
-   */
-  last_checkpoint_lsn: string | null;
-
-  /**
-   * If set, no new checkpoints may be created < this value.
-   */
-  no_checkpoint_before: string | null;
-
-  /**
-   * Goes together with no_checkpoint_before.
-   *
-   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
-   * then the checkpoint must be equal to this keepalive_op.
-   */
-  keepalive_op: string | null;
-
-  slot_name: string | null;
-
-  /**
-   * Last time we persisted a checkpoint.
-   *
-   * This may be old if no data is incoming.
-   */
-  last_checkpoint_ts: Date | null;
-
-  /**
-   * Last time we persisted a checkpoint or keepalive.
-   *
-   * This should stay fairly current while replicating.
-   */
-  last_keepalive_ts: Date | null;
-
-  /**
-   * If an error is stopping replication, it will be stored here.
-   */
-  last_fatal_error: string | null;
-
-  content: string;
-}
-
-export interface CustomWriteCheckpointDocument {
-  _id: bson.ObjectId;
-  user_id: string;
-  checkpoint: bigint;
-  sync_rules_id: number;
-}
-
-export interface WriteCheckpointDocument {
-  _id: bson.ObjectId;
-  user_id: string;
-  lsns: Record<string, string>;
-  client_id: bigint;
-}
-
-export interface InstanceDocument {
-  // The instance UUID
-  _id: string;
-}

package/src/storage/implementation/util.ts

@@ -1,154 +0,0 @@
-import { storage, utils } from '@powersync/service-core';
-import * as bson from 'bson';
-import * as crypto from 'crypto';
-import * as mongo from 'mongodb';
-import * as uuid from 'uuid';
-import { BucketDataDocument } from './models.js';
-
-export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
-  let filter = {
-    $gte: {
-      ...prefix
-    } as any,
-    $lt: {
-      ...prefix
-    } as any
-  };
-
-  for (let key of rest) {
-    filter.$gte[key] = new bson.MinKey();
-    filter.$lt[key] = new bson.MaxKey();
-  }
-
-  return filter;
-}
-
-export function generateSlotName(prefix: string, sync_rules_id: number) {
-  const slot_suffix = crypto.randomBytes(2).toString('hex');
-  return `${prefix}${sync_rules_id}_${slot_suffix}`;
-}
-
-/**
- * Read a single batch of data from a cursor, then close it.
- *
- * We do our best to avoid MongoDB fetching any more data than this single batch.
- *
- * This is similar to using `singleBatch: true` in find options.
- * However, that makes `has_more` detection very difficult, since the cursor is always closed
- * after the first batch. Instead, we do a workaround to only fetch a single batch below.
- *
- * For this to be effective, set batchSize = limit in the find command.
- */
-export async function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
-  try {
-    let data: T[];
-    let hasMore = true;
-    // Let MongoDB load the first batch of data
-    const hasAny = await cursor.hasNext();
-    // Now it's in memory, and we can read it
-    data = cursor.readBufferedDocuments();
-    if (!hasAny || cursor.id?.isZero()) {
-      // A zero id means the cursor is exhaused.
-      // No results (hasAny == false) means even this batch doesn't have data.
-      // This should similar results as `await cursor.hasNext()`, but without
-      // actually fetching the next batch.
-      //
-      // Note that it is safe (but slightly inefficient) to return `hasMore: true`
-      // without there being more data, as long as the next batch
-      // will return `hasMore: false`.
-      hasMore = false;
-    }
-    return { data, hasMore };
-  } finally {
-    // Match the from the cursor iterator logic here:
-    // https://github.com/mongodb/node-mongodb-native/blob/e02534e7d1c627bf50b85ca39f5995dbf165ad44/src/cursor/abstract_cursor.ts#L327-L331
-    if (!cursor.closed) {
-      await cursor.close();
-    }
-  }
-}
-
-export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
-  if (row.op == 'PUT' || row.op == 'REMOVE') {
-    return {
-      op_id: utils.timestampToOpId(row._id.o),
-      op: row.op,
-      object_type: row.table,
-      object_id: row.row_id,
-      checksum: Number(row.checksum),
-      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
-      data: row.data
-    };
-  } else {
-    // MOVE, CLEAR
-
-    return {
-      op_id: utils.timestampToOpId(row._id.o),
-      op: row.op,
-      checksum: Number(row.checksum)
-    };
-  }
-}
-
-export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string {
-  if (storage.isUUID(id)) {
-    // Special case for UUID for backwards-compatiblity
-    return `${table.toHexString()}/${id.toHexString()}`;
-  } else {
-    // Hashed UUID from the table and id
-    const repr = bson.serialize({ table, id });
-    return uuid.v5(repr, utils.ID_NAMESPACE);
-  }
-}
-
-/**
- * Helper function for creating a MongoDB client from consumers of this package
- */
-export const createMongoClient = (url: string, options?: mongo.MongoClientOptions) => {
-  return new mongo.MongoClient(url, options);
-};
-
-/**
- * MongoDB bulkWrite internally splits the operations into batches
- * so that no batch exceeds 16MB. However, there are cases where
- * the batch size is very close to 16MB, where additional metadata
- * on the server pushes it over the limit, resulting in this error
- * from the server:
- *
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
- *
- * We work around the issue by doing our own batching, limiting the
- * batch size to 15MB. This does add additional overhead with
- * BSON.calculateObjectSize.
- */
-export async function safeBulkWrite<T extends mongo.Document>(
-  collection: mongo.Collection<T>,
-  operations: mongo.AnyBulkWriteOperation<T>[],
-  options: mongo.BulkWriteOptions
-) {
-  // Must be below 16MB.
-  // We could probably go a little closer, but 15MB is a safe threshold.
-  const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
-
-  let batch: mongo.AnyBulkWriteOperation<T>[] = [];
-  let currentSize = 0;
-  // Estimated overhead per operation, should be smaller in reality.
-  const keySize = 8;
-  for (let op of operations) {
-    const bsonSize =
-      mongo.BSON.calculateObjectSize(op, {
-        checkKeys: false,
-        ignoreUndefined: true
-      } as any) + keySize;
-    if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
-      await collection.bulkWrite(batch, options);
-      currentSize = 0;
-      batch = [];
-    }
-    batch.push(op);
-    currentSize += bsonSize;
-  }
-  if (batch.length > 0) {
-    await collection.bulkWrite(batch, options);
-  }
-}

package/src/storage/storage-index.ts

@@ -1,14 +0,0 @@
-export * from './implementation/config.js';
-export * from './implementation/db.js';
-export * from './implementation/models.js';
-export * from './implementation/MongoBucketBatch.js';
-export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
-export * from './implementation/MongoPersistedSyncRulesContent.js';
-export * from './implementation/MongoStorageProvider.js';
-export * from './implementation/MongoSyncBucketStorage.js';
-export * from './implementation/MongoSyncRulesLock.js';
-export * from './implementation/OperationBatch.js';
-export * from './implementation/PersistedBatch.js';
-export * from './implementation/util.js';
-export * from './MongoBucketStorage.js';

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -1,332 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -93886621,
-          "count": 2,
-        },
-      ],
-      "last_op_id": "2",
-      "write_checkpoint": undefined,
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
-[
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": -93886621n,
-          "op": "CLEAR",
-          "op_id": "2",
-        },
-      ],
-      "has_more": false,
-      "next_after": "2",
-    },
-  },
-  {
-    "checkpoint_diff": {
-      "last_op_id": "4",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 499012468,
-          "count": 4,
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "2",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 1859363232n,
-          "data": "{"id":"t1","description":"Test 1b"}",
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "3",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-        {
-          "checksum": 3028503153n,
-          "data": "{"id":"t2","description":"Test 2b"}",
-          "object_id": "t2",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "4",
-          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
-        },
-      ],
-      "has_more": false,
-      "next_after": "4",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "4",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > expired token 1`] = `
-[
-  {
-    "token_expires_in": 0,
-  },
-]
-`;
-
-exports[`sync - mongodb > expiring token 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 0,
-          "count": 0,
-        },
-      ],
-      "last_op_id": "0",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "0",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > expiring token 2`] = `
-[
-  {
-    "token_expires_in": 0,
-  },
-]
-`;
-
-exports[`sync - mongodb > sync global data 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -93886621,
-          "count": 2,
-        },
-      ],
-      "last_op_id": "2",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 920318466n,
-          "data": "{"id":"t1","description":"Test 1"}",
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "1",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-        {
-          "checksum": 3280762209n,
-          "data": "{"id":"t2","description":"Test 2"}",
-          "object_id": "t2",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "2",
-          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
-        },
-      ],
-      "has_more": false,
-      "next_after": "2",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "2",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync legacy non-raw data 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -852817836,
-          "count": 1,
-        },
-      ],
-      "last_op_id": "1",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 3442149460n,
-          "data": {
-            "description": "Test
-"string"",
-            "id": "t1",
-            "large_num": 12345678901234567890n,
-          },
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "1",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-      ],
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "1",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync updates to global data 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 0,
-          "count": 0,
-        },
-      ],
-      "last_op_id": "0",
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "0",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync updates to global data 2`] = `
-[
-  {
-    "checkpoint_diff": {
-      "last_op_id": "1",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": 920318466,
-          "count": 1,
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 920318466n,
-          "data": "{"id":"t1","description":"Test 1"}",
-          "object_id": "t1",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "1",
-          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
-        },
-      ],
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "1",
-    },
-  },
-]
-`;
-
-exports[`sync - mongodb > sync updates to global data 3`] = `
-[
-  {
-    "checkpoint_diff": {
-      "last_op_id": "2",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "mybucket[]",
-          "checksum": -93886621,
-          "count": 2,
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "1",
-      "bucket": "mybucket[]",
-      "data": [
-        {
-          "checksum": 3280762209n,
-          "data": "{"id":"t2","description":"Test 2"}",
-          "object_id": "t2",
-          "object_type": "test",
-          "op": "PUT",
-          "op_id": "2",
-          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
-        },
-      ],
-      "has_more": false,
-      "next_after": "2",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "2",
-    },
-  },
-]
-`;

package/test/src/storage.test.ts

@@ -1,7 +0,0 @@
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
-
-describe('Mongo Sync Bucket Storage', () => register.registerDataStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY));
-
-describe('Sync Bucket Validation', register.registerBucketValidationTests);

package/test/src/storage_compacting.test.ts

@@ -1,6 +0,0 @@
-import { MongoCompactOptions } from '@module/storage/implementation/MongoCompactor.js';
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
-
-describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));