@powersync/service-core 0.13.0 → 0.15.0

package/src/storage/mongo/db.ts

@@ -1,88 +0,0 @@
-import * as mongo from 'mongodb';
-
-import { configFile } from '@powersync/service-types';
-import * as db from '../../db/db-index.js';
-import * as locks from '../../locks/locks-index.js';
-import {
-  BucketDataDocument,
-  BucketParameterDocument,
-  CurrentDataDocument,
-  CustomWriteCheckpointDocument,
-  IdSequenceDocument,
-  InstanceDocument,
-  SourceTableDocument,
-  SyncRuleDocument,
-  WriteCheckpointDocument
-} from './models.js';
-import { BSON_DESERIALIZE_OPTIONS } from './util.js';
-
-export interface PowerSyncMongoOptions {
-  /**
-   * Optional - uses the database from the MongoClient connection URI if not specified.
-   */
-  database?: string;
-}
-
-export function createPowerSyncMongo(config: configFile.PowerSyncConfig['storage']) {
-  return new PowerSyncMongo(db.mongo.createMongoClient(config), { database: config.database });
-}
-
-export class PowerSyncMongo {
-  readonly current_data: mongo.Collection<CurrentDataDocument>;
-  readonly bucket_data: mongo.Collection<BucketDataDocument>;
-  readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
-  readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
-  readonly sync_rules: mongo.Collection<SyncRuleDocument>;
-  readonly source_tables: mongo.Collection<SourceTableDocument>;
-  readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
-  readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
-  readonly instance: mongo.Collection<InstanceDocument>;
-  readonly locks: mongo.Collection<locks.Lock>;
-
-  readonly client: mongo.MongoClient;
-  readonly db: mongo.Db;
-
-  constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions) {
-    this.client = client;
-
-    const db = client.db(options?.database, {
-      ...BSON_DESERIALIZE_OPTIONS
-    });
-    this.db = db;
-
-    this.current_data = db.collection<CurrentDataDocument>('current_data');
-    this.bucket_data = db.collection('bucket_data');
-    this.bucket_parameters = db.collection('bucket_parameters');
-    this.op_id_sequence = db.collection('op_id_sequence');
-    this.sync_rules = db.collection('sync_rules');
-    this.source_tables = db.collection('source_tables');
-    this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
-    this.write_checkpoints = db.collection('write_checkpoints');
-    this.instance = db.collection('instance');
-    this.locks = this.db.collection('locks');
-  }
-
-  /**
-   * Clear all collections.
-   */
-  async clear() {
-    await this.current_data.deleteMany({});
-    await this.bucket_data.deleteMany({});
-    await this.bucket_parameters.deleteMany({});
-    await this.op_id_sequence.deleteMany({});
-    await this.sync_rules.deleteMany({});
-    await this.source_tables.deleteMany({});
-    await this.write_checkpoints.deleteMany({});
-    await this.instance.deleteOne({});
-    await this.locks.deleteMany({});
-  }
-
-  /**
-   * Drop the entire database.
-   *
-   * Primarily for tests.
-   */
-  async drop() {
-    await this.db.dropDatabase();
-  }
-}

package/src/storage/mongo/models.ts

@@ -1,187 +0,0 @@
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-
-/**
- * Replica id uniquely identifying a row on the source database.
- *
- * Can be any value serializable to BSON.
- *
- * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
- */
-export type ReplicaId = bson.UUID | bson.Document | any;
-
-export interface SourceKey {
-  /** group_id */
-  g: number;
-  /** source table id */
-  t: bson.ObjectId;
-  /** source key */
-  k: ReplicaId;
-}
-
-export interface BucketDataKey {
-  /** group_id */
-  g: number;
-  /** bucket name */
-  b: string;
-  /** op_id */
-  o: bigint;
-}
-
-export interface CurrentDataDocument {
-  _id: SourceKey;
-  data: bson.Binary;
-  buckets: CurrentBucket[];
-  lookups: bson.Binary[];
-}
-
-export interface CurrentBucket {
-  bucket: string;
-  table: string;
-  id: string;
-}
-
-export interface BucketParameterDocument {
-  _id: bigint;
-  key: SourceKey;
-  lookup: bson.Binary;
-  bucket_parameters: Record<string, SqliteJsonValue>[];
-}
-
-export interface BucketDataDocument {
-  _id: BucketDataKey;
-  op: OpType;
-  source_table?: bson.ObjectId;
-  source_key?: ReplicaId;
-  table?: string;
-  row_id?: string;
-  checksum: number;
-  data: string | null;
-  target_op?: bigint | null;
-}
-
-export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
-
-export interface SourceTableDocument {
-  _id: bson.ObjectId;
-  group_id: number;
-  connection_id: number;
-  relation_id: number | string | undefined;
-  schema_name: string;
-  table_name: string;
-  replica_id_columns: string[] | null;
-  replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
-  snapshot_done: boolean | undefined;
-}
-
-export interface IdSequenceDocument {
-  _id: string;
-  op_id: bigint;
-}
-
-export enum SyncRuleState {
-  /**
-   * New sync rules - needs to be processed (initial replication).
-   *
-   * While multiple sets of sync rules _can_ be in PROCESSING,
-   * it's generally pointless, so we only keep one in that state.
-   */
-  PROCESSING = 'PROCESSING',
-
-  /**
-   * Sync rule processing is done, and can be used for sync.
-   *
-   * Only one set of sync rules should be in ACTIVE state.
-   */
-  ACTIVE = 'ACTIVE',
-  /**
-   * This state is used when the sync rules has been replaced,
-   * and replication is or should be stopped.
-   */
-  STOP = 'STOP',
-  /**
-   * After sync rules have been stopped, the data needs to be
-   * deleted. Once deleted, the state is TERMINATED.
-   */
-  TERMINATED = 'TERMINATED'
-}
-
-export interface SyncRuleDocument {
-  _id: number;
-
-  state: SyncRuleState;
-
-  /**
-   * True if initial snapshot has been replicated.
-   *
-   * Can only be false if state == PROCESSING.
-   */
-  snapshot_done: boolean;
-
-  /**
-   * The last consistent checkpoint.
-   *
-   * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
-   */
-  last_checkpoint: bigint | null;
-
-  /**
-   * The LSN associated with the last consistent checkpoint.
-   */
-  last_checkpoint_lsn: string | null;
-
-  /**
-   * If set, no new checkpoints may be created < this value.
-   */
-  no_checkpoint_before: string | null;
-
-  /**
-   * Goes together with no_checkpoint_before.
-   *
-   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
-   * then the checkpoint must be equal to this keepalive_op.
-   */
-  keepalive_op: string | null;
-
-  slot_name: string | null;
-
-  /**
-   * Last time we persisted a checkpoint.
-   *
-   * This may be old if no data is incoming.
-   */
-  last_checkpoint_ts: Date | null;
-
-  /**
-   * Last time we persisted a checkpoint or keepalive.
-   *
-   * This should stay fairly current while replicating.
-   */
-  last_keepalive_ts: Date | null;
-
-  /**
-   * If an error is stopping replication, it will be stored here.
-   */
-  last_fatal_error: string | null;
-
-  content: string;
-}
-
-export interface CustomWriteCheckpointDocument {
-  _id: bson.ObjectId;
-  user_id: string;
-  checkpoint: bigint;
-  sync_rules_id: number;
-}
-
-export interface WriteCheckpointDocument {
-  _id: bson.ObjectId;
-  user_id: string;
-  lsns: Record<string, string>;
-  client_id: bigint;
-}
-
-export interface InstanceDocument {
-  // The instance UUID
-  _id: string;
-}

package/src/storage/mongo/util.ts

@@ -1,203 +0,0 @@
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-import * as crypto from 'crypto';
-import * as mongo from 'mongodb';
-import * as uuid from 'uuid';
-import { OplogEntry } from '../../util/protocol-types.js';
-import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js';
-import { BucketDataDocument, ReplicaId } from './models.js';
-
-/**
- * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
- * @param lookup
- */
-
-export function serializeLookup(lookup: SqliteJsonValue[]) {
-  const normalized = lookup.map((value) => {
-    if (typeof value == 'number' && Number.isInteger(value)) {
-      return BigInt(value);
-    } else {
-      return value;
-    }
-  });
-  return new bson.Binary(bson.serialize({ l: normalized }));
-}
-
-export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
-  let filter = {
-    $gte: {
-      ...prefix
-    } as any,
-    $lt: {
-      ...prefix
-    } as any
-  };
-
-  for (let key of rest) {
-    filter.$gte[key] = new bson.MinKey();
-    filter.$lt[key] = new bson.MaxKey();
-  }
-
-  return filter;
-}
-
-export function generateSlotName(prefix: string, sync_rules_id: number) {
-  const slot_suffix = crypto.randomBytes(2).toString('hex');
-  return `${prefix}${sync_rules_id}_${slot_suffix}`;
-}
-
-/**
- * Read a single batch of data from a cursor, then close it.
- *
- * We do our best to avoid MongoDB fetching any more data than this single batch.
- *
- * This is similar to using `singleBatch: true` in find options.
- * However, that makes `has_more` detection very difficult, since the cursor is always closed
- * after the first batch. Instead, we do a workaround to only fetch a single batch below.
- *
- * For this to be effective, set batchSize = limit in the find command.
- */
-export async function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
-  try {
-    let data: T[];
-    let hasMore = true;
-    // Let MongoDB load the first batch of data
-    const hasAny = await cursor.hasNext();
-    // Now it's in memory, and we can read it
-    data = cursor.readBufferedDocuments();
-    if (!hasAny || cursor.id?.isZero()) {
-      // A zero id means the cursor is exhaused.
-      // No results (hasAny == false) means even this batch doesn't have data.
-      // This should similar results as `await cursor.hasNext()`, but without
-      // actually fetching the next batch.
-      //
-      // Note that it is safe (but slightly inefficient) to return `hasMore: true`
-      // without there being more data, as long as the next batch
-      // will return `hasMore: false`.
-      hasMore = false;
-    }
-    return { data, hasMore };
-  } finally {
-    // Match the from the cursor iterator logic here:
-    // https://github.com/mongodb/node-mongodb-native/blob/e02534e7d1c627bf50b85ca39f5995dbf165ad44/src/cursor/abstract_cursor.ts#L327-L331
-    if (!cursor.closed) {
-      await cursor.close();
-    }
-  }
-}
-
-export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
-  // use bigint instead of Long
-  useBigInt64: true
-};
-
-export function mapOpEntry(row: BucketDataDocument): OplogEntry {
-  if (row.op == 'PUT' || row.op == 'REMOVE') {
-    return {
-      op_id: timestampToOpId(row._id.o),
-      op: row.op,
-      object_type: row.table,
-      object_id: row.row_id,
-      checksum: Number(row.checksum),
-      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
-      data: row.data
-    };
-  } else {
-    // MOVE, CLEAR
-
-    return {
-      op_id: timestampToOpId(row._id.o),
-      op: row.op,
-      checksum: Number(row.checksum)
-    };
-  }
-}
-
-/**
- * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
- */
-export function replicaIdEquals(a: ReplicaId, b: ReplicaId) {
-  if (a === b) {
-    return true;
-  } else if (typeof a == 'string' && typeof b == 'string') {
-    return a == b;
-  } else if (isUUID(a) && isUUID(b)) {
-    return a.equals(b);
-  } else if (a == null && b == null) {
-    return true;
-  } else if (a != null || b != null) {
-    return false;
-  } else {
-    // There are many possible primitive values, this covers them all
-    return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
-  }
-}
-
-export function replicaIdToSubkey(table: bson.ObjectId, id: ReplicaId): string {
-  if (isUUID(id)) {
-    // Special case for UUID for backwards-compatiblity
-    return `${table.toHexString()}/${id.toHexString()}`;
-  } else {
-    // Hashed UUID from the table and id
-    const repr = bson.serialize({ table, id });
-    return uuid.v5(repr, ID_NAMESPACE);
-  }
-}
-
-/**
- * True if this is a bson.UUID.
- *
- * Works even with multiple copies of the bson package.
- */
-export function isUUID(value: any): value is bson.UUID {
-  if (value == null || typeof value != 'object') {
-    return false;
-  }
-  const uuid = value as bson.UUID;
-  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
-}
-
-/**
- * MongoDB bulkWrite internally splits the operations into batches
- * so that no batch exceeds 16MB. However, there are cases where
- * the batch size is very close to 16MB, where additional metadata
- * on the server pushes it over the limit, resulting in this error
- * from the server:
- *
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
- *
- * We work around the issue by doing our own batching, limiting the
- * batch size to 15MB. This does add additional overhead with
- * BSON.calculateObjectSize.
- */
-export async function safeBulkWrite<T extends mongo.Document>(
-  collection: mongo.Collection<T>,
-  operations: mongo.AnyBulkWriteOperation<T>[],
-  options: mongo.BulkWriteOptions
-) {
-  // Must be below 16MB.
-  // We could probably go a little closer, but 15MB is a safe threshold.
-  const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
-
-  let batch: mongo.AnyBulkWriteOperation<T>[] = [];
-  let currentSize = 0;
-  // Estimated overhead per operation, should be smaller in reality.
-  const keySize = 8;
-  for (let op of operations) {
-    const bsonSize =
-      mongo.BSON.calculateObjectSize(op, {
-        checkKeys: false,
-        ignoreUndefined: true
-      } as any) + keySize;
-    if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
-      await collection.bulkWrite(batch, options);
-      currentSize = 0;
-      batch = [];
-    }
-    batch.push(op);
-    currentSize += bsonSize;
-  }
-  if (batch.length > 0) {
-    await collection.bulkWrite(batch, options);
-  }
-}