@powersync/service-core 0.13.0 → 0.15.0

package/src/storage/mongo/MongoSyncRulesLock.ts

@@ -1,85 +0,0 @@
-import crypto from 'crypto';
-
-import { PersistedSyncRulesContent, ReplicationLock } from '../BucketStorage.js';
-import { PowerSyncMongo } from './db.js';
-import { logger } from '@powersync/lib-services-framework';
-
-/**
- * Manages a lock on a sync rules document, so that only one process
- * replicates those sync rules at a time.
- */
-export class MongoSyncRulesLock implements ReplicationLock {
-  private readonly refreshInterval: NodeJS.Timeout;
-
-  static async createLock(db: PowerSyncMongo, sync_rules: PersistedSyncRulesContent): Promise<MongoSyncRulesLock> {
-    const lockId = crypto.randomBytes(8).toString('hex');
-    const doc = await db.sync_rules.findOneAndUpdate(
-      { _id: sync_rules.id, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
-      {
-        $set: {
-          lock: {
-            id: lockId,
-            expires_at: new Date(Date.now() + 60 * 1000)
-          }
-        }
-      },
-      {
-        projection: { lock: 1 },
-        returnDocument: 'before'
-      }
-    );
-
-    if (doc == null) {
-      throw new Error(`Sync rules: ${sync_rules.id} have been locked by another process for replication.`);
-    }
-    return new MongoSyncRulesLock(db, sync_rules.id, lockId);
-  }
-
-  constructor(
-    private db: PowerSyncMongo,
-    public sync_rules_id: number,
-    private lock_id: string
-  ) {
-    this.refreshInterval = setInterval(async () => {
-      try {
-        await this.refresh();
-      } catch (e) {
-        logger.error('Failed to refresh lock', e);
-        clearInterval(this.refreshInterval);
-      }
-    }, 30_130);
-  }
-
-  async release(): Promise<void> {
-    clearInterval(this.refreshInterval);
-    const result = await this.db.sync_rules.updateOne(
-      {
-        _id: this.sync_rules_id,
-        'lock.id': this.lock_id
-      },
-      {
-        $unset: { lock: 1 }
-      }
-    );
-    if (result.modifiedCount == 0) {
-      // Log and ignore
-      logger.warn(`Lock already released: ${this.sync_rules_id}/${this.lock_id}`);
-    }
-  }
-
-  private async refresh(): Promise<void> {
-    const result = await this.db.sync_rules.findOneAndUpdate(
-      {
-        _id: this.sync_rules_id,
-        'lock.id': this.lock_id
-      },
-      {
-        $set: { 'lock.expires_at': new Date(Date.now() + 60 * 1000) }
-      },
-      { returnDocument: 'after' }
-    );
-    if (result == null) {
-      throw new Error(`Lock not held anymore: ${this.sync_rules_id}/${this.lock_id}`);
-    }
-  }
-}

package/src/storage/mongo/MongoWriteCheckpointAPI.ts

@@ -1,154 +0,0 @@
-import * as framework from '@powersync/lib-services-framework';
-import {
-  CustomWriteCheckpointFilters,
-  CustomWriteCheckpointOptions,
-  LastWriteCheckpointFilters,
-  ManagedWriteCheckpointFilters,
-  ManagedWriteCheckpointOptions,
-  WriteCheckpointAPI,
-  WriteCheckpointMode
-} from '../WriteCheckpointAPI.js';
-import { PowerSyncMongo } from './db.js';
-import { safeBulkWrite } from './util.js';
-
-export type MongoCheckpointAPIOptions = {
-  db: PowerSyncMongo;
-  mode: WriteCheckpointMode;
-};
-
-export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
-  readonly db: PowerSyncMongo;
-  private _mode: WriteCheckpointMode;
-
-  constructor(options: MongoCheckpointAPIOptions) {
-    this.db = options.db;
-    this._mode = options.mode;
-  }
-
-  get writeCheckpointMode() {
-    return this._mode;
-  }
-
-  setWriteCheckpointMode(mode: WriteCheckpointMode): void {
-    this._mode = mode;
-  }
-
-  async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
-    return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
-  }
-
-  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
-    if (this.writeCheckpointMode !== WriteCheckpointMode.CUSTOM) {
-      throw new framework.errors.ValidationError(
-        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
-      );
-    }
-
-    const { checkpoint, user_id, sync_rules_id } = options;
-    const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id,
-        sync_rules_id
-      },
-      {
-        $set: {
-          checkpoint
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
-    );
-    return doc!.checkpoint;
-  }
-
-  async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
-    if (this.writeCheckpointMode !== WriteCheckpointMode.MANAGED) {
-      throw new framework.errors.ValidationError(
-        `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
-      );
-    }
-
-    const { user_id, heads: lsns } = checkpoint;
-    const doc = await this.db.write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id
-      },
-      {
-        $set: {
-          lsns
-        },
-        $inc: {
-          client_id: 1n
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
-    );
-    return doc!.client_id;
-  }
-
-  async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
-    switch (this.writeCheckpointMode) {
-      case WriteCheckpointMode.CUSTOM:
-        if (false == 'sync_rules_id' in filters) {
-          throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
-        }
-        return this.lastCustomWriteCheckpoint(filters);
-      case WriteCheckpointMode.MANAGED:
-        if (false == 'heads' in filters) {
-          throw new framework.errors.ValidationError(
-            `Replication HEAD is required for managed Write Checkpoint filtering`
-          );
-        }
-        return this.lastManagedWriteCheckpoint(filters);
-    }
-  }
-
-  protected async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters) {
-    const { user_id, sync_rules_id } = filters;
-    const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
-      user_id,
-      sync_rules_id
-    });
-    return lastWriteCheckpoint?.checkpoint ?? null;
-  }
-
-  protected async lastManagedWriteCheckpoint(filters: ManagedWriteCheckpointFilters) {
-    const { user_id, heads } = filters;
-    // TODO: support multiple heads when we need to support multiple connections
-    const lsn = heads['1'];
-    if (lsn == null) {
-      // Can happen if we haven't replicated anything yet.
-      return null;
-    }
-    const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
-      user_id: user_id,
-      'lsns.1': { $lte: lsn }
-    });
-    return lastWriteCheckpoint?.client_id ?? null;
-  }
-}
-
-export async function batchCreateCustomWriteCheckpoints(
-  db: PowerSyncMongo,
-  checkpoints: CustomWriteCheckpointOptions[]
-): Promise<void> {
-  if (!checkpoints.length) {
-    return;
-  }
-
-  await safeBulkWrite(
-    db.custom_write_checkpoints,
-    checkpoints.map((checkpointOptions) => ({
-      updateOne: {
-        filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
-        update: {
-          $set: {
-            checkpoint: checkpointOptions.checkpoint,
-            sync_rules_id: checkpointOptions.sync_rules_id
-          }
-        },
-        upsert: true
-      }
-    })),
-    {}
-  );
-}

package/src/storage/mongo/OperationBatch.ts

@@ -1,131 +0,0 @@
-import { ToastableSqliteRow } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-
-import { SaveOptions } from '../BucketStorage.js';
-import { isUUID } from './util.js';
-import { ReplicaId } from './models.js';
-
-/**
- * Maximum number of operations in a batch.
- */
-const MAX_BATCH_COUNT = 2000;
-
-/**
- * Maximum size of operations in the batch (estimated).
- */
-const MAX_RECORD_BATCH_SIZE = 5_000_000;
-
-/**
- * Maximum size of size of current_data documents we lookup at a time.
- */
-const MAX_CURRENT_DATA_BATCH_SIZE = 16_000_000;
-
-/**
- * Batch of input operations.
- *
- * We accumulate operations up to MAX_RECORD_BATCH_SIZE,
- * then further split into sub-batches if MAX_CURRENT_DATA_BATCH_SIZE is exceeded.
- */
-export class OperationBatch {
-  batch: RecordOperation[] = [];
-  currentSize: number = 0;
-
-  get length() {
-    return this.batch.length;
-  }
-
-  push(op: RecordOperation) {
-    this.batch.push(op);
-    this.currentSize += op.estimatedSize;
-  }
-
-  shouldFlush() {
-    return this.batch.length >= MAX_BATCH_COUNT || this.currentSize > MAX_RECORD_BATCH_SIZE;
-  }
-
-  /**
-   *
-   * @param sizes Map of source key to estimated size of the current_data document, or undefined if current_data is not persisted.
-   *
-   */
-  *batched(sizes: Map<string, number> | undefined): Generator<RecordOperation[]> {
-    if (sizes == null) {
-      yield this.batch;
-      return;
-    }
-    let currentBatch: RecordOperation[] = [];
-    let currentBatchSize = 0;
-    for (let op of this.batch) {
-      const key = op.internalBeforeKey;
-      const size = sizes.get(key) ?? 0;
-      if (currentBatchSize + size > MAX_CURRENT_DATA_BATCH_SIZE && currentBatch.length > 0) {
-        yield currentBatch;
-        currentBatch = [];
-        currentBatchSize = 0;
-      }
-      currentBatchSize += size;
-      currentBatch.push(op);
-    }
-    if (currentBatch.length > 0) {
-      yield currentBatch;
-    }
-  }
-}
-
-export class RecordOperation {
-  public readonly afterId: ReplicaId | null;
-  public readonly beforeId: ReplicaId;
-  public readonly internalBeforeKey: string;
-  public readonly internalAfterKey: string | null;
-  public readonly estimatedSize: number;
-
-  constructor(public readonly record: SaveOptions) {
-    const afterId = record.afterReplicaId ?? null;
-    const beforeId = record.beforeReplicaId ?? record.afterReplicaId;
-    this.afterId = afterId;
-    this.beforeId = beforeId;
-    this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId);
-    this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null;
-
-    this.estimatedSize = estimateRowSize(record.before) + estimateRowSize(record.after);
-  }
-}
-
-/**
- * In-memory cache key - must not be persisted.
- */
-export function cacheKey(table: bson.ObjectId, id: ReplicaId) {
-  if (isUUID(id)) {
-    return `${table.toHexString()}.${id.toHexString()}`;
-  } else if (typeof id == 'string') {
-    return `${table.toHexString()}.${id}`;
-  } else {
-    return `${table.toHexString()}.${(bson.serialize({ id: id }) as Buffer).toString('base64')}`;
-  }
-}
-
-/**
- * Estimate in-memory size of row.
- */
-function estimateRowSize(record: ToastableSqliteRow | undefined) {
-  if (record == null) {
-    return 12;
-  }
-  let size = 0;
-  for (let [key, value] of Object.entries(record)) {
-    size += 12 + key.length;
-    // number | string | null | bigint | Uint8Array
-    if (value == null) {
-      size += 4;
-    } else if (typeof value == 'number') {
-      size += 8;
-    } else if (typeof value == 'bigint') {
-      size += 8;
-    } else if (typeof value == 'string') {
-      size += value.length;
-    } else if (value instanceof Uint8Array) {
-      size += value.byteLength;
-    }
-  }
-  return size;
-}

package/src/storage/mongo/PersistedBatch.ts

@@ -1,285 +0,0 @@
-import { JSONBig } from '@powersync/service-jsonbig';
-import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
-import * as mongo from 'mongodb';
-
-import * as util from '../../util/util-index.js';
-import { SourceTable } from '../SourceTable.js';
-import { currentBucketKey } from './MongoBucketBatch.js';
-import { MongoIdSequence } from './MongoIdSequence.js';
-import { PowerSyncMongo } from './db.js';
-import {
-  BucketDataDocument,
-  BucketParameterDocument,
-  CurrentBucket,
-  CurrentDataDocument,
-  SourceKey,
-  ReplicaId
-} from './models.js';
-import { replicaIdToSubkey, safeBulkWrite, serializeLookup } from './util.js';
-import { logger } from '@powersync/lib-services-framework';
-
-/**
- * Maximum size of operations we write in a single transaction.
- *
- * It's tricky to find the exact limit, but from experience, over 100MB
- * can cause an error:
- * > transaction is too large and will not fit in the storage engine cache
- *
- * Additionally, unbounded size here can balloon our memory usage in some edge
- * cases.
- *
- * When we reach this threshold, we commit the transaction and start a new one.
- */
-const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
-
-/**
- * Limit number of documents to write in a single transaction.
- *
- * This has an effect on error message size in some cases.
- */
-const MAX_TRANSACTION_DOC_COUNT = 2_000;
-
-/**
- * Keeps track of bulkwrite operations within a transaction.
- *
- * There may be multiple of these batches per transaction, but it may not span
- * multiple transactions.
- */
-export class PersistedBatch {
-  bucketData: mongo.AnyBulkWriteOperation<BucketDataDocument>[] = [];
-  bucketParameters: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
-  currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
-
-  /**
-   * For debug logging only.
-   */
-  debugLastOpId: bigint | null = null;
-
-  /**
-   * Very rough estimate of transaction size.
-   */
-  currentSize = 0;
-
-  constructor(
-    private group_id: number,
-    writtenSize: number
-  ) {
-    this.currentSize = writtenSize;
-  }
-
-  saveBucketData(options: {
-    op_seq: MongoIdSequence;
-    sourceKey: ReplicaId;
-    table: SourceTable;
-    evaluated: EvaluatedRow[];
-    before_buckets: CurrentBucket[];
-  }) {
-    const remaining_buckets = new Map<string, CurrentBucket>();
-    for (let b of options.before_buckets) {
-      const key = currentBucketKey(b);
-      remaining_buckets.set(key, b);
-    }
-
-    const dchecksum = util.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey));
-
-    for (let k of options.evaluated) {
-      const key = currentBucketKey(k);
-      remaining_buckets.delete(key);
-
-      // INSERT
-      const recordData = JSONBig.stringify(k.data);
-      const checksum = util.hashData(k.table, k.id, recordData);
-      this.currentSize += recordData.length + 200;
-
-      const op_id = options.op_seq.next();
-      this.debugLastOpId = op_id;
-
-      this.bucketData.push({
-        insertOne: {
-          document: {
-            _id: {
-              g: this.group_id,
-              b: k.bucket,
-              o: op_id
-            },
-            op: 'PUT',
-            source_table: options.table.id,
-            source_key: options.sourceKey,
-            table: k.table,
-            row_id: k.id,
-            checksum: checksum,
-            data: recordData
-          }
-        }
-      });
-    }
-
-    for (let bd of remaining_buckets.values()) {
-      // REMOVE
-
-      const op_id = options.op_seq.next();
-      this.debugLastOpId = op_id;
-
-      this.bucketData.push({
-        insertOne: {
-          document: {
-            _id: {
-              g: this.group_id,
-              b: bd.bucket,
-              o: op_id
-            },
-            op: 'REMOVE',
-            source_table: options.table.id,
-            source_key: options.sourceKey,
-            table: bd.table,
-            row_id: bd.id,
-            checksum: dchecksum,
-            data: null
-          }
-        }
-      });
-      this.currentSize += 200;
-    }
-  }
-
-  saveParameterData(data: {
-    op_seq: MongoIdSequence;
-    sourceKey: ReplicaId;
-    sourceTable: SourceTable;
-    evaluated: EvaluatedParameters[];
-    existing_lookups: bson.Binary[];
-  }) {
-    // This is similar to saving bucket data.
-    // A key difference is that we don't need to keep the history intact.
-    // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint.
-    // Instead of storing per bucket id, we store per "lookup".
-    // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is
-    // either persisted or removed.
-    // We also don't need to keep history intact.
-    const { sourceTable, sourceKey, evaluated } = data;
-
-    const remaining_lookups = new Map<string, bson.Binary>();
-    for (let l of data.existing_lookups) {
-      remaining_lookups.set(l.toString('base64'), l);
-    }
-
-    // 1. Insert new entries
-    for (let result of evaluated) {
-      const binLookup = serializeLookup(result.lookup);
-      const hex = binLookup.toString('base64');
-      remaining_lookups.delete(hex);
-
-      const op_id = data.op_seq.next();
-      this.debugLastOpId = op_id;
-      this.bucketParameters.push({
-        insertOne: {
-          document: {
-            _id: op_id,
-            key: {
-              g: this.group_id,
-              t: sourceTable.id,
-              k: sourceKey
-            },
-            lookup: binLookup,
-            bucket_parameters: result.bucket_parameters
-          }
-        }
-      });
-
-      this.currentSize += 200;
-    }
-
-    // 2. "REMOVE" entries for any lookup not touched.
-    for (let lookup of remaining_lookups.values()) {
-      const op_id = data.op_seq.next();
-      this.debugLastOpId = op_id;
-      this.bucketParameters.push({
-        insertOne: {
-          document: {
-            _id: op_id,
-            key: {
-              g: this.group_id,
-              t: sourceTable.id,
-              k: sourceKey
-            },
-            lookup: lookup,
-            bucket_parameters: []
-          }
-        }
-      });
-
-      this.currentSize += 200;
-    }
-  }
-
-  deleteCurrentData(id: SourceKey) {
-    const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
-      deleteOne: {
-        filter: { _id: id }
-      }
-    };
-    this.currentData.push(op);
-    this.currentSize += 50;
-  }
-
-  upsertCurrentData(id: SourceKey, values: Partial<CurrentDataDocument>) {
-    const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
-      updateOne: {
-        filter: { _id: id },
-        update: {
-          $set: values
-        },
-        upsert: true
-      }
-    };
-    this.currentData.push(op);
-    this.currentSize += (values.data?.length() ?? 0) + 100;
-  }
-
-  shouldFlushTransaction() {
-    return (
-      this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
-      this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT ||
-      this.currentData.length >= MAX_TRANSACTION_DOC_COUNT ||
-      this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
-    );
-  }
-
-  async flush(db: PowerSyncMongo, session: mongo.ClientSession) {
-    if (this.bucketData.length > 0) {
-      // calculate total size
-      await safeBulkWrite(db.bucket_data, this.bucketData, {
-        session,
-        // inserts only - order doesn't matter
-        ordered: false
-      });
-    }
-    if (this.bucketParameters.length > 0) {
-      await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
-        session,
-        // inserts only - order doesn't matter
-        ordered: false
-      });
-    }
-    if (this.currentData.length > 0) {
-      await safeBulkWrite(db.current_data, this.currentData, {
-        session,
-        // may update and delete data within the same batch - order matters
-        ordered: true
-      });
-    }
-
-    logger.info(
-      `powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${
-        this.currentData.length
-      } updates, ${Math.round(this.currentSize / 1024)}kb. Last op_id: ${this.debugLastOpId}`
-    );
-
-    this.bucketData = [];
-    this.bucketParameters = [];
-    this.currentData = [];
-    this.currentSize = 0;
-    this.debugLastOpId = null;
-  }
-}

package/src/storage/mongo/config.ts

@@ -1,40 +0,0 @@
-import * as urijs from 'uri-js';
-
-export interface MongoConnectionConfig {
-  uri: string;
-  username?: string;
-  password?: string;
-  database?: string;
-}
-
-/**
- * Validate and normalize connection options.
- *
- * Returns destructured options.
- *
- * For use by both storage and mongo module.
- */
-export function normalizeMongoConfig(options: MongoConnectionConfig) {
-  let uri = urijs.parse(options.uri);
-
-  const database = options.database ?? uri.path?.substring(1) ?? '';
-
-  const userInfo = uri.userinfo?.split(':');
-
-  const username = options.username ?? userInfo?.[0];
-  const password = options.password ?? userInfo?.[1];
-
-  if (database == '') {
-    throw new Error(`database required`);
-  }
-
-  delete uri.userinfo;
-
-  return {
-    uri: urijs.serialize(uri),
-    database,
-
-    username,
-    password
-  };
-}