@powersync/service-core 0.0.0-dev-20241007145127 → 0.0.0-dev-20241015210820

package/src/storage/mongo/MongoWriteCheckpointAPI.ts

@@ -0,0 +1,136 @@
+import * as framework from '@powersync/lib-services-framework';
+import {
+  CustomWriteCheckpointFilters,
+  CustomWriteCheckpointOptions,
+  LastWriteCheckpointFilters,
+  ManagedWriteCheckpointFilters,
+  ManagedWriteCheckpointOptions,
+  WriteCheckpointAPI,
+  WriteCheckpointMode
+} from '../write-checkpoint.js';
+import { PowerSyncMongo } from './db.js';
+
+export type MongoCheckpointAPIOptions = {
+  db: PowerSyncMongo;
+  mode: WriteCheckpointMode;
+};
+
+export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
+  readonly db: PowerSyncMongo;
+  readonly mode: WriteCheckpointMode;
+
+  constructor(options: MongoCheckpointAPIOptions) {
+    this.db = options.db;
+    this.mode = options.mode;
+  }
+
+  async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
+    return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
+  }
+
+  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
+    if (this.mode !== WriteCheckpointMode.CUSTOM) {
+      throw new framework.errors.ValidationError(
+        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
+      );
+    }
+
+    const { checkpoint, user_id, sync_rules_id } = options;
+    const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
+      {
+        user_id: user_id,
+        sync_rules_id
+      },
+      {
+        $set: {
+          checkpoint
+        }
+      },
+      { upsert: true, returnDocument: 'after' }
+    );
+    return doc!.checkpoint;
+  }
+
+  async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
+    if (this.mode !== WriteCheckpointMode.MANAGED) {
+      throw new framework.errors.ValidationError(
+        `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
+      );
+    }
+
+    const { user_id, heads: lsns } = checkpoint;
+    const doc = await this.db.write_checkpoints.findOneAndUpdate(
+      {
+        user_id: user_id
+      },
+      {
+        $set: {
+          lsns
+        },
+        $inc: {
+          client_id: 1n
+        }
+      },
+      { upsert: true, returnDocument: 'after' }
+    );
+    return doc!.client_id;
+  }
+
+  async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
+    switch (this.mode) {
+      case WriteCheckpointMode.CUSTOM:
+        if (false == 'sync_rules_id' in filters) {
+          throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
+        }
+        return this.lastCustomWriteCheckpoint(filters);
+      case WriteCheckpointMode.MANAGED:
+        if (false == 'heads' in filters) {
+          throw new framework.errors.ValidationError(
+            `Replication HEAD is required for managed Write Checkpoint filtering`
+          );
+        }
+        return this.lastManagedWriteCheckpoint(filters);
+    }
+  }
+
+  protected async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters) {
+    const { user_id, sync_rules_id } = filters;
+    const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
+      user_id,
+      sync_rules_id
+    });
+    return lastWriteCheckpoint?.checkpoint ?? null;
+  }
+
+  protected async lastManagedWriteCheckpoint(filters: ManagedWriteCheckpointFilters) {
+    const { user_id } = filters;
+    const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
+      user_id: user_id
+    });
+    return lastWriteCheckpoint?.client_id ?? null;
+  }
+}
+
+export async function batchCreateCustomWriteCheckpoints(
+  db: PowerSyncMongo,
+  checkpoints: CustomWriteCheckpointOptions[]
+): Promise<void> {
+  if (!checkpoints.length) {
+    return;
+  }
+
+  await db.custom_write_checkpoints.bulkWrite(
+    checkpoints.map((checkpointOptions) => ({
+      updateOne: {
+        filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
+        update: {
+          $set: {
+            checkpoint: checkpointOptions.checkpoint,
+            sync_rules_id: checkpointOptions.sync_rules_id
+          }
+        },
+        upsert: true
+      }
+    }))
+  );
+}

package/src/storage/mongo/db.ts

@@ -1,11 +1,13 @@
 import * as mongo from 'mongodb';
 
+import { configFile } from '@powersync/service-types';
 import * as db from '../../db/db-index.js';
 import * as locks from '../../locks/locks-index.js';
 import {
   BucketDataDocument,
   BucketParameterDocument,
   CurrentDataDocument,
+  CustomWriteCheckpointDocument,
   IdSequenceDocument,
   InstanceDocument,
   SourceTableDocument,
@@ -13,7 +15,6 @@ import {
   WriteCheckpointDocument
 } from './models.js';
 import { BSON_DESERIALIZE_OPTIONS } from './util.js';
-import { configFile } from '@powersync/service-types';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -33,6 +34,7 @@ export class PowerSyncMongo {
   readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
   readonly sync_rules: mongo.Collection<SyncRuleDocument>;
   readonly source_tables: mongo.Collection<SourceTableDocument>;
+  readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
   readonly instance: mongo.Collection<InstanceDocument>;
   readonly locks: mongo.Collection<locks.Lock>;
@@ -54,6 +56,7 @@ export class PowerSyncMongo {
     this.op_id_sequence = db.collection('op_id_sequence');
     this.sync_rules = db.collection('sync_rules');
     this.source_tables = db.collection('source_tables');
+    this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
     this.write_checkpoints = db.collection('write_checkpoints');
     this.instance = db.collection('instance');
     this.locks = this.db.collection('locks');

package/src/storage/mongo/models.ts

@@ -1,5 +1,5 @@
-import * as bson from 'bson';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -159,6 +159,13 @@ export interface SyncRuleDocument {
   content: string;
 }
 
+export interface CustomWriteCheckpointDocument {
+  _id: bson.ObjectId;
+  user_id: string;
+  checkpoint: bigint;
+  sync_rules_id: number;
+}
+
 export interface WriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;

package/src/storage/storage-index.ts

@@ -1,5 +1,6 @@
 export * from './BucketStorage.js';
 export * from './MongoBucketStorage.js';
+export * from './ReplicationEventPayload.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
@@ -17,3 +18,4 @@ export * from './mongo/OperationBatch.js';
 export * from './mongo/PersistedBatch.js';
 export * from './mongo/util.js';
 export * from './mongo/config.js';
+export * from './write-checkpoint.js';

package/src/storage/write-checkpoint.ts

@@ -0,0 +1,67 @@
+export enum WriteCheckpointMode {
+  /**
+   * Raw mappings of `user_id` to `write_checkpoint`s should
+   * be supplied for each set of sync rules.
+   */
+  CUSTOM = 'manual',
+  /**
+   * Write checkpoints are stored as a mapping of `user_id` plus
+   * replication HEAD (lsn in Postgres) to an automatically generated
+   * incrementing `write_checkpoint` (stored as`client_id`).
+   */
+  MANAGED = 'managed'
+}
+
+export interface BaseWriteCheckpointIdentifier {
+  /**
+   * Identifier for User's account.
+   */
+  user_id: string;
+}
+
+export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+  /**
+   * Sync rules which were active when this checkpoint was created.
+   */
+  sync_rules_id: number;
+}
+
+export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilters {
+  /**
+   * A supplied incrementing Write Checkpoint number
+   */
+  checkpoint: bigint;
+}
+
+/**
+ * Options for creating a custom Write Checkpoint in a batch.
+ * A {@link BucketStorageBatch} is already associated with a Sync Rules instance.
+ * The `sync_rules_id` is not required here.
+ */
+export type BatchedCustomWriteCheckpointOptions = Omit<CustomWriteCheckpointOptions, 'sync_rules_id'>;
+
+/**
+ * Managed Write Checkpoints are a mapping of User ID to replication HEAD
+ */
+export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+  /**
+   * Replication HEAD(s) at the creation of the checkpoint.
+   */
+  heads: Record<string, string>;
+}
+
+export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
+
+export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
+
+export interface WriteCheckpointAPI {
+  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
+
+  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
+
+  createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
+
+  lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
+}
+
+export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;

package/src/util/protocol-types.ts

@@ -88,7 +88,7 @@ export type StreamingSyncLine =
  */
 export type OpId = string;
 
-export interface Checkpoint {
+interface Checkpoint {
   last_op_id: OpId;
   write_checkpoint?: OpId;
   buckets: BucketChecksum[];

package/test/src/compacting.test.ts

@@ -1,11 +1,9 @@
+import { SaveOperationTag } from '@/storage/BucketStorage.js';
 import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { validateCompactedBucket } from './bucket_validation.js';
 import { oneFromAsync } from './stream_utils.js';
-import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules, ZERO_LSN } from './util.js';
-import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
-import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -31,7 +29,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1'
         },
@@ -40,7 +38,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2'
         },
@@ -49,7 +47,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't2'
         },
@@ -128,7 +126,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1'
         },
@@ -137,7 +135,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2'
         },
@@ -146,7 +144,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'delete',
+        tag: SaveOperationTag.DELETE,
         before: {
           id: 't1'
         },
@@ -155,7 +153,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't2'
         },
@@ -233,7 +231,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1'
         },
@@ -242,7 +240,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2'
         },
@@ -251,7 +249,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'delete',
+        tag: SaveOperationTag.DELETE,
         before: {
           id: 't1'
         },
@@ -265,7 +263,7 @@ bucket_definitions:
     const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'delete',
+        tag: SaveOperationTag.DELETE,
         before: {
           id: 't2'
         },