@powersync/service-core 0.0.0-dev-20241015210820 → 0.0.0-dev-20241016113053

package/src/storage/BucketStorage.ts

@@ -90,6 +90,11 @@ export interface BucketStorageFactory
    */
   getActiveCheckpoint(): Promise<ActiveCheckpoint>;
 
+  /**
+   * Yields the latest sync checkpoint.
+   */
+  watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint>;
+
   /**
    * Yields the latest user write checkpoint whenever the sync checkpoint updates.
    */
@@ -106,20 +111,20 @@ export interface BucketStorageFactory
   getPowerSyncInstanceId(): Promise<string>;
 }
 
-export interface Checkpoint {
+export interface WriteCheckpoint {
+  base: ActiveCheckpoint;
+  writeCheckpoint: bigint | null;
+}
+
+export interface ActiveCheckpoint {
   readonly checkpoint: util.OpId;
   readonly lsn: string | null;
-}
 
-export interface ActiveCheckpoint extends Checkpoint {
   hasSyncRules(): boolean;
 
   getBucketStorage(): Promise<SyncRulesBucketStorage | null>;
-}
 
-export interface WriteCheckpoint {
-  base: ActiveCheckpoint;
-  writeCheckpoint: bigint | null;
+  syncRules: PersistedSyncRules | null;
 }
 
 export interface StorageMetrics {
@@ -221,7 +226,7 @@ export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRul
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getCheckpoint(): Promise<Checkpoint>;
+  getCheckpoint(): Promise<{ checkpoint: util.OpId }>;
 
   getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
 
@@ -389,14 +394,8 @@ export type SaveOp = 'insert' | 'update' | 'delete';
 
 export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete;
 
-export enum SaveOperationTag {
-  INSERT = 'insert',
-  UPDATE = 'update',
-  DELETE = 'delete'
-}
-
 export interface SaveInsert {
-  tag: SaveOperationTag.INSERT;
+  tag: 'insert';
   sourceTable: SourceTable;
   before?: undefined;
   beforeReplicaId?: undefined;
@@ -405,7 +404,7 @@ export interface SaveInsert {
 }
 
 export interface SaveUpdate {
-  tag: SaveOperationTag.UPDATE;
+  tag: 'update';
   sourceTable: SourceTable;
 
   /**
@@ -424,7 +423,7 @@ export interface SaveUpdate {
 }
 
 export interface SaveDelete {
-  tag: SaveOperationTag.DELETE;
+  tag: 'delete';
   sourceTable: SourceTable;
   before?: SqliteRow;
   beforeReplicaId: ReplicaId;

package/src/storage/MongoBucketStorage.ts

@@ -78,6 +78,9 @@ export class MongoBucketStorage
     db: PowerSyncMongo,
     options: {
       slot_name_prefix: string;
+      /**
+       * Initial Write Checkpoint Mode
+       */
       write_checkpoint_mode?: WriteCheckpointMode;
     }
   ) {
@@ -303,6 +306,10 @@ export class MongoBucketStorage
     return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
   }
 
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void {
+    return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
+  }
+
   async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
     return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
   }
@@ -408,14 +415,19 @@ export class MongoBucketStorage
           return null;
         }
         return (await this.storageCache.fetch(doc._id)) ?? null;
-      }
-    };
+      },
+      syncRules: doc
+        ? new MongoPersistedSyncRulesContent(this.db, doc).parsed({
+            defaultSchema: ''
+          })
+        : null
+    } satisfies ActiveCheckpoint;
   }
 
   /**
    * Instance-wide watch on the latest available checkpoint (op_id + lsn).
    */
-  private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
+  private async *_watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
     const pipeline: mongo.Document[] = [
       {
         $match: {
@@ -428,7 +440,8 @@ export class MongoBucketStorage
           operationType: 1,
           'fullDocument._id': 1,
           'fullDocument.last_checkpoint': 1,
-          'fullDocument.last_checkpoint_lsn': 1
+          'fullDocument.last_checkpoint_lsn': 1,
+          'fullDocument.content': 1
         }
       }
     ];
@@ -450,7 +463,8 @@ export class MongoBucketStorage
           projection: {
             _id: 1,
             last_checkpoint: 1,
-            last_checkpoint_lsn: 1
+            last_checkpoint_lsn: 1,
+            content: 1
           }
         }
       );
@@ -499,6 +513,7 @@ export class MongoBucketStorage
       if (doc == null) {
         continue;
       }
+
       const op = this.makeActiveCheckpoint(doc);
       // Check for LSN / checkpoint changes - ignore other metadata changes
       if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
@@ -510,9 +525,16 @@ export class MongoBucketStorage
 
   // Nothing is done here until a subscriber starts to iterate
   private readonly sharedIter = new sync.BroadcastIterable((signal) => {
-    return this.watchActiveCheckpoint(signal);
+    return this._watchActiveCheckpoint(signal);
   });
 
+  /**
+   * Watch changes to the active sync rules and checkpoint.
+   */
+  watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
+    return wrapWithAbort(this.sharedIter, signal);
+  }
+
   /**
    * User-specific watch on the latest checkpoint and/or write checkpoint.
    */

package/src/storage/mongo/MongoBucketBatch.ts

@@ -81,7 +81,6 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     this.session = this.client.startSession();
     this.slot_name = slot_name;
     this.sync_rules = sync_rules;
-    this.batch = new OperationBatch();
   }
 
   addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void {

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -8,7 +8,6 @@ import * as util from '../../util/util-index.js';
 import {
   BucketDataBatchOptions,
   BucketStorageBatch,
-  Checkpoint,
   CompactOptions,
   DEFAULT_DOCUMENT_BATCH_LIMIT,
   DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES,
@@ -61,16 +60,15 @@ export class MongoSyncBucketStorage
     return this.parsedSyncRulesCache;
   }
 
-  async getCheckpoint(): Promise<Checkpoint> {
+  async getCheckpoint() {
     const doc = await this.db.sync_rules.findOne(
       { _id: this.group_id },
       {
-        projection: { last_checkpoint: 1, last_checkpoint_lsn: 1 }
+        projection: { last_checkpoint: 1 }
       }
     );
     return {
-      checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
-      lsn: doc?.last_checkpoint_lsn ?? null
+      checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n)
     };
   }
 

package/src/storage/mongo/MongoWriteCheckpointAPI.ts

@@ -17,11 +17,19 @@ export type MongoCheckpointAPIOptions = {
 
 export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
   readonly db: PowerSyncMongo;
-  readonly mode: WriteCheckpointMode;
+  private _mode: WriteCheckpointMode;
 
   constructor(options: MongoCheckpointAPIOptions) {
     this.db = options.db;
-    this.mode = options.mode;
+    this._mode = options.mode;
+  }
+
+  get mode() {
+    return this._mode;
+  }
+
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void {
+    this._mode = mode;
   }
 
   async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
@@ -29,12 +37,11 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
   }
 
   async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
-    if (this.mode !== WriteCheckpointMode.CUSTOM) {
-      throw new framework.errors.ValidationError(
-        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
-      );
-    }
-
+    /**
+     * Allow creating custom checkpoints even if the current mode is not `custom`.
+     * There might be a state where the next sync rules rely on replicating custom
+     * write checkpoints, but the current active sync rules uses managed checkpoints.
+     */
     const { checkpoint, user_id, sync_rules_id } = options;
     const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
       {

package/src/storage/write-checkpoint.ts

@@ -55,6 +55,8 @@ export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
 
 export interface WriteCheckpointAPI {
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void;
+
   batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
 
   createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;

package/src/util/config/compound-config-collector.ts

@@ -122,7 +122,8 @@ export class CompoundConfigCollector {
       },
       // TODO maybe move this out of the connection or something
       // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
-      slot_name_prefix: 'powersync_'
+      slot_name_prefix: 'powersync_',
+      parameters: baseConfig.parameters ?? {}
     };
 
     return config;

package/src/util/config/types.ts

@@ -64,4 +64,5 @@ export type ResolvedPowerSyncConfig = {
 
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
+  parameters: Record<string, number | string | boolean | null>;
 };

package/src/util/protocol-types.ts

@@ -88,7 +88,7 @@ export type StreamingSyncLine =
  */
 export type OpId = string;
 
-interface Checkpoint {
+export interface Checkpoint {
   last_op_id: OpId;
   write_checkpoint?: OpId;
   buckets: BucketChecksum[];

package/test/src/compacting.test.ts

@@ -1,9 +1,11 @@
-import { SaveOperationTag } from '@/storage/BucketStorage.js';
 import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { validateCompactedBucket } from './bucket_validation.js';
 import { oneFromAsync } from './stream_utils.js';
-import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules, ZERO_LSN } from './util.js';
+import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
+import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -29,7 +31,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't1'
         },
@@ -38,7 +40,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't2'
         },
@@ -47,7 +49,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.UPDATE,
+        tag: 'update',
         after: {
           id: 't2'
         },
@@ -126,7 +128,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't1'
         },
@@ -135,7 +137,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't2'
         },
@@ -144,7 +146,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
+        tag: 'delete',
         before: {
           id: 't1'
         },
@@ -153,7 +155,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.UPDATE,
+        tag: 'update',
         after: {
           id: 't2'
         },
@@ -231,7 +233,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't1'
         },
@@ -240,7 +242,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't2'
         },
@@ -249,7 +251,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
+        tag: 'delete',
         before: {
           id: 't1'
         },
@@ -263,7 +265,7 @@ bucket_definitions:
     const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
+        tag: 'delete',
         before: {
           id: 't2'
         },