@powersync/service-core 0.0.0-dev-20241007145127 → 0.0.0-dev-20241015210820

package/src/storage/MongoBucketStorage.ts

@@ -8,11 +8,12 @@ import * as locks from '../locks/locks-index.js';
 import * as sync from '../sync/sync-index.js';
 import * as util from '../util/util-index.js';
 
-import { logger } from '@powersync/lib-services-framework';
+import { DisposableObserver, logger } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 import {
   ActiveCheckpoint,
   BucketStorageFactory,
+  BucketStorageFactoryListener,
   ParseSyncRulesOptions,
   PersistedSyncRules,
   PersistedSyncRulesContent,
@@ -20,20 +21,36 @@ import {
   UpdateSyncRulesOptions,
   WriteCheckpoint
 } from './BucketStorage.js';
-import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
-import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
 import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js';
 import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
+import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
+import { MongoWriteCheckpointAPI } from './mongo/MongoWriteCheckpointAPI.js';
 import { generateSlotName } from './mongo/util.js';
+import {
+  CustomWriteCheckpointOptions,
+  DEFAULT_WRITE_CHECKPOINT_MODE,
+  LastWriteCheckpointFilters,
+  ManagedWriteCheckpointOptions,
+  WriteCheckpointAPI,
+  WriteCheckpointMode
+} from './write-checkpoint.js';
 
 export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {}
 
-export class MongoBucketStorage implements BucketStorageFactory {
+export class MongoBucketStorage
+  extends DisposableObserver<BucketStorageFactoryListener>
+  implements BucketStorageFactory
+{
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
   // TODO: This is still Postgres specific and needs to be reworked
   public readonly slot_name_prefix: string;
 
+  readonly write_checkpoint_mode: WriteCheckpointMode;
+
+  protected readonly writeCheckpointAPI: WriteCheckpointAPI;
+
   private readonly storageCache = new LRUCache<number, MongoSyncBucketStorage>({
     max: 3,
     fetchMethod: async (id) => {
@@ -49,16 +66,31 @@ export class MongoBucketStorage implements BucketStorageFactory {
       }
       const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
       return this.getInstance(rules);
+    },
+    dispose: (storage) => {
+      storage[Symbol.dispose]();
     }
   });
 
   public readonly db: PowerSyncMongo;
 
-  constructor(db: PowerSyncMongo, options: { slot_name_prefix: string }) {
+  constructor(
+    db: PowerSyncMongo,
+    options: {
+      slot_name_prefix: string;
+      write_checkpoint_mode?: WriteCheckpointMode;
+    }
+  ) {
+    super();
     this.client = db.client;
     this.db = db;
     this.session = this.client.startSession();
     this.slot_name_prefix = options.slot_name_prefix;
+    this.write_checkpoint_mode = options.write_checkpoint_mode ?? DEFAULT_WRITE_CHECKPOINT_MODE;
+    this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
+      db,
+      mode: this.write_checkpoint_mode
+    });
   }
 
   getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage {
@@ -66,7 +98,17 @@ export class MongoBucketStorage implements BucketStorageFactory {
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    return new MongoSyncBucketStorage(this, id, options, slot_name);
+    const storage = new MongoSyncBucketStorage(this, id, options, slot_name);
+    this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+    storage.registerListener({
+      batchStarted: (batch) => {
+        // This nested listener will be automatically disposed when the storage is disposed
+        batch.registerManagedListener(storage, {
+          replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
+        });
+      }
+    });
+    return storage;
   }
 
   async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) {
@@ -257,30 +299,20 @@ export class MongoBucketStorage implements BucketStorageFactory {
     });
   }
 
-  async createWriteCheckpoint(user_id: string, lsns: Record<string, string>): Promise<bigint> {
-    const doc = await this.db.write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id
-      },
-      {
-        $set: {
-          lsns: lsns
-        },
-        $inc: {
-          client_id: 1n
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
-    );
-    return doc!.client_id;
+  async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
+    return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
   }
 
-  async lastWriteCheckpoint(user_id: string, lsn: string): Promise<bigint | null> {
-    const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
-      user_id: user_id,
-      'lsns.1': { $lte: lsn }
-    });
-    return lastWriteCheckpoint?.client_id ?? null;
+  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
+    return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
+  }
+
+  async createManagedWriteCheckpoint(options: ManagedWriteCheckpointOptions): Promise<bigint> {
+    return this.writeCheckpointAPI.createManagedWriteCheckpoint(options);
+  }
+
+  async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
+    return this.writeCheckpointAPI.lastWriteCheckpoint(filters);
   }
 
   async getActiveCheckpoint(): Promise<ActiveCheckpoint> {
@@ -496,8 +528,17 @@ export class MongoBucketStorage implements BucketStorageFactory {
       // What is important is:
       // 1. checkpoint (op_id) changes.
       // 2. write checkpoint changes for the specific user
+      const bucketStorage = await cp.getBucketStorage();
 
-      const currentWriteCheckpoint = await this.lastWriteCheckpoint(user_id, lsn ?? '');
+      const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
+
+      const currentWriteCheckpoint = await this.lastWriteCheckpoint({
+        user_id,
+        sync_rules_id: bucketStorage?.group_id,
+        heads: {
+          ...lsnFilters
+        }
+      });
 
       if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
         // No change - wait for next one

package/src/storage/ReplicationEventPayload.ts

@@ -0,0 +1,16 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import { BucketStorageBatch, SaveOp } from './BucketStorage.js';
+import { SourceTable } from './SourceTable.js';
+
+export type EventData = {
+  op: SaveOp;
+  before?: sync_rules.SqliteRow;
+  after?: sync_rules.SqliteRow;
+};
+
+export type ReplicationEventPayload = {
+  batch: BucketStorageBatch;
+  data: EventData;
+  event: sync_rules.SqlEventDescriptor;
+  table: SourceTable;
+};

package/src/storage/SourceTable.ts

@@ -23,6 +23,15 @@ export class SourceTable {
    */
   public syncParameters = true;
 
+  /**
+   * True if the table is used in sync rules for events.
+   *
+   * This value is resolved externally, and cached here.
+   *
+   * Defaults to true for tests.
+   */
+  public syncEvent = true;
+
   constructor(
     public readonly id: any,
     public readonly connectionTag: string,
@@ -53,6 +62,6 @@ export class SourceTable {
   }
 
   get syncAny() {
-    return this.syncData || this.syncParameters;
+    return this.syncData || this.syncParameters || this.syncEvent;
   }
 }

package/src/storage/StorageEngine.ts

@@ -1,18 +1,31 @@
+import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
 import { ResolvedPowerSyncConfig } from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorage.js';
-import { BucketStorageProvider, ActiveStorage } from './StorageProvider.js';
-import { logger } from '@powersync/lib-services-framework';
+import { ActiveStorage, BucketStorageProvider, StorageSettings } from './StorageProvider.js';
+import { DEFAULT_WRITE_CHECKPOINT_MODE } from './write-checkpoint.js';
 
 export type StorageEngineOptions = {
   configuration: ResolvedPowerSyncConfig;
 };
 
-export class StorageEngine {
+export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
+  writeCheckpointMode: DEFAULT_WRITE_CHECKPOINT_MODE
+};
+
+export interface StorageEngineListener extends DisposableListener {
+  storageActivated: (storage: BucketStorageFactory) => void;
+}
+
+export class StorageEngine extends DisposableObserver<StorageEngineListener> {
   // TODO: This will need to revisited when we actually support multiple storage providers.
   private storageProviders: Map<string, BucketStorageProvider> = new Map();
   private currentActiveStorage: ActiveStorage | null = null;
+  private _activeSettings: StorageSettings;
 
-  constructor(private options: StorageEngineOptions) {}
+  constructor(private options: StorageEngineOptions) {
+    super();
+    this._activeSettings = DEFAULT_STORAGE_SETTINGS;
+  }
 
   get activeBucketStorage(): BucketStorageFactory {
     return this.activeStorage.storage;
@@ -26,6 +39,20 @@ export class StorageEngine {
     return this.currentActiveStorage;
   }
 
+  get activeSettings(): StorageSettings {
+    return { ...this._activeSettings };
+  }
+
+  updateSettings(settings: Partial<StorageSettings>) {
+    if (this.currentActiveStorage) {
+      throw new Error(`Storage is already active, settings cannot be modified.`);
+    }
+    this._activeSettings = {
+      ...this._activeSettings,
+      ...settings
+    };
+  }
+
   /**
    * Register a provider which generates a {@link BucketStorageFactory}
    * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
@@ -38,8 +65,10 @@ export class StorageEngine {
     logger.info('Starting Storage Engine...');
     const { configuration } = this.options;
     this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
-      resolvedConfig: configuration
+      resolvedConfig: configuration,
+      ...this.activeSettings
     });
+    this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
     logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
     logger.info('Successfully started Storage Engine.');
   }

package/src/storage/StorageProvider.ts

@@ -1,5 +1,6 @@
-import { BucketStorageFactory } from './BucketStorage.js';
 import * as util from '../util/util-index.js';
+import { BucketStorageFactory } from './BucketStorage.js';
+import { WriteCheckpointMode } from './write-checkpoint.js';
 
 export interface ActiveStorage {
   storage: BucketStorageFactory;
@@ -11,7 +12,14 @@ export interface ActiveStorage {
   tearDown(): Promise<boolean>;
 }
 
-export interface GetStorageOptions {
+/**
+ * Settings which can be modified by various modules in their initialization.
+ */
+export interface StorageSettings {
+  writeCheckpointMode: WriteCheckpointMode;
+}
+
+export interface GetStorageOptions extends StorageSettings {
   // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
   resolvedConfig: util.ResolvedPowerSyncConfig;
 }

package/src/storage/mongo/MongoBucketBatch.ts

@@ -1,14 +1,22 @@
-import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
+import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import * as mongo from 'mongodb';
 
-import { container, errors, logger } from '@powersync/lib-services-framework';
+import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework';
 import * as util from '../../util/util-index.js';
-import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js';
+import {
+  BucketBatchStorageListener,
+  BucketStorageBatch,
+  FlushedResult,
+  mergeToast,
+  SaveOptions
+} from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
+import { CustomWriteCheckpointOptions } from '../write-checkpoint.js';
 import { PowerSyncMongo } from './db.js';
 import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
 import { MongoIdSequence } from './MongoIdSequence.js';
+import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
@@ -25,7 +33,7 @@ const MAX_ROW_SIZE = 15 * 1024 * 1024;
 // In the future, we can investigate allowing multiple replication streams operating independently.
 const replicationMutex = new util.Mutex();
 
-export class MongoBucketBatch implements BucketStorageBatch {
+export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListener> implements BucketStorageBatch {
   private readonly client: mongo.MongoClient;
   public readonly db: PowerSyncMongo;
   public readonly session: mongo.ClientSession;
@@ -36,6 +44,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
   private readonly slot_name: string;
 
   private batch: OperationBatch | null = null;
+  private write_checkpoint_batch: CustomWriteCheckpointOptions[] = [];
 
   /**
    * Last LSN received associated with a checkpoint.
@@ -63,14 +72,23 @@ export class MongoBucketBatch implements BucketStorageBatch {
     last_checkpoint_lsn: string | null,
     no_checkpoint_before_lsn: string
   ) {
-    this.db = db;
+    super();
     this.client = db.client;
-    this.sync_rules = sync_rules;
+    this.db = db;
     this.group_id = group_id;
-    this.slot_name = slot_name;
-    this.session = this.client.startSession();
     this.last_checkpoint_lsn = last_checkpoint_lsn;
     this.no_checkpoint_before_lsn = no_checkpoint_before_lsn;
+    this.session = this.client.startSession();
+    this.slot_name = slot_name;
+    this.sync_rules = sync_rules;
+    this.batch = new OperationBatch();
+  }
+
+  addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void {
+    this.write_checkpoint_batch.push({
+      ...checkpoint,
+      sync_rules_id: this.group_id
+    });
   }
 
   get lastCheckpointLsn() {
@@ -87,6 +105,8 @@ export class MongoBucketBatch implements BucketStorageBatch {
         result = r;
       }
     }
+    await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch);
+    this.write_checkpoint_batch = [];
     return result;
   }
 
@@ -532,8 +552,9 @@ export class MongoBucketBatch implements BucketStorageBatch {
     });
   }
 
-  async abort() {
+  async [Symbol.asyncDispose]() {
     await this.session.endSession();
+    super[Symbol.dispose]();
   }
 
   async commit(lsn: string): Promise<boolean> {
@@ -550,26 +571,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
       return false;
     }
 
+    const now = new Date();
+    const update: Partial<SyncRuleDocument> = {
+      last_checkpoint_lsn: lsn,
+      last_checkpoint_ts: now,
+      last_keepalive_ts: now,
+      snapshot_done: true,
+      last_fatal_error: null
+    };
+
     if (this.persisted_op != null) {
-      const now = new Date();
-      await this.db.sync_rules.updateOne(
-        {
-          _id: this.group_id
-        },
-        {
-          $set: {
-            last_checkpoint: this.persisted_op,
-            last_checkpoint_lsn: lsn,
-            last_checkpoint_ts: now,
-            last_keepalive_ts: now,
-            snapshot_done: true,
-            last_fatal_error: null
-          }
-        },
-        { session: this.session }
-      );
-      this.persisted_op = null;
+      update.last_checkpoint = this.persisted_op;
     }
+
+    await this.db.sync_rules.updateOne(
+      {
+        _id: this.group_id
+      },
+      {
+        $set: update
+      },
+      { session: this.session }
+    );
+    this.persisted_op = null;
     this.last_checkpoint_lsn = lsn;
     return true;
   }
@@ -610,6 +634,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
   }
 
   async save(record: SaveOptions): Promise<FlushedResult | null> {
+    const { after, before, sourceTable, tag } = record;
+    for (const event of this.getTableEvents(sourceTable)) {
+      this.iterateListeners((cb) =>
+        cb.replicationEvent?.({
+          batch: this,
+          table: sourceTable,
+          data: {
+            op: tag,
+            after: after && util.isCompleteRow(after) ? after : undefined,
+            before: before && util.isCompleteRow(before) ? before : undefined
+          },
+          event
+        })
+      );
+    }
+
+    /**
+     * Return if the table is just an event table
+     */
+    if (!sourceTable.syncData && !sourceTable.syncParameters) {
+      return null;
+    }
+
     logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
 
     this.batch ??= new OperationBatch();
@@ -758,6 +805,15 @@ export class MongoBucketBatch implements BucketStorageBatch {
       return copy;
     });
   }
+
+  /**
+   * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
+   */
+  protected getTableEvents(table: SourceTable): SqlEventDescriptor[] {
+    return this.sync_rules.event_descriptors.filter((evt) =>
+      [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
+    );
+  }
 }
 
 export function currentBucketKey(b: CurrentBucket) {

package/src/storage/mongo/MongoStorageProvider.ts

@@ -1,8 +1,8 @@
+import { logger } from '@powersync/lib-services-framework';
 import * as db from '../../db/db-index.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { BucketStorageProvider, ActiveStorage, GetStorageOptions } from '../StorageProvider.js';
+import { ActiveStorage, BucketStorageProvider, GetStorageOptions } from '../StorageProvider.js';
 import { PowerSyncMongo } from './db.js';
-import { logger } from '@powersync/lib-services-framework';
 
 export class MongoStorageProvider implements BucketStorageProvider {
   get type() {
@@ -19,7 +19,8 @@ export class MongoStorageProvider implements BucketStorageProvider {
     return {
       storage: new MongoBucketStorage(database, {
         // TODO currently need the entire resolved config due to this
-        slot_name_prefix: resolvedConfig.slot_name_prefix
+        slot_name_prefix: resolvedConfig.slot_name_prefix,
+        write_checkpoint_mode: options.writeCheckpointMode
       }),
       shutDown: () => client.close(),
       tearDown: () => {

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -2,23 +2,25 @@ import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service
 import * as bson from 'bson';
 import * as mongo from 'mongodb';
 
+import { DisposableObserver } from '@powersync/lib-services-framework';
 import * as db from '../../db/db-index.js';
 import * as util from '../../util/util-index.js';
 import {
   BucketDataBatchOptions,
   BucketStorageBatch,
+  Checkpoint,
   CompactOptions,
   DEFAULT_DOCUMENT_BATCH_LIMIT,
   DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES,
   FlushedResult,
   ParseSyncRulesOptions,
-  PersistedSyncRules,
   PersistedSyncRulesContent,
   ResolveTableOptions,
   ResolveTableResult,
   StartBatchOptions,
   SyncBucketDataBatch,
   SyncRulesBucketStorage,
+  SyncRulesBucketStorageListener,
   SyncRuleStatus,
   TerminateOptions
 } from '../BucketStorage.js';
@@ -31,7 +33,10 @@ import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
 
-export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
+export class MongoSyncBucketStorage
+  extends DisposableObserver<SyncRulesBucketStorageListener>
+  implements SyncRulesBucketStorage
+{
   private readonly db: PowerSyncMongo;
   private checksumCache = new ChecksumCache({
     fetchChecksums: (batch) => {
@@ -47,6 +52,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
     private readonly sync_rules: PersistedSyncRulesContent,
     public readonly slot_name: string
   ) {
+    super();
     this.db = factory.db;
   }
 
@@ -55,15 +61,16 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
     return this.parsedSyncRulesCache;
   }
 
-  async getCheckpoint() {
+  async getCheckpoint(): Promise<Checkpoint> {
     const doc = await this.db.sync_rules.findOne(
       { _id: this.group_id },
       {
-        projection: { last_checkpoint: 1 }
+        projection: { last_checkpoint: 1, last_checkpoint_lsn: 1 }
       }
     );
     return {
-      checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n)
+      checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
+      lsn: doc?.last_checkpoint_lsn ?? null
     };
   }
 
@@ -79,7 +86,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
     );
     const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null;
 
-    const batch = new MongoBucketBatch(
+    await using batch = new MongoBucketBatch(
       this.db,
       this.sync_rules.parsed(options).sync_rules,
       this.group_id,
@@ -87,18 +94,14 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
       checkpoint_lsn,
       doc?.no_checkpoint_before ?? options.zeroLSN
     );
-    try {
-      await callback(batch);
-      await batch.flush();
-      await batch.abort();
-      if (batch.last_flushed_op) {
-        return { flushed_op: String(batch.last_flushed_op) };
-      } else {
-        return null;
-      }
-    } catch (e) {
-      await batch.abort();
-      throw e;
+    this.iterateListeners((cb) => cb.batchStarted?.(batch));
+
+    await callback(batch);
+    await batch.flush();
+    if (batch.last_flushed_op) {
+      return { flushed_op: String(batch.last_flushed_op) };
+    } else {
+      return null;
     }
   }
 
@@ -150,6 +153,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
         replicationColumns,
         doc.snapshot_done ?? true
       );
+      sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable);
       sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
       sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);