@powersync/service-core 0.0.0-dev-20241007120318 → 0.0.0-dev-20241015084348

package/src/storage/mongo/MongoWriteCheckpointAPI.ts

@@ -0,0 +1,136 @@
+import * as framework from '@powersync/lib-services-framework';
+import {
+  CustomWriteCheckpointFilters,
+  CustomWriteCheckpointOptions,
+  LastWriteCheckpointFilters,
+  ManagedWriteCheckpointFilters,
+  ManagedWriteCheckpointOptions,
+  WriteCheckpointAPI,
+  WriteCheckpointMode
+} from '../write-checkpoint.js';
+import { PowerSyncMongo } from './db.js';
+
+export type MongoCheckpointAPIOptions = {
+  db: PowerSyncMongo;
+  mode: WriteCheckpointMode;
+};
+
+export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
+  readonly db: PowerSyncMongo;
+  readonly mode: WriteCheckpointMode;
+
+  constructor(options: MongoCheckpointAPIOptions) {
+    this.db = options.db;
+    this.mode = options.mode;
+  }
+
+  async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
+    return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
+  }
+
+  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
+    if (this.mode !== WriteCheckpointMode.CUSTOM) {
+      throw new framework.errors.ValidationError(
+        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
+      );
+    }
+
+    const { checkpoint, user_id, sync_rules_id } = options;
+    const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
+      {
+        user_id: user_id,
+        sync_rules_id
+      },
+      {
+        $set: {
+          checkpoint
+        }
+      },
+      { upsert: true, returnDocument: 'after' }
+    );
+    return doc!.checkpoint;
+  }
+
+  async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
+    if (this.mode !== WriteCheckpointMode.MANAGED) {
+      throw new framework.errors.ValidationError(
+        `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
+      );
+    }
+
+    const { user_id, heads: lsns } = checkpoint;
+    const doc = await this.db.write_checkpoints.findOneAndUpdate(
+      {
+        user_id: user_id
+      },
+      {
+        $set: {
+          lsns
+        },
+        $inc: {
+          client_id: 1n
+        }
+      },
+      { upsert: true, returnDocument: 'after' }
+    );
+    return doc!.client_id;
+  }
+
+  async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
+    switch (this.mode) {
+      case WriteCheckpointMode.CUSTOM:
+        if (false == 'sync_rules_id' in filters) {
+          throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
+        }
+        return this.lastCustomWriteCheckpoint(filters);
+      case WriteCheckpointMode.MANAGED:
+        if (false == 'heads' in filters) {
+          throw new framework.errors.ValidationError(
+            `Replication HEAD is required for managed Write Checkpoint filtering`
+          );
+        }
+        return this.lastManagedWriteCheckpoint(filters);
+    }
+  }
+
+  protected async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters) {
+    const { user_id, sync_rules_id } = filters;
+    const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
+      user_id,
+      sync_rules_id
+    });
+    return lastWriteCheckpoint?.checkpoint ?? null;
+  }
+
+  protected async lastManagedWriteCheckpoint(filters: ManagedWriteCheckpointFilters) {
+    const { user_id } = filters;
+    const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
+      user_id: user_id
+    });
+    return lastWriteCheckpoint?.client_id ?? null;
+  }
+}
+
+export async function batchCreateCustomWriteCheckpoints(
+  db: PowerSyncMongo,
+  checkpoints: CustomWriteCheckpointOptions[]
+): Promise<void> {
+  if (!checkpoints.length) {
+    return;
+  }
+
+  await db.custom_write_checkpoints.bulkWrite(
+    checkpoints.map((checkpointOptions) => ({
+      updateOne: {
+        filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
+        update: {
+          $set: {
+            checkpoint: checkpointOptions.checkpoint,
+            sync_rules_id: checkpointOptions.sync_rules_id
+          }
+        },
+        upsert: true
+      }
+    }))
+  );
+}

package/src/storage/mongo/db.ts

@@ -1,11 +1,13 @@
 import * as mongo from 'mongodb';
 
+import { configFile } from '@powersync/service-types';
 import * as db from '../../db/db-index.js';
 import * as locks from '../../locks/locks-index.js';
 import {
   BucketDataDocument,
   BucketParameterDocument,
   CurrentDataDocument,
+  CustomWriteCheckpointDocument,
   IdSequenceDocument,
   InstanceDocument,
   SourceTableDocument,
@@ -13,7 +15,6 @@ import {
   WriteCheckpointDocument
 } from './models.js';
 import { BSON_DESERIALIZE_OPTIONS } from './util.js';
-import { configFile } from '@powersync/service-types';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -33,6 +34,7 @@ export class PowerSyncMongo {
   readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
   readonly sync_rules: mongo.Collection<SyncRuleDocument>;
   readonly source_tables: mongo.Collection<SourceTableDocument>;
+  readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
   readonly instance: mongo.Collection<InstanceDocument>;
   readonly locks: mongo.Collection<locks.Lock>;
@@ -54,6 +56,7 @@ export class PowerSyncMongo {
     this.op_id_sequence = db.collection('op_id_sequence');
     this.sync_rules = db.collection('sync_rules');
     this.source_tables = db.collection('source_tables');
+    this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
     this.write_checkpoints = db.collection('write_checkpoints');
     this.instance = db.collection('instance');
     this.locks = this.db.collection('locks');

package/src/storage/mongo/models.ts

@@ -1,5 +1,5 @@
-import * as bson from 'bson';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -159,6 +159,13 @@ export interface SyncRuleDocument {
   content: string;
 }
 
+export interface CustomWriteCheckpointDocument {
+  _id: bson.ObjectId;
+  user_id: string;
+  checkpoint: bigint;
+  sync_rules_id: number;
+}
+
 export interface WriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;

package/src/storage/storage-index.ts

@@ -1,5 +1,6 @@
 export * from './BucketStorage.js';
 export * from './MongoBucketStorage.js';
+export * from './ReplicationEventPayload.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
@@ -17,3 +18,4 @@ export * from './mongo/OperationBatch.js';
 export * from './mongo/PersistedBatch.js';
 export * from './mongo/util.js';
 export * from './mongo/config.js';
+export * from './write-checkpoint.js';

package/src/storage/write-checkpoint.ts

@@ -0,0 +1,67 @@
+export enum WriteCheckpointMode {
+  /**
+   * Raw mappings of `user_id` to `write_checkpoint`s should
+   * be supplied for each set of sync rules.
+   */
+  CUSTOM = 'manual',
+  /**
+   * Write checkpoints are stored as a mapping of `user_id` plus
+   * replication HEAD (lsn in Postgres) to an automatically generated
+   * incrementing `write_checkpoint` (stored as`client_id`).
+   */
+  MANAGED = 'managed'
+}
+
+export interface BaseWriteCheckpointIdentifier {
+  /**
+   * Identifier for User's account.
+   */
+  user_id: string;
+}
+
+export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+  /**
+   * Sync rules which were active when this checkpoint was created.
+   */
+  sync_rules_id: number;
+}
+
+export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilters {
+  /**
+   * A supplied incrementing Write Checkpoint number
+   */
+  checkpoint: bigint;
+}
+
+/**
+ * Options for creating a custom Write Checkpoint in a batch.
+ * A {@link BucketStorageBatch} is already associated with a Sync Rules instance.
+ * The `sync_rules_id` is not required here.
+ */
+export type BatchedCustomWriteCheckpointOptions = Omit<CustomWriteCheckpointOptions, 'sync_rules_id'>;
+
+/**
+ * Managed Write Checkpoints are a mapping of User ID to replication HEAD
+ */
+export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+  /**
+   * Replication HEAD(s) at the creation of the checkpoint.
+   */
+  heads: Record<string, string>;
+}
+
+export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
+
+export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
+
+export interface WriteCheckpointAPI {
+  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
+
+  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
+
+  createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
+
+  lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
+}
+
+export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;

package/src/util/config/compound-config-collector.ts

@@ -122,7 +122,8 @@ export class CompoundConfigCollector {
       },
       // TODO maybe move this out of the connection or something
       // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
-      slot_name_prefix: 'powersync_'
+      slot_name_prefix: 'powersync_',
+      parameters: baseConfig.parameters ?? {}
     };
 
     return config;

package/src/util/config/types.ts

@@ -64,4 +64,5 @@ export type ResolvedPowerSyncConfig = {
 
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
+  parameters: Record<string, number | string | boolean | null>;
 };

package/test/src/data_storage.test.ts

@@ -1,10 +1,6 @@
-import {
-  BucketDataBatchOptions,
-  ParseSyncRulesOptions,
-  PersistedSyncRulesContent,
-  StartBatchOptions
-} from '@/storage/BucketStorage.js';
-import { RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
+import { BucketDataBatchOptions } from '@/storage/BucketStorage.js';
+import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
+import { RequestParameters } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { fromAsync, oneFromAsync } from './stream_utils.js';
 import {
@@ -16,10 +12,8 @@ import {
   PARSE_OPTIONS,
   rid,
   StorageFactory,
-  testRules,
-  ZERO_LSN
+  testRules
 } from './util.js';
-import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -1406,4 +1400,42 @@ bucket_definitions:
 
     expect(getBatchMeta(batch3)).toEqual(null);
   });
+
+  test('batch should be disposed automatically', async () => {
+    const sync_rules = testRules(`
+      bucket_definitions:
+        global:
+          data: [] 
+          `);
+
+    const storage = (await factory()).getInstance(sync_rules);
+
+    let isDisposed = false;
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      batch.registerListener({
+        disposed: () => {
+          isDisposed = true;
+        }
+      });
+    });
+    expect(isDisposed).true;
+
+    isDisposed = false;
+    let errorCaught = false;
+    try {
+      await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+        batch.registerListener({
+          disposed: () => {
+            isDisposed = true;
+          }
+        });
+        throw new Error(`Testing exceptions`);
+      });
+    } catch (ex) {
+      errorCaught = true;
+      expect(ex.message.includes('Testing')).true;
+    }
+    expect(errorCaught).true;
+    expect(isDisposed).true;
+  });
 }

package/test/src/util.ts

@@ -11,11 +11,10 @@ import { SourceTable } from '@/storage/SourceTable.js';
 import { PowerSyncMongo } from '@/storage/mongo/db.js';
 import { SyncBucketData } from '@/util/protocol-types.js';
 import { getUuidReplicaIdentityBson, hashData } from '@/util/utils.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import * as mongo from 'mongodb';
 import { env } from './env.js';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { ReplicaId } from '@/storage/storage-index.js';
 
 // The metrics need to be initialised before they can be used
 await Metrics.initialise({