@powersync/service-core 0.0.0-dev-20241022094219 → 0.0.0-dev-20241023191639

package/src/storage/StorageEngine.ts

@@ -1,17 +1,12 @@
 import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
 import { ResolvedPowerSyncConfig } from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorage.js';
-import { ActiveStorage, BucketStorageProvider, StorageSettings } from './StorageProvider.js';
-import { DEFAULT_WRITE_CHECKPOINT_MODE } from './write-checkpoint.js';
+import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
 
 export type StorageEngineOptions = {
   configuration: ResolvedPowerSyncConfig;
 };
 
-export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
-  writeCheckpointMode: DEFAULT_WRITE_CHECKPOINT_MODE
-};
-
 export interface StorageEngineListener extends DisposableListener {
   storageActivated: (storage: BucketStorageFactory) => void;
 }
@@ -20,11 +15,9 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
   // TODO: This will need to revisited when we actually support multiple storage providers.
   private storageProviders: Map<string, BucketStorageProvider> = new Map();
   private currentActiveStorage: ActiveStorage | null = null;
-  private _activeSettings: StorageSettings;
 
   constructor(private options: StorageEngineOptions) {
     super();
-    this._activeSettings = DEFAULT_STORAGE_SETTINGS;
   }
 
   get activeBucketStorage(): BucketStorageFactory {
@@ -39,20 +32,6 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
     return this.currentActiveStorage;
   }
 
-  get activeSettings(): StorageSettings {
-    return { ...this._activeSettings };
-  }
-
-  updateSettings(settings: Partial<StorageSettings>) {
-    if (this.currentActiveStorage) {
-      throw new Error(`Storage is already active, settings cannot be modified.`);
-    }
-    this._activeSettings = {
-      ...this._activeSettings,
-      ...settings
-    };
-  }
-
   /**
    * Register a provider which generates a {@link BucketStorageFactory}
    * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
@@ -65,8 +44,7 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
     logger.info('Starting Storage Engine...');
     const { configuration } = this.options;
     this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
-      resolvedConfig: configuration,
-      ...this.activeSettings
+      resolvedConfig: configuration
     });
     this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
     logger.info(`Successfully activated storage: ${configuration.storage.type}.`);

package/src/storage/StorageProvider.ts

@@ -1,6 +1,5 @@
 import * as util from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorage.js';
-import { WriteCheckpointMode } from './write-checkpoint.js';
 
 export interface ActiveStorage {
   storage: BucketStorageFactory;
@@ -12,14 +11,7 @@ export interface ActiveStorage {
   tearDown(): Promise<boolean>;
 }
 
-/**
- * Settings which can be modified by various modules in their initialization.
- */
-export interface StorageSettings {
-  writeCheckpointMode: WriteCheckpointMode;
-}
-
-export interface GetStorageOptions extends StorageSettings {
+export interface GetStorageOptions {
   // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
   resolvedConfig: util.ResolvedPowerSyncConfig;
 }

package/src/storage/{write-checkpoint.ts → WriteCheckpointAPI.ts}

@@ -3,7 +3,7 @@ export enum WriteCheckpointMode {
    * Raw mappings of `user_id` to `write_checkpoint`s should
    * be supplied for each set of sync rules.
    */
-  CUSTOM = 'manual',
+  CUSTOM = 'custom',
   /**
    * Write checkpoints are stored as a mapping of `user_id` plus
    * replication HEAD (lsn in Postgres) to an automatically generated
@@ -26,19 +26,19 @@ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentif
   sync_rules_id: number;
 }
 
-export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilters {
+export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
   /**
    * A supplied incrementing Write Checkpoint number
    */
   checkpoint: bigint;
 }
 
-/**
- * Options for creating a custom Write Checkpoint in a batch.
- * A {@link BucketStorageBatch} is already associated with a Sync Rules instance.
- * The `sync_rules_id` is not required here.
- */
-export type BatchedCustomWriteCheckpointOptions = Omit<CustomWriteCheckpointOptions, 'sync_rules_id'>;
+export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
+  /**
+   * Sync rules which were active when this checkpoint was created.
+   */
+  sync_rules_id: number;
+}
 
 /**
  * Managed Write Checkpoints are a mapping of User ID to replication HEAD
@@ -52,15 +52,33 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
 
 export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
 
+export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
 
-export interface WriteCheckpointAPI {
-  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
-
-  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
-
+export interface BaseWriteCheckpointAPI {
+  readonly writeCheckpointMode: WriteCheckpointMode;
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void;
   createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
+}
+
+/**
+ * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
+ * This storage corresponds with a set of sync rules. These APIs don't require specifying a
+ * sync rules id.
+ */
+export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
+  batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
+  createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
+  lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
+}
 
+/**
+ * Write Checkpoint API which is interfaced directly with the storage layer. This requires
+ * sync rules identifiers for custom write checkpoints.
+ */
+export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
+  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
+  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
   lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
 }
 

package/src/storage/mongo/MongoBucketBatch.ts

@@ -12,7 +12,7 @@ import {
   SaveOptions
 } from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
-import { CustomWriteCheckpointOptions } from '../write-checkpoint.js';
+import { BatchedCustomWriteCheckpointOptions, CustomWriteCheckpointOptions } from '../WriteCheckpointAPI.js';
 import { PowerSyncMongo } from './db.js';
 import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
 import { MongoIdSequence } from './MongoIdSequence.js';
@@ -84,7 +84,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     this.batch = new OperationBatch();
   }
 
-  addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void {
+  addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void {
     this.write_checkpoint_batch.push({
       ...checkpoint,
       sync_rules_id: this.group_id

package/src/storage/mongo/MongoStorageProvider.ts

@@ -19,8 +19,7 @@ export class MongoStorageProvider implements BucketStorageProvider {
     return {
       storage: new MongoBucketStorage(database, {
         // TODO currently need the entire resolved config due to this
-        slot_name_prefix: resolvedConfig.slot_name_prefix,
-        write_checkpoint_mode: options.writeCheckpointMode
+        slot_name_prefix: resolvedConfig.slot_name_prefix
       }),
       shutDown: () => client.close(),
       tearDown: () => {

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -2,7 +2,8 @@ import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service
 import * as bson from 'bson';
 import * as mongo from 'mongodb';
 
-import { DisposableObserver } from '@powersync/lib-services-framework';
+import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import * as timers from 'timers/promises';
 import * as db from '../../db/db-index.js';
 import * as util from '../../util/util-index.js';
 import {
@@ -27,13 +28,19 @@ import {
 import { ChecksumCache, FetchPartialBucketChecksum, PartialChecksum, PartialChecksumMap } from '../ChecksumCache.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
+import {
+  BatchedCustomWriteCheckpointOptions,
+  ManagedWriteCheckpointOptions,
+  SyncStorageLastWriteCheckpointFilters,
+  WriteCheckpointAPI,
+  WriteCheckpointMode
+} from '../WriteCheckpointAPI.js';
 import { PowerSyncMongo } from './db.js';
 import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
+import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
-import { logger } from '@powersync/lib-services-framework';
-import * as timers from 'timers/promises';
 
 export class MongoSyncBucketStorage
   extends DisposableObserver<SyncRulesBucketStorageListener>
@@ -47,15 +54,53 @@ export class MongoSyncBucketStorage
   });
 
   private parsedSyncRulesCache: SqlSyncRules | undefined;
+  private writeCheckpointAPI: WriteCheckpointAPI;
 
   constructor(
     public readonly factory: MongoBucketStorage,
     public readonly group_id: number,
     private readonly sync_rules: PersistedSyncRulesContent,
-    public readonly slot_name: string
+    public readonly slot_name: string,
+    writeCheckpointMode: WriteCheckpointMode = WriteCheckpointMode.MANAGED
   ) {
     super();
     this.db = factory.db;
+    this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
+      db: this.db,
+      mode: writeCheckpointMode
+    });
+  }
+
+  get writeCheckpointMode() {
+    return this.writeCheckpointAPI.writeCheckpointMode;
+  }
+
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void {
+    this.writeCheckpointAPI.setWriteCheckpointMode(mode);
+  }
+
+  batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void> {
+    return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(
+      checkpoints.map((checkpoint) => ({ ...checkpoint, sync_rules_id: this.group_id }))
+    );
+  }
+
+  createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint> {
+    return this.writeCheckpointAPI.createCustomWriteCheckpoint({
+      ...checkpoint,
+      sync_rules_id: this.group_id
+    });
+  }
+
+  createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
+    return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
+  }
+
+  lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
+    return this.writeCheckpointAPI.lastWriteCheckpoint({
+      ...filters,
+      sync_rules_id: this.group_id
+    });
   }
 
   getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules {

package/src/storage/mongo/MongoWriteCheckpointAPI.ts

@@ -7,7 +7,7 @@ import {
   ManagedWriteCheckpointOptions,
   WriteCheckpointAPI,
   WriteCheckpointMode
-} from '../write-checkpoint.js';
+} from '../WriteCheckpointAPI.js';
 import { PowerSyncMongo } from './db.js';
 
 export type MongoCheckpointAPIOptions = {
@@ -17,11 +17,19 @@ export type MongoCheckpointAPIOptions = {
 
 export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
   readonly db: PowerSyncMongo;
-  readonly mode: WriteCheckpointMode;
+  private _mode: WriteCheckpointMode;
 
   constructor(options: MongoCheckpointAPIOptions) {
     this.db = options.db;
-    this.mode = options.mode;
+    this._mode = options.mode;
+  }
+
+  get writeCheckpointMode() {
+    return this._mode;
+  }
+
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void {
+    this._mode = mode;
   }
 
   async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
@@ -29,9 +37,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
   }
 
   async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
-    if (this.mode !== WriteCheckpointMode.CUSTOM) {
+    if (this.writeCheckpointMode !== WriteCheckpointMode.CUSTOM) {
       throw new framework.errors.ValidationError(
-        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
+        `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
       );
     }
 
@@ -52,9 +60,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
   }
 
   async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
-    if (this.mode !== WriteCheckpointMode.MANAGED) {
+    if (this.writeCheckpointMode !== WriteCheckpointMode.MANAGED) {
       throw new framework.errors.ValidationError(
-        `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
+        `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
       );
     }
 
@@ -77,7 +85,7 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
   }
 
   async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
-    switch (this.mode) {
+    switch (this.writeCheckpointMode) {
       case WriteCheckpointMode.CUSTOM:
         if (false == 'sync_rules_id' in filters) {
           throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);

package/src/storage/storage-index.ts

@@ -5,6 +5,7 @@ export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
 
+export * from './mongo/config.js';
 export * from './mongo/db.js';
 export * from './mongo/models.js';
 export * from './mongo/MongoBucketBatch.js';
@@ -17,5 +18,4 @@ export * from './mongo/MongoSyncRulesLock.js';
 export * from './mongo/OperationBatch.js';
 export * from './mongo/PersistedBatch.js';
 export * from './mongo/util.js';
-export * from './mongo/config.js';
-export * from './write-checkpoint.js';
+export * from './WriteCheckpointAPI.js';

package/src/util/config/compound-config-collector.ts

@@ -122,7 +122,8 @@ export class CompoundConfigCollector {
       },
       // TODO maybe move this out of the connection or something
       // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
-      slot_name_prefix: 'powersync_'
+      slot_name_prefix: 'powersync_',
+      parameters: baseConfig.parameters ?? {}
     };
 
     return config;

package/src/util/config/types.ts

@@ -64,4 +64,5 @@ export type ResolvedPowerSyncConfig = {
 
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
+  parameters: Record<string, number | string | boolean | null>;
 };