@powersync/service-core 0.8.7 → 0.9.0

package/src/storage/MongoBucketStorage.ts

@@ -1,36 +1,39 @@
-import * as mongo from 'mongodb';
-import * as timers from 'timers/promises';
-import { LRUCache } from 'lru-cache/min';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import { LRUCache } from 'lru-cache/min';
+import * as mongo from 'mongodb';
+import * as timers from 'timers/promises';
 
-import * as replication from '../replication/replication-index.js';
+import * as locks from '../locks/locks-index.js';
 import * as sync from '../sync/sync-index.js';
 import * as util from '../util/util-index.js';
-import * as locks from '../locks/locks-index.js';
 
+import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { v4 as uuid } from 'uuid';
 import {
   ActiveCheckpoint,
   BucketStorageFactory,
+  BucketStorageFactoryListener,
+  ParseSyncRulesOptions,
   PersistedSyncRules,
   PersistedSyncRulesContent,
   StorageMetrics,
   UpdateSyncRulesOptions,
   WriteCheckpoint
 } from './BucketStorage.js';
+import { PowerSyncMongo } from './mongo/db.js';
+import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
 import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
 import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
-import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js';
-import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
 import { generateSlotName } from './mongo/util.js';
-import { v4 as uuid } from 'uuid';
-import { logger } from '@powersync/lib-services-framework';
-
-export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {}
 
-export class MongoBucketStorage implements BucketStorageFactory {
+export class MongoBucketStorage
+  extends DisposableObserver<BucketStorageFactoryListener>
+  implements BucketStorageFactory
+{
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
+  // TODO: This is still Postgres specific and needs to be reworked
   public readonly slot_name_prefix: string;
 
   private readonly storageCache = new LRUCache<number, MongoSyncBucketStorage>({
@@ -47,26 +50,44 @@ export class MongoBucketStorage implements BucketStorageFactory {
         return undefined;
       }
       const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
-      const storage = this.getInstance(rules.parsed());
-      return storage;
+      return this.getInstance(rules);
+    },
+    dispose: (storage) => {
+      storage[Symbol.dispose]();
     }
   });
 
   public readonly db: PowerSyncMongo;
 
-  constructor(db: PowerSyncMongo, options: { slot_name_prefix: string }) {
+  constructor(
+    db: PowerSyncMongo,
+    options: {
+      slot_name_prefix: string;
+    }
+  ) {
+    super();
     this.client = db.client;
     this.db = db;
     this.session = this.client.startSession();
     this.slot_name_prefix = options.slot_name_prefix;
   }
 
-  getInstance(options: PersistedSyncRules): MongoSyncBucketStorage {
-    let { id, sync_rules, slot_name } = options;
+  getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage {
+    let { id, slot_name } = options;
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    return new MongoSyncBucketStorage(this, id, sync_rules, slot_name);
+    const storage = new MongoSyncBucketStorage(this, id, options, slot_name);
+    this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+    storage.registerListener({
+      batchStarted: (batch) => {
+        // This nested listener will be automatically disposed when the storage is disposed
+        batch.registerManagedListener(storage, {
+          replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
+        });
+      }
+    });
+    return storage;
   }
 
   async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) {
@@ -136,7 +157,12 @@ export class MongoBucketStorage implements BucketStorageFactory {
 
   async updateSyncRules(options: UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
     // Parse and validate before applying any changes
-    const parsed = SqlSyncRules.fromYaml(options.content);
+    const parsed = SqlSyncRules.fromYaml(options.content, {
+      // No schema-based validation at this point
+      schema: undefined,
+      defaultSchema: 'not_applicable', // Not needed for validation
+      throwOnError: true
+    });
 
     let rules: MongoPersistedSyncRulesContent | undefined = undefined;
 
@@ -204,9 +230,9 @@ export class MongoBucketStorage implements BucketStorageFactory {
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getActiveSyncRules(): Promise<PersistedSyncRules | null> {
+  async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getActiveSyncRulesContent();
-    return content?.parsed() ?? null;
+    return content?.parsed(options) ?? null;
   }
 
   async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
@@ -223,9 +249,9 @@ export class MongoBucketStorage implements BucketStorageFactory {
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getNextSyncRules(): Promise<PersistedSyncRules | null> {
+  async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getNextSyncRulesContent();
-    return content?.parsed() ?? null;
+    return content?.parsed(options) ?? null;
   }
 
   async getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]> {
@@ -252,32 +278,6 @@ export class MongoBucketStorage implements BucketStorageFactory {
     });
   }
 
-  async createWriteCheckpoint(user_id: string, lsns: Record<string, string>): Promise<bigint> {
-    const doc = await this.db.write_checkpoints.findOneAndUpdate(
-      {
-        user_id: user_id
-      },
-      {
-        $set: {
-          lsns: lsns
-        },
-        $inc: {
-          client_id: 1n
-        }
-      },
-      { upsert: true, returnDocument: 'after' }
-    );
-    return doc!.client_id;
-  }
-
-  async lastWriteCheckpoint(user_id: string, lsn: string): Promise<bigint | null> {
-    const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
-      user_id: user_id,
-      'lsns.1': { $lte: lsn }
-    });
-    return lastWriteCheckpoint?.client_id ?? null;
-  }
-
   async getActiveCheckpoint(): Promise<ActiveCheckpoint> {
     const doc = await this.db.sync_rules.findOne(
       {
@@ -303,7 +303,7 @@ export class MongoBucketStorage implements BucketStorageFactory {
       }
     };
 
-    const active_sync_rules = await this.getActiveSyncRules();
+    const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
     if (active_sync_rules == null) {
       return {
         operations_size_bytes: 0,
@@ -379,7 +379,7 @@ export class MongoBucketStorage implements BucketStorageFactory {
   private makeActiveCheckpoint(doc: SyncRuleDocument | null) {
     return {
       checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
-      lsn: doc?.last_checkpoint_lsn ?? replication.ZERO_LSN,
+      lsn: doc?.last_checkpoint_lsn ?? null,
       hasSyncRules() {
         return doc != null;
       },
@@ -389,7 +389,7 @@ export class MongoBucketStorage implements BucketStorageFactory {
         }
         return (await this.storageCache.fetch(doc._id)) ?? null;
       }
-    };
+    } satisfies ActiveCheckpoint;
   }
 
   /**
@@ -479,6 +479,7 @@ export class MongoBucketStorage implements BucketStorageFactory {
       if (doc == null) {
         continue;
       }
+
       const op = this.makeActiveCheckpoint(doc);
       // Check for LSN / checkpoint changes - ignore other metadata changes
       if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
@@ -508,8 +509,19 @@ export class MongoBucketStorage implements BucketStorageFactory {
       // What is important is:
       // 1. checkpoint (op_id) changes.
       // 2. write checkpoint changes for the specific user
+      const bucketStorage = await cp.getBucketStorage();
+      if (!bucketStorage) {
+        continue;
+      }
+
+      const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
 
-      const currentWriteCheckpoint = await this.lastWriteCheckpoint(user_id, lsn ?? '');
+      const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
+        user_id,
+        heads: {
+          ...lsnFilters
+        }
+      });
 
       if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
         // No change - wait for next one

package/src/storage/ReplicationEventPayload.ts

@@ -0,0 +1,16 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import { BucketStorageBatch, SaveOp } from './BucketStorage.js';
+import { SourceTable } from './SourceTable.js';
+
+export type EventData = {
+  op: SaveOp;
+  before?: sync_rules.SqliteRow;
+  after?: sync_rules.SqliteRow;
+};
+
+export type ReplicationEventPayload = {
+  batch: BucketStorageBatch;
+  data: EventData;
+  event: sync_rules.SqlEventDescriptor;
+  table: SourceTable;
+};

package/src/storage/SourceEntity.ts

@@ -0,0 +1,22 @@
+export interface ColumnDescriptor {
+  name: string;
+  /**
+   *  The type of the column ie VARCHAR, INT, etc
+   */
+  type?: string;
+  /**
+   *  Some data sources have a type id that can be used to identify the type of the column
+   */
+  typeId?: number;
+}
+
+// TODO: This needs to be consolidated with SourceTable into something new.
+export interface SourceEntityDescriptor {
+  /**
+   *  The internal id of the data source structure in the database
+   */
+  objectId: number | string;
+  schema: string;
+  name: string;
+  replicationColumns: ColumnDescriptor[];
+}

package/src/storage/SourceTable.ts

@@ -1,10 +1,8 @@
-import { DEFAULT_SCHEMA, DEFAULT_TAG } from '@powersync/service-sync-rules';
-
-import * as replication from '../replication/replication-index.js';
+import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
+import { ColumnDescriptor } from './SourceEntity.js';
 
 export class SourceTable {
-  static readonly DEFAULT_SCHEMA = DEFAULT_SCHEMA;
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
@@ -25,14 +23,23 @@ export class SourceTable {
    */
   public syncParameters = true;
 
+  /**
+   * True if the table is used in sync rules for events.
+   *
+   * This value is resolved externally, and cached here.
+   *
+   * Defaults to true for tests.
+   */
+  public syncEvent = true;
+
   constructor(
     public readonly id: any,
     public readonly connectionTag: string,
-    public readonly relationId: number,
+    public readonly objectId: number | string,
     public readonly schema: string,
     public readonly table: string,
 
-    public readonly replicaIdColumns: replication.ReplicationColumn[],
+    public readonly replicaIdColumns: ColumnDescriptor[],
     public readonly snapshotComplete: boolean
   ) {}
 
@@ -55,6 +62,6 @@ export class SourceTable {
   }
 
   get syncAny() {
-    return this.syncData || this.syncParameters;
+    return this.syncData || this.syncParameters || this.syncEvent;
   }
 }

package/src/storage/StorageEngine.ts

@@ -0,0 +1,62 @@
+import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { ResolvedPowerSyncConfig } from '../util/util-index.js';
+import { BucketStorageFactory } from './BucketStorage.js';
+import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
+
+export type StorageEngineOptions = {
+  configuration: ResolvedPowerSyncConfig;
+};
+
+export interface StorageEngineListener extends DisposableListener {
+  storageActivated: (storage: BucketStorageFactory) => void;
+}
+
+export class StorageEngine extends DisposableObserver<StorageEngineListener> {
+  // TODO: This will need to revisited when we actually support multiple storage providers.
+  private storageProviders: Map<string, BucketStorageProvider> = new Map();
+  private currentActiveStorage: ActiveStorage | null = null;
+
+  constructor(private options: StorageEngineOptions) {
+    super();
+  }
+
+  get activeBucketStorage(): BucketStorageFactory {
+    return this.activeStorage.storage;
+  }
+
+  get activeStorage(): ActiveStorage {
+    if (!this.currentActiveStorage) {
+      throw new Error(`No storage provider has been initialized yet.`);
+    }
+
+    return this.currentActiveStorage;
+  }
+
+  /**
+   * Register a provider which generates a {@link BucketStorageFactory}
+   * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
+   */
+  registerProvider(provider: BucketStorageProvider) {
+    this.storageProviders.set(provider.type, provider);
+  }
+
+  public async start(): Promise<void> {
+    logger.info('Starting Storage Engine...');
+    const { configuration } = this.options;
+    this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
+      resolvedConfig: configuration
+    });
+    this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
+    logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
+    logger.info('Successfully started Storage Engine.');
+  }
+
+  /**
+   *  Shutdown the storage engine, safely shutting down any activated storage providers.
+   */
+  public async shutDown(): Promise<void> {
+    logger.info('Shutting down Storage Engine...');
+    await this.currentActiveStorage?.shutDown();
+    logger.info('Successfully shut down Storage Engine.');
+  }
+}

package/src/storage/StorageProvider.ts

@@ -0,0 +1,27 @@
+import * as util from '../util/util-index.js';
+import { BucketStorageFactory } from './BucketStorage.js';
+
+export interface ActiveStorage {
+  storage: BucketStorageFactory;
+  shutDown(): Promise<void>;
+
+  /**
+   *  Tear down / drop the storage permanently
+   */
+  tearDown(): Promise<boolean>;
+}
+
+export interface GetStorageOptions {
+  // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
+  resolvedConfig: util.ResolvedPowerSyncConfig;
+}
+
+export interface BucketStorageProvider {
+  /**
+   *  The storage type that this provider provides.
+   *  The type should match the `type` field in the config.
+   */
+  type: string;
+
+  getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
+}

package/src/storage/WriteCheckpointAPI.ts

@@ -0,0 +1,85 @@
+export enum WriteCheckpointMode {
+  /**
+   * Raw mappings of `user_id` to `write_checkpoint`s should
+   * be supplied for each set of sync rules.
+   */
+  CUSTOM = 'custom',
+  /**
+   * Write checkpoints are stored as a mapping of `user_id` plus
+   * replication HEAD (lsn in Postgres) to an automatically generated
+   * incrementing `write_checkpoint` (stored as`client_id`).
+   */
+  MANAGED = 'managed'
+}
+
+export interface BaseWriteCheckpointIdentifier {
+  /**
+   * Identifier for User's account.
+   */
+  user_id: string;
+}
+
+export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+  /**
+   * Sync rules which were active when this checkpoint was created.
+   */
+  sync_rules_id: number;
+}
+
+export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
+  /**
+   * A supplied incrementing Write Checkpoint number
+   */
+  checkpoint: bigint;
+}
+
+export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
+  /**
+   * Sync rules which were active when this checkpoint was created.
+   */
+  sync_rules_id: number;
+}
+
+/**
+ * Managed Write Checkpoints are a mapping of User ID to replication HEAD
+ */
+export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+  /**
+   * Replication HEAD(s) at the creation of the checkpoint.
+   */
+  heads: Record<string, string>;
+}
+
+export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
+
+export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
+export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
+
+export interface BaseWriteCheckpointAPI {
+  readonly writeCheckpointMode: WriteCheckpointMode;
+  setWriteCheckpointMode(mode: WriteCheckpointMode): void;
+  createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
+}
+
+/**
+ * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
+ * This storage corresponds with a set of sync rules. These APIs don't require specifying a
+ * sync rules id.
+ */
+export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
+  batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
+  createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
+  lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
+}
+
+/**
+ * Write Checkpoint API which is interfaced directly with the storage layer. This requires
+ * sync rules identifiers for custom write checkpoints.
+ */
+export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
+  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
+  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
+  lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
+}
+
+export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;