@powersync/service-core 1.12.0 → 1.13.0

package/src/api/RouteAPI.ts

@@ -47,12 +47,7 @@ export interface RouteAPI {
    * @returns The replication lag: that is the amount of data which has not been
    *          replicated yet, in bytes.
    */
-  getReplicationLag(options: ReplicationLagOptions): Promise<number | undefined>;
-
-  /**
-   * Get the current LSN or equivalent replication HEAD position identifier
-   */
-  getReplicationHead(): Promise<string>;
+  getReplicationLagBytes(options: ReplicationLagOptions): Promise<number | undefined>;
 
   /**
    * Get the current LSN or equivalent replication HEAD position identifier.

package/src/api/diagnostics.ts

@@ -78,7 +78,7 @@ export async function getSyncRulesStatus(
 
     if (systemStorage) {
       try {
-        replication_lag_bytes = await apiHandler.getReplicationLag({
+        replication_lag_bytes = await apiHandler.getReplicationLagBytes({
           bucketStorage: systemStorage
         });
       } catch (e) {

package/src/entry/cli-entry.ts

@@ -36,8 +36,8 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
       try {
         await entryProgram.parseAsync();
       } catch (e) {
-        logger.error('Fatal error', e);
-        process.exit(1);
+        logger.error('Fatal startup error - exiting with code 150.', e);
+        process.exit(150);
       }
     }
   };

package/src/index.ts

@@ -41,3 +41,5 @@ export * as utils from './util/util-index.js';
 
 export * from './streams/streams-index.js';
 export * as streams from './streams/streams-index.js';
+
+export * as bson from 'bson';

package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts

@@ -1,11 +1,11 @@
 import { Meter, ValueType } from '@opentelemetry/api';
 import {
   Counter,
-  ObservableGauge,
-  UpDownCounter,
   MetricMetadata,
   MetricsFactory,
-  Precision
+  ObservableGauge,
+  Precision,
+  UpDownCounter
 } from '../metrics-interfaces.js';
 
 export class OpenTelemetryMetricsFactory implements MetricsFactory {

package/src/replication/AbstractReplicationJob.ts

@@ -78,4 +78,9 @@ export abstract class AbstractReplicationJob {
   public get isStopped(): boolean {
     return this.abortController.signal.aborted;
   }
+
+  /**
+   * Get replication lag for this job in ms.
+   */
+  abstract getReplicationLagMillis(): Promise<number | undefined>;
 }

package/src/replication/AbstractReplicator.ts

@@ -8,6 +8,7 @@ import { AbstractReplicationJob } from './AbstractReplicationJob.js';
 import { ErrorRateLimiter } from './ErrorRateLimiter.js';
 import { ConnectionTestResult } from './ReplicationModule.js';
 import { MetricsEngine } from '../metrics/MetricsEngine.js';
+import { ReplicationMetric } from '@powersync/service-types';
 
 // 5 minutes
 const PING_INTERVAL = 1_000_000_000n * 300n;
@@ -42,11 +43,16 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
    *  @private
    */
   private replicationJobs = new Map<number, T>();
-  private stopped = false;
+  /**
+   * Used for replication lag computation.
+   */
+  private activeReplicationJob: T | undefined = undefined;
 
   // First ping is only after 5 minutes, not when starting
   private lastPing = hrtime.bigint();
 
+  private abortController: AbortController | undefined;
+
   protected constructor(private options: AbstractReplicatorOptions) {
     this.logger = logger.child({ name: `Replicator:${options.id}` });
   }
@@ -79,7 +85,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     return this.options.metricsEngine;
   }
 
+  protected get stopped() {
+    return this.abortController?.signal.aborted;
+  }
+
   public async start(): Promise<void> {
+    this.abortController = new AbortController();
     this.runLoop().catch((e) => {
       this.logger.error('Data source fatal replication error', e);
       container.reporter.captureException(e);
@@ -87,10 +98,21 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
         process.exit(1);
       }, 1000);
     });
+    this.metrics.getObservableGauge(ReplicationMetric.REPLICATION_LAG_SECONDS).setValueProvider(async () => {
+      const lag = await this.getReplicationLagMillis().catch((e) => {
+        this.logger.error('Failed to get replication lag', e);
+        return undefined;
+      });
+      if (lag == null) {
+        return undefined;
+      }
+      // ms to seconds
+      return Math.round(lag / 1000);
+    });
   }
 
   public async stop(): Promise<void> {
-    this.stopped = true;
+    this.abortController?.abort();
     let promises: Promise<void>[] = [];
     for (const job of this.replicationJobs.values()) {
       promises.push(job.stop());
@@ -161,8 +183,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     const existingJobs = new Map<number, T>(this.replicationJobs.entries());
     const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
     const newJobs = new Map<number, T>();
+    let activeJob: T | undefined = undefined;
     for (let syncRules of replicatingSyncRules) {
       const existingJob = existingJobs.get(syncRules.id);
+      if (syncRules.active && activeJob == null) {
+        activeJob = existingJob;
+      }
       if (existingJob && !existingJob.isStopped) {
         // No change
         existingJobs.delete(syncRules.id);
@@ -188,6 +214,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
 
           newJobs.set(syncRules.id, newJob);
           newJob.start();
+          if (syncRules.active) {
+            activeJob = newJob;
+          }
         } catch (e) {
           // Could be a sync rules parse error,
           // for example from stricter validation that was added.
@@ -199,6 +228,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     }
 
     this.replicationJobs = newJobs;
+    this.activeReplicationJob = activeJob;
 
     // Stop any orphaned jobs that no longer have sync rules.
     // Termination happens below
@@ -216,6 +246,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     const stopped = await this.storage.getStoppedSyncRules();
     for (let syncRules of stopped) {
       try {
+        // TODO: Do this in the "background", allowing the periodic refresh to continue
         const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
         await this.terminateSyncRules(syncRuleStorage);
       } catch (e) {
@@ -231,9 +262,31 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
   protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
     this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
     await this.cleanUp(syncRuleStorage);
-    await syncRuleStorage.terminate();
+    await syncRuleStorage.terminate({ signal: this.abortController?.signal, clearStorage: true });
     this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
   }
 
   abstract testConnection(): Promise<ConnectionTestResult>;
+
+  /**
+   * Measure replication lag in milliseconds.
+   *
+   * In general, this is the difference between now() and the time the oldest record, that we haven't committed yet,
+   * has been written (committed) to the source database.
+   *
+   * This is roughly a measure of the _average_ amount of time we're behind.
+   * If we get a new change as soon as each previous one has finished processing, and each change takes 1000ms
+   * to process, the average replication lag will be 500ms, not 1000ms.
+   *
+   * 1. When we are actively replicating, this is the difference between now and when the time the change was
+   *    written to the source database.
+   * 2. When the replication stream is idle, this is either 0, or the delay for keepalive messages to make it to us.
+   * 3. When the active replication stream is an error state, this is the time since the last successful commit.
+   * 4. If there is no active replication stream, this is undefined.
+   *
+   * "processing" replication streams are not taken into account for this metric.
+   */
+  async getReplicationLagMillis(): Promise<number | undefined> {
+    return this.activeReplicationJob?.getReplicationLagMillis();
+  }
 }

package/src/replication/RelationCache.ts

@@ -0,0 +1,25 @@
+import { SourceTable } from '../storage/SourceTable.js';
+
+export class RelationCache<T> {
+  private cache = new Map<string | number, SourceTable>();
+  private idFunction: (item: T | SourceTable) => string | number;
+
+  constructor(idFunction: (item: T | SourceTable) => string | number) {
+    this.idFunction = idFunction;
+  }
+
+  update(table: SourceTable) {
+    const id = this.idFunction(table);
+    this.cache.set(id, table);
+  }
+
+  get(source: T | SourceTable): SourceTable | undefined {
+    const id = this.idFunction(source);
+    return this.cache.get(id);
+  }
+
+  delete(source: T | SourceTable): boolean {
+    const id = this.idFunction(source);
+    return this.cache.delete(id);
+  }
+}

package/src/replication/replication-index.ts

@@ -4,3 +4,4 @@ export * from './ErrorRateLimiter.js';
 export * from './ReplicationEngine.js';
 export * from './ReplicationModule.js';
 export * from './replication-metrics.js';
+export * from './RelationCache.js';

package/src/replication/replication-metrics.ts

@@ -26,6 +26,11 @@ export function createCoreReplicationMetrics(engine: MetricsEngine): void {
     name: ReplicationMetric.CHUNKS_REPLICATED,
     description: 'Total number of replication chunks'
   });
+
+  engine.createObservableGauge({
+    name: ReplicationMetric.REPLICATION_LAG_SECONDS,
+    description: 'Replication lag between the source database and PowerSync instance'
+  });
 }
 
 /**
@@ -42,4 +47,6 @@ export function initializeCoreReplicationMetrics(engine: MetricsEngine): void {
   rows_replicated_total.add(0);
   transactions_replicated_total.add(0);
   chunks_replicated_total.add(0);
+  // REPLICATION_LAG_SECONDS is not explicitly initialized - the value remains "unknown" until the first value
+  // is reported.
 }

package/src/routes/endpoints/admin.ts

@@ -168,6 +168,8 @@ export const validate = routeDefinition({
       // Dummy values
       id: 0,
       slot_name: '',
+      active: false,
+      last_checkpoint_lsn: '',
 
       parsed() {
         return {

package/src/routes/endpoints/sync-rules.ts

@@ -202,13 +202,13 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
 
     return {
       valid: true,
-      bucket_definitions: rules.bucket_descriptors.map((d) => {
-        let all_parameter_queries = [...d.parameter_queries.values()].flat();
-        let all_data_queries = [...d.data_queries.values()].flat();
+      bucket_definitions: rules.bucketDescriptors.map((d) => {
+        let all_parameter_queries = [...d.parameterQueries.values()].flat();
+        let all_data_queries = [...d.dataQueries.values()].flat();
         return {
           name: d.name,
-          bucket_parameters: d.bucket_parameters,
-          global_parameter_queries: d.global_parameter_queries.map((q) => {
+          bucket_parameters: d.bucketParameters,
+          global_parameter_queries: d.globalParameterQueries.map((q) => {
             return {
               sql: q.sql
             };
@@ -217,7 +217,7 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
             return {
               sql: q.sql,
               table: q.sourceTable,
-              input_parameters: q.input_parameters
+              input_parameters: q.inputParameters
             };
           }),
 

package/src/storage/BucketStorageBatch.ts

@@ -2,12 +2,13 @@ import { ObserverClient } from '@powersync/lib-services-framework';
 import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
 import { BSON } from 'bson';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
-import { SourceTable } from './SourceTable.js';
+import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
 import { InternalOpId } from '../util/utils.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
-  createEmptyCheckpoints: true
+  createEmptyCheckpoints: true,
+  oldestUncommittedChange: null
 };
 
 export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
@@ -38,12 +39,14 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    *
    * @returns null if there are no changes to flush.
    */
-  flush(): Promise<FlushedResult | null>;
+  flush(options?: BatchBucketFlushOptions): Promise<FlushedResult | null>;
 
   /**
    * Flush and commit any saved ops. This creates a new checkpoint by default.
    *
    * Only call this after a transaction.
+   *
+   * Returns true if either (1) a new checkpoint was created, or (2) there are no changes to commit.
    */
   commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
 
@@ -56,6 +59,14 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   keepalive(lsn: string): Promise<boolean>;
 
+  /**
+   * Set the LSN for a snapshot, before starting replication.
+   *
+   * Not required if the source database keeps track of this, for example with
+   * PostgreSQL logical replication slots.
+   */
+  setSnapshotLsn(lsn: string): Promise<void>;
+
   /**
    * Get the last checkpoint LSN, from either commit or keepalive.
    */
@@ -63,6 +74,8 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
 
   markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
 
+  updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
+
   /**
    * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
    */
@@ -148,7 +161,16 @@ export interface FlushedResult {
   flushed_op: InternalOpId;
 }
 
-export interface BucketBatchCommitOptions {
+export interface BatchBucketFlushOptions {
+  /**
+   * The timestamp of the first change in this batch, according to the source database.
+   *
+   * Used to estimate replication lag.
+   */
+  oldestUncommittedChange?: Date | null;
+}
+
+export interface BucketBatchCommitOptions extends BatchBucketFlushOptions {
   /**
    * Creates a new checkpoint even if there were no persisted operations.
    * Defaults to true.

package/src/storage/PersistedSyncRulesContent.ts

@@ -9,6 +9,12 @@ export interface PersistedSyncRulesContent {
   readonly id: number;
   readonly sync_rules_content: string;
   readonly slot_name: string;
+  /**
+   * True if this is the "active" copy of the sync rules.
+   */
+  readonly active: boolean;
+
+  readonly last_checkpoint_lsn: string | null;
 
   readonly last_fatal_error?: string | null;
   readonly last_keepalive_ts?: Date | null;

package/src/storage/SourceTable.ts

@@ -2,6 +2,12 @@ import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor } from './SourceEntity.js';
 
+export interface TableSnapshotStatus {
+  totalEstimatedCount: number;
+  replicatedCount: number;
+  lastKey: Uint8Array | null;
+}
+
 export class SourceTable {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
@@ -32,6 +38,13 @@ export class SourceTable {
    */
   public syncEvent = true;
 
+  /**
+   * Always undefined if snapshotComplete = true.
+   *
+   * May be set if snapshotComplete = false.
+   */
+  public snapshotStatus: TableSnapshotStatus | undefined = undefined;
+
   constructor(
     public readonly id: any,
     public readonly connectionTag: string,
@@ -40,7 +53,7 @@ export class SourceTable {
     public readonly table: string,
 
     public readonly replicaIdColumns: ColumnDescriptor[],
-    public readonly snapshotComplete: boolean
+    public snapshotComplete: boolean
   ) {}
 
   get hasReplicaIdentity() {
@@ -68,4 +81,34 @@ export class SourceTable {
   get syncAny() {
     return this.syncData || this.syncParameters || this.syncEvent;
   }
+
+  /**
+   * In-memory clone of the table status.
+   */
+  clone() {
+    const copy = new SourceTable(
+      this.id,
+      this.connectionTag,
+      this.objectId,
+      this.schema,
+      this.table,
+      this.replicaIdColumns,
+      this.snapshotComplete
+    );
+    copy.syncData = this.syncData;
+    copy.syncParameters = this.syncParameters;
+    copy.snapshotStatus = this.snapshotStatus;
+    return copy;
+  }
+
+  formatSnapshotProgress() {
+    if (this.snapshotComplete || this.snapshotStatus == null) {
+      // Should not happen
+      return '-';
+    } else if (this.snapshotStatus.totalEstimatedCount < 0) {
+      return `${this.snapshotStatus.replicatedCount}/?`;
+    } else {
+      return `${this.snapshotStatus.replicatedCount}/~${this.snapshotStatus.totalEstimatedCount}`;
+    }
+  }
 }

package/src/storage/StorageEngine.ts

@@ -1,7 +1,7 @@
-import { BaseObserver, logger } from '@powersync/lib-services-framework';
+import { BaseObserver, logger, ServiceError } from '@powersync/lib-services-framework';
 import { ResolvedPowerSyncConfig } from '../util/util-index.js';
-import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
+import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
 
 export type StorageEngineOptions = {
   configuration: ResolvedPowerSyncConfig;
@@ -9,6 +9,7 @@ export type StorageEngineOptions = {
 
 export interface StorageEngineListener {
   storageActivated: (storage: BucketStorageFactory) => void;
+  storageFatalError: (error: ServiceError) => void;
 }
 
 export class StorageEngine extends BaseObserver<StorageEngineListener> {
@@ -47,6 +48,9 @@ export class StorageEngine extends BaseObserver<StorageEngineListener> {
       resolvedConfig: configuration
     });
     this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
+    this.currentActiveStorage.onFatalError?.((error) => {
+      this.iterateListeners((cb) => cb.storageFatalError?.(error));
+    });
     logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
     logger.info('Successfully started Storage Engine.');
   }

package/src/storage/StorageProvider.ts

@@ -1,3 +1,4 @@
+import { ServiceError } from '@powersync/lib-services-framework';
 import * as util from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
 
@@ -9,6 +10,8 @@ export interface ActiveStorage {
    *  Tear down / drop the storage permanently
    */
   tearDown(): Promise<boolean>;
+
+  onFatalError?(callback: (error: ServiceError) => void): void;
 }
 
 export interface GetStorageOptions {

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,7 +1,7 @@
-import { ObserverClient } from '@powersync/lib-services-framework';
+import { Logger, ObserverClient } from '@powersync/lib-services-framework';
 import { ParameterLookup, SqlSyncRules, SqliteJsonRow } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
-import { BucketStorageBatch, FlushedResult } from './BucketStorageBatch.js';
+import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
 import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
@@ -48,7 +48,7 @@ export interface SyncRulesBucketStorage
   /**
    * Clear the storage, without changing state.
    */
-  clear(): Promise<void>;
+  clear(options?: ClearStorageOptions): Promise<void>;
 
   autoActivate(): Promise<void>;
 
@@ -125,6 +125,7 @@ export interface SyncRuleStatus {
   checkpoint_lsn: string | null;
   active: boolean;
   snapshot_done: boolean;
+  snapshot_lsn: string | null;
 }
 export interface ResolveTableOptions {
   group_id: number;
@@ -159,6 +160,15 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
    * This will avoid creating new operations for rows previously replicated.
    */
   skipExistingRows?: boolean;
+
+  /**
+   * Callback called if we streamed an update to a record that we don't have yet.
+   *
+   * This is expected to happen in some initial replication edge cases, only if storeCurrentData = true.
+   */
+  markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
+
+  logger?: Logger;
 }
 
 export interface CompactOptions {
@@ -200,7 +210,11 @@ export interface CompactOptions {
   moveBatchQueryLimit?: number;
 }
 
-export interface TerminateOptions {
+export interface ClearStorageOptions {
+  signal?: AbortSignal;
+}
+
+export interface TerminateOptions extends ClearStorageOptions {
   /**
    * If true, also clear the storage before terminating.
    */
@@ -256,8 +270,8 @@ export interface StorageCheckpointUpdate extends WriteCheckpoint {
 }
 
 export interface GetCheckpointChangesOptions {
-  lastCheckpoint: util.InternalOpId;
-  nextCheckpoint: util.InternalOpId;
+  lastCheckpoint: ReplicationCheckpoint;
+  nextCheckpoint: ReplicationCheckpoint;
 }
 
 export interface CheckpointChanges {
@@ -274,3 +288,5 @@ export const CHECKPOINT_INVALIDATE_ALL: CheckpointChanges = {
   updatedParameterLookups: new Set<string>(),
   invalidateParameterBuckets: true
 };
+
+export type BucketStorageMarkRecordUnavailable = (record: SaveUpdate) => void;

package/src/storage/WriteCheckpointAPI.ts

@@ -50,37 +50,11 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
   heads: Record<string, string>;
 }
 
-export interface WriteCheckpointResult {
-  /**
-   * Write checkpoint id (also referred to as client_id).
-   *
-   * If null, there is no write checkpoint for the client.
-   */
-  id: bigint | null;
-
-  /**
-   * LSN for the checkpoint.
-   *
-   * This will change when we support multiple connections.
-   *
-   * For managed write checkpoints, this LSN must be exceeded by the checkpoint / replication head to be valid.
-   *
-   * For custom write checkpoints, this will be null, and the write checkpoint is valid for all LSNs.
-   */
-  lsn: string | null;
-}
-
 export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
 
 export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
 
-export interface WatchUserWriteCheckpointOptions {
-  user_id: string;
-  sync_rules_id: number;
-  signal: AbortSignal;
-}
-
 export interface BaseWriteCheckpointAPI {
   readonly writeCheckpointMode: WriteCheckpointMode;
   setWriteCheckpointMode(mode: WriteCheckpointMode): void;
@@ -93,7 +67,6 @@ export interface BaseWriteCheckpointAPI {
  * sync rules id.
  */
 export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
-  batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
   lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
 }
 
@@ -102,10 +75,7 @@ export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
  * sync rules identifiers for custom write checkpoints.
  */
 export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
-  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
   lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
-
-  watchUserWriteCheckpoint(options: WatchUserWriteCheckpointOptions): AsyncIterable<WriteCheckpointResult>;
 }
 
 export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;

package/src/storage/bson.ts

@@ -10,7 +10,10 @@ type NodeBuffer = Buffer<ArrayBuffer>;
  */
 export const BSON_DESERIALIZE_INTERNAL_OPTIONS: bson.DeserializeOptions = {
   // use bigint instead of Long
-  useBigInt64: true
+  useBigInt64: true,
+  // We cannot use promoteBuffers: true, since that also converst UUID to Buffer
+  // Instead, we need to handle bson.Binary when reading data
+  promoteBuffers: false
 };
 
 /**

package/src/streams/BroadcastIterable.ts

@@ -97,7 +97,7 @@ export class BroadcastIterable<T> implements AsyncIterable<T> {
     }
   }
 
-  async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterator<T> {
+  async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterableIterator<T> {
     const sink = new LastValueSink(this.last);
     this.addSink(sink);
     try {

package/src/streams/streams-index.ts

@@ -1,4 +1,3 @@
 export * from './merge.js';
-export * from './Demultiplexer.js';
 export * from './LastValueSink.js';
 export * from './BroadcastIterable.js';

package/src/sync/BucketChecksumState.ts

@@ -92,7 +92,7 @@ export class BucketChecksumState {
    */
   async buildNextCheckpointLine(next: storage.StorageCheckpointUpdate): Promise<CheckpointLine | null> {
     const { writeCheckpoint, base } = next;
-    const user_id = this.parameterState.syncParams.user_id;
+    const user_id = this.parameterState.syncParams.userId;
 
     const storage = this.bucketStorage;
 
@@ -378,7 +378,7 @@ export class BucketParameterState {
       );
       this.logger.error(error.message, {
         checkpoint: checkpoint,
-        user_id: this.syncParams.user_id,
+        user_id: this.syncParams.userId,
         buckets: update.buckets.length
       });