@powersync/service-core 1.12.1 → 1.13.0

package/src/storage/BucketStorageBatch.ts

@@ -2,12 +2,13 @@ import { ObserverClient } from '@powersync/lib-services-framework';
 import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
 import { BSON } from 'bson';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
-import { SourceTable } from './SourceTable.js';
+import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
 import { InternalOpId } from '../util/utils.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
-  createEmptyCheckpoints: true
+  createEmptyCheckpoints: true,
+  oldestUncommittedChange: null
 };
 
 export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
@@ -38,12 +39,14 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    *
    * @returns null if there are no changes to flush.
    */
-  flush(): Promise<FlushedResult | null>;
+  flush(options?: BatchBucketFlushOptions): Promise<FlushedResult | null>;
 
   /**
    * Flush and commit any saved ops. This creates a new checkpoint by default.
    *
    * Only call this after a transaction.
+   *
+   * Returns true if either (1) a new checkpoint was created, or (2) there are no changes to commit.
    */
   commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
 
@@ -56,6 +59,14 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   keepalive(lsn: string): Promise<boolean>;
 
+  /**
+   * Set the LSN for a snapshot, before starting replication.
+   *
+   * Not required if the source database keeps track of this, for example with
+   * PostgreSQL logical replication slots.
+   */
+  setSnapshotLsn(lsn: string): Promise<void>;
+
   /**
    * Get the last checkpoint LSN, from either commit or keepalive.
    */
@@ -63,6 +74,8 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
 
   markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
 
+  updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
+
   /**
    * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
    */
@@ -148,7 +161,16 @@ export interface FlushedResult {
   flushed_op: InternalOpId;
 }
 
-export interface BucketBatchCommitOptions {
+export interface BatchBucketFlushOptions {
+  /**
+   * The timestamp of the first change in this batch, according to the source database.
+   *
+   * Used to estimate replication lag.
+   */
+  oldestUncommittedChange?: Date | null;
+}
+
+export interface BucketBatchCommitOptions extends BatchBucketFlushOptions {
   /**
    * Creates a new checkpoint even if there were no persisted operations.
    * Defaults to true.

package/src/storage/PersistedSyncRulesContent.ts

@@ -9,6 +9,12 @@ export interface PersistedSyncRulesContent {
   readonly id: number;
   readonly sync_rules_content: string;
   readonly slot_name: string;
+  /**
+   * True if this is the "active" copy of the sync rules.
+   */
+  readonly active: boolean;
+
+  readonly last_checkpoint_lsn: string | null;
 
   readonly last_fatal_error?: string | null;
   readonly last_keepalive_ts?: Date | null;

package/src/storage/SourceTable.ts

@@ -2,6 +2,12 @@ import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor } from './SourceEntity.js';
 
+export interface TableSnapshotStatus {
+  totalEstimatedCount: number;
+  replicatedCount: number;
+  lastKey: Uint8Array | null;
+}
+
 export class SourceTable {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
@@ -32,6 +38,13 @@ export class SourceTable {
    */
   public syncEvent = true;
 
+  /**
+   * Always undefined if snapshotComplete = true.
+   *
+   * May be set if snapshotComplete = false.
+   */
+  public snapshotStatus: TableSnapshotStatus | undefined = undefined;
+
   constructor(
     public readonly id: any,
     public readonly connectionTag: string,
@@ -40,7 +53,7 @@ export class SourceTable {
     public readonly table: string,
 
     public readonly replicaIdColumns: ColumnDescriptor[],
-    public readonly snapshotComplete: boolean
+    public snapshotComplete: boolean
   ) {}
 
   get hasReplicaIdentity() {
@@ -68,4 +81,34 @@ export class SourceTable {
   get syncAny() {
     return this.syncData || this.syncParameters || this.syncEvent;
   }
+
+  /**
+   * In-memory clone of the table status.
+   */
+  clone() {
+    const copy = new SourceTable(
+      this.id,
+      this.connectionTag,
+      this.objectId,
+      this.schema,
+      this.table,
+      this.replicaIdColumns,
+      this.snapshotComplete
+    );
+    copy.syncData = this.syncData;
+    copy.syncParameters = this.syncParameters;
+    copy.snapshotStatus = this.snapshotStatus;
+    return copy;
+  }
+
+  formatSnapshotProgress() {
+    if (this.snapshotComplete || this.snapshotStatus == null) {
+      // Should not happen
+      return '-';
+    } else if (this.snapshotStatus.totalEstimatedCount < 0) {
+      return `${this.snapshotStatus.replicatedCount}/?`;
+    } else {
+      return `${this.snapshotStatus.replicatedCount}/~${this.snapshotStatus.totalEstimatedCount}`;
+    }
+  }
 }

package/src/storage/StorageEngine.ts

@@ -1,7 +1,7 @@
-import { BaseObserver, logger } from '@powersync/lib-services-framework';
+import { BaseObserver, logger, ServiceError } from '@powersync/lib-services-framework';
 import { ResolvedPowerSyncConfig } from '../util/util-index.js';
-import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
+import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
 
 export type StorageEngineOptions = {
   configuration: ResolvedPowerSyncConfig;
@@ -9,6 +9,7 @@ export type StorageEngineOptions = {
 
 export interface StorageEngineListener {
   storageActivated: (storage: BucketStorageFactory) => void;
+  storageFatalError: (error: ServiceError) => void;
 }
 
 export class StorageEngine extends BaseObserver<StorageEngineListener> {
@@ -47,6 +48,9 @@ export class StorageEngine extends BaseObserver<StorageEngineListener> {
       resolvedConfig: configuration
     });
     this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
+    this.currentActiveStorage.onFatalError?.((error) => {
+      this.iterateListeners((cb) => cb.storageFatalError?.(error));
+    });
     logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
     logger.info('Successfully started Storage Engine.');
   }

package/src/storage/StorageProvider.ts

@@ -1,3 +1,4 @@
+import { ServiceError } from '@powersync/lib-services-framework';
 import * as util from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
 
@@ -9,6 +10,8 @@ export interface ActiveStorage {
    *  Tear down / drop the storage permanently
    */
   tearDown(): Promise<boolean>;
+
+  onFatalError?(callback: (error: ServiceError) => void): void;
 }
 
 export interface GetStorageOptions {

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,7 +1,7 @@
-import { ObserverClient } from '@powersync/lib-services-framework';
+import { Logger, ObserverClient } from '@powersync/lib-services-framework';
 import { ParameterLookup, SqlSyncRules, SqliteJsonRow } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
-import { BucketStorageBatch, FlushedResult } from './BucketStorageBatch.js';
+import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
 import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
@@ -48,7 +48,7 @@ export interface SyncRulesBucketStorage
   /**
    * Clear the storage, without changing state.
    */
-  clear(): Promise<void>;
+  clear(options?: ClearStorageOptions): Promise<void>;
 
   autoActivate(): Promise<void>;
 
@@ -125,6 +125,7 @@ export interface SyncRuleStatus {
   checkpoint_lsn: string | null;
   active: boolean;
   snapshot_done: boolean;
+  snapshot_lsn: string | null;
 }
 export interface ResolveTableOptions {
   group_id: number;
@@ -159,6 +160,15 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
    * This will avoid creating new operations for rows previously replicated.
    */
   skipExistingRows?: boolean;
+
+  /**
+   * Callback called if we streamed an update to a record that we don't have yet.
+   *
+   * This is expected to happen in some initial replication edge cases, only if storeCurrentData = true.
+   */
+  markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
+
+  logger?: Logger;
 }
 
 export interface CompactOptions {
@@ -200,7 +210,11 @@ export interface CompactOptions {
   moveBatchQueryLimit?: number;
 }
 
-export interface TerminateOptions {
+export interface ClearStorageOptions {
+  signal?: AbortSignal;
+}
+
+export interface TerminateOptions extends ClearStorageOptions {
   /**
    * If true, also clear the storage before terminating.
    */
@@ -256,8 +270,8 @@ export interface StorageCheckpointUpdate extends WriteCheckpoint {
 }
 
 export interface GetCheckpointChangesOptions {
-  lastCheckpoint: util.InternalOpId;
-  nextCheckpoint: util.InternalOpId;
+  lastCheckpoint: ReplicationCheckpoint;
+  nextCheckpoint: ReplicationCheckpoint;
 }
 
 export interface CheckpointChanges {
@@ -274,3 +288,5 @@ export const CHECKPOINT_INVALIDATE_ALL: CheckpointChanges = {
   updatedParameterLookups: new Set<string>(),
   invalidateParameterBuckets: true
 };
+
+export type BucketStorageMarkRecordUnavailable = (record: SaveUpdate) => void;

package/src/storage/WriteCheckpointAPI.ts

@@ -50,37 +50,11 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
   heads: Record<string, string>;
 }
 
-export interface WriteCheckpointResult {
-  /**
-   * Write checkpoint id (also referred to as client_id).
-   *
-   * If null, there is no write checkpoint for the client.
-   */
-  id: bigint | null;
-
-  /**
-   * LSN for the checkpoint.
-   *
-   * This will change when we support multiple connections.
-   *
-   * For managed write checkpoints, this LSN must be exceeded by the checkpoint / replication head to be valid.
-   *
-   * For custom write checkpoints, this will be null, and the write checkpoint is valid for all LSNs.
-   */
-  lsn: string | null;
-}
-
 export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
 
 export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
 
-export interface WatchUserWriteCheckpointOptions {
-  user_id: string;
-  sync_rules_id: number;
-  signal: AbortSignal;
-}
-
 export interface BaseWriteCheckpointAPI {
   readonly writeCheckpointMode: WriteCheckpointMode;
   setWriteCheckpointMode(mode: WriteCheckpointMode): void;
@@ -93,7 +67,6 @@ export interface BaseWriteCheckpointAPI {
  * sync rules id.
  */
 export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
-  batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
   lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
 }
 
@@ -102,10 +75,7 @@ export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
  * sync rules identifiers for custom write checkpoints.
  */
 export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
-  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
   lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
-
-  watchUserWriteCheckpoint(options: WatchUserWriteCheckpointOptions): AsyncIterable<WriteCheckpointResult>;
 }
 
 export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;

package/src/storage/bson.ts

@@ -10,7 +10,10 @@ type NodeBuffer = Buffer<ArrayBuffer>;
  */
 export const BSON_DESERIALIZE_INTERNAL_OPTIONS: bson.DeserializeOptions = {
   // use bigint instead of Long
-  useBigInt64: true
+  useBigInt64: true,
+  // We cannot use promoteBuffers: true, since that also converst UUID to Buffer
+  // Instead, we need to handle bson.Binary when reading data
+  promoteBuffers: false
 };
 
 /**

package/src/streams/BroadcastIterable.ts

@@ -97,7 +97,7 @@ export class BroadcastIterable<T> implements AsyncIterable<T> {
     }
   }
 
-  async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterator<T> {
+  async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterableIterator<T> {
     const sink = new LastValueSink(this.last);
     this.addSink(sink);
     try {

package/src/streams/streams-index.ts

@@ -1,4 +1,3 @@
 export * from './merge.js';
-export * from './Demultiplexer.js';
 export * from './LastValueSink.js';
 export * from './BroadcastIterable.js';

package/src/system/ServiceContext.ts

@@ -59,6 +59,12 @@ export class ServiceContextContainer implements ServiceContext {
     this.storageEngine = new storage.StorageEngine({
       configuration
     });
+    this.storageEngine.registerListener({
+      storageFatalError: (error) => {
+        // Propagate the error to the lifecycle engine
+        this.lifeCycleEngine.stopWithError(error);
+      }
+    });
 
     this.lifeCycleEngine.withLifecycle(this.storageEngine, {
       start: (storageEngine) => storageEngine.start(),