@powersync/common 1.41.0 → 1.42.0

package/src/client/sync/stream/WebsocketClientTransport.ts

@@ -0,0 +1,80 @@
+/**
+ * Adapted from rsocket-websocket-client
+ * https://github.com/rsocket/rsocket-js/blob/e224cf379e747c4f1ddc4f2fa111854626cc8575/packages/rsocket-websocket-client/src/WebsocketClientTransport.ts#L17
+ * This adds additional error handling for React Native iOS.
+ * This particularly adds a close listener to handle cases where the WebSocket
+ * connection closes immediately after opening without emitting an error.
+ */
+import {
+  ClientTransport,
+  Closeable,
+  Demultiplexer,
+  Deserializer,
+  DuplexConnection,
+  FrameHandler,
+  Multiplexer,
+  Outbound
+} from 'rsocket-core';
+import { ClientOptions } from 'rsocket-websocket-client';
+import { WebsocketDuplexConnection } from 'rsocket-websocket-client/dist/WebsocketDuplexConnection.js';
+
+export class WebsocketClientTransport implements ClientTransport {
+  private readonly url: string;
+  private readonly factory: (url: string) => WebSocket;
+
+  constructor(options: ClientOptions) {
+    this.url = options.url;
+    this.factory = options.wsCreator ?? ((url: string) => new WebSocket(url));
+  }
+
+  connect(
+    multiplexerDemultiplexerFactory: (outbound: Outbound & Closeable) => Multiplexer & Demultiplexer & FrameHandler
+  ): Promise<DuplexConnection> {
+    return new Promise((resolve, reject) => {
+      const websocket = this.factory(this.url);
+
+      websocket.binaryType = 'arraybuffer';
+
+      let removeListeners: () => void;
+
+      const openListener = () => {
+        removeListeners();
+        resolve(new WebsocketDuplexConnection(websocket, new Deserializer(), multiplexerDemultiplexerFactory));
+      };
+
+      const errorListener = (ev: ErrorEvent) => {
+        removeListeners();
+        // We add a default error in that case.
+        if (ev.error != null) {
+          // undici typically provides an error object
+          reject(ev.error);
+        } else if (ev.message != null) {
+          // React Native typically does not provide an error object, but does provide a message
+          reject(new Error(`Failed to create websocket connection: ${ev.message}`));
+        } else {
+          // Browsers often provide no details at all
+          reject(new Error(`Failed to create websocket connection to ${this.url}`));
+        }
+      };
+
+      /**
+       * In some cases, such as React Native iOS, the WebSocket connection may close immediately after opening
+       * without and error. In such cases, we need to handle the close event to reject the promise.
+       */
+      const closeListener = () => {
+        removeListeners();
+        reject(new Error('WebSocket connection closed while opening'));
+      };
+
+      removeListeners = () => {
+        websocket.removeEventListener('open', openListener);
+        websocket.removeEventListener('error', errorListener);
+        websocket.removeEventListener('close', closeListener);
+      };
+
+      websocket.addEventListener('open', openListener);
+      websocket.addEventListener('error', errorListener);
+      websocket.addEventListener('close', closeListener);
+    });
+  }
+}

package/src/client/sync/stream/core-instruction.ts

@@ -0,0 +1,99 @@
+import { StreamingSyncRequest } from './streaming-sync-types.js';
+import * as sync_status from '../../../db/crud/SyncStatus.js';
+import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
+
+/**
+ * An internal instruction emitted by the sync client in the core extension in response to the JS
+ * SDK passing sync data into the extension.
+ */
+export type Instruction =
+  | { LogLine: LogLine }
+  | { UpdateSyncStatus: UpdateSyncStatus }
+  | { EstablishSyncStream: EstablishSyncStream }
+  | { FetchCredentials: FetchCredentials }
+  | { CloseSyncStream: { hide_disconnect: boolean } }
+  | { FlushFileSystem: any }
+  | { DidCompleteSync: any };
+
+export interface LogLine {
+  severity: 'DEBUG' | 'INFO' | 'WARNING';
+  line: string;
+}
+
+export interface EstablishSyncStream {
+  request: StreamingSyncRequest;
+}
+
+export interface UpdateSyncStatus {
+  status: CoreSyncStatus;
+}
+
+export interface CoreSyncStatus {
+  connected: boolean;
+  connecting: boolean;
+  priority_status: SyncPriorityStatus[];
+  downloading: DownloadProgress | null;
+  streams: CoreStreamSubscription[];
+}
+
+/// An `ActiveStreamSubscription` from the core extension + serialized progress information.
+export interface CoreStreamSubscription {
+  progress: { total: number; downloaded: number };
+  name: string;
+  parameters: any;
+  priority: number | null;
+  active: boolean;
+  is_default: boolean;
+  has_explicit_subscription: boolean;
+  expires_at: number | null;
+  last_synced_at: number | null;
+}
+
+export interface SyncPriorityStatus {
+  priority: number;
+  last_synced_at: number | number;
+  has_synced: boolean | null;
+}
+
+export interface DownloadProgress {
+  buckets: Record<string, BucketProgress>;
+}
+
+export interface BucketProgress {
+  priority: number;
+  at_last: number;
+  since_last: number;
+  target_count: number;
+}
+
+export interface FetchCredentials {
+  did_expire: boolean;
+}
+
+function priorityToJs(status: SyncPriorityStatus): sync_status.SyncPriorityStatus {
+  return {
+    priority: status.priority,
+    hasSynced: status.has_synced ?? undefined,
+    lastSyncedAt: status.last_synced_at != null ? new Date(status.last_synced_at * 1000) : undefined
+  };
+}
+
+export function coreStatusToJs(status: CoreSyncStatus): sync_status.SyncStatusOptions {
+  const coreCompleteSync = status.priority_status.find((s) => s.priority == FULL_SYNC_PRIORITY);
+  const completeSync = coreCompleteSync != null ? priorityToJs(coreCompleteSync) : null;
+
+  return {
+    connected: status.connected,
+    connecting: status.connecting,
+    dataFlow: {
+      // We expose downloading as a boolean field, the core extension reports download information as a nullable
+      // download status. When that status is non-null, a download is in progress.
+      downloading: status.downloading != null,
+      downloadProgress: status.downloading?.buckets,
+      internalStreamSubscriptions: status.streams
+    },
+    lastSyncedAt: completeSync?.lastSyncedAt,
+    hasSynced: completeSync?.hasSynced,
+    priorityStatusEntries: status.priority_status.map(priorityToJs)
+  };
+}

package/src/client/sync/stream/streaming-sync-types.ts

@@ -0,0 +1,205 @@
+import { BucketChecksum, Checkpoint } from '../bucket/BucketStorageAdapter.js';
+import { CrudEntry, OpId } from '../bucket/CrudEntry.js';
+import { SyncDataBucketJSON } from '../bucket/SyncDataBucket.js';
+
+/**
+ * For sync2.json
+ */
+export interface ContinueCheckpointRequest {
+  /**
+   * Existing bucket states. Only these buckets are synchronized.
+   */
+  buckets: BucketRequest[];
+
+  checkpoint_token: string;
+
+  limit?: number;
+}
+
+export interface SyncNewCheckpointRequest {
+  /**
+   * Existing bucket states. Used if include_data is specified.
+   */
+  buckets?: BucketRequest[];
+
+  request_checkpoint: {
+    /**
+     * Whether or not to include an initial data request.
+     */
+    include_data: boolean;
+
+    /**
+     * Whether or not to compute a checksum.
+     */
+    include_checksum: boolean;
+  };
+
+  limit?: number;
+}
+
+export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
+
+export interface SyncResponse {
+  /**
+   * Data for the buckets returned. May not have an an entry for each bucket in the request.
+   */
+  data?: SyncDataBucketJSON[];
+
+  /**
+   * True if the response limit has been reached, and another request must be made.
+   */
+  has_more: boolean;
+
+  checkpoint_token?: string;
+
+  checkpoint?: Checkpoint;
+}
+
+type JSONValue = string | number | boolean | null | undefined | JSONObject | JSONArray;
+
+interface JSONObject {
+  [key: string]: JSONValue;
+}
+type JSONArray = JSONValue[];
+
+export type StreamingSyncRequestParameterType = JSONValue;
+
+export interface StreamingSyncRequest {
+  /**
+   * Existing bucket states.
+   */
+  buckets?: BucketRequest[];
+
+  /**
+   * If specified, limit the response to only include these buckets.
+   */
+  only?: string[];
+
+  /**
+   * Whether or not to compute a checksum for each checkpoint
+   */
+  include_checksum: boolean;
+
+  /**
+   * Changes the response to stringified data in each OplogEntry
+   */
+  raw_data: boolean;
+
+  /**
+   * Client parameters to be passed to the sync rules.
+   */
+  parameters?: Record<string, StreamingSyncRequestParameterType>;
+
+  client_id?: string;
+}
+
+export interface StreamingSyncCheckpoint {
+  checkpoint: Checkpoint;
+}
+
+export interface StreamingSyncCheckpointDiff {
+  checkpoint_diff: {
+    last_op_id: OpId;
+    updated_buckets: BucketChecksum[];
+    removed_buckets: string[];
+    write_checkpoint?: string;
+  };
+}
+
+export interface StreamingSyncDataJSON {
+  data: SyncDataBucketJSON;
+}
+
+export interface StreamingSyncCheckpointComplete {
+  checkpoint_complete: {
+    last_op_id: OpId;
+  };
+}
+
+export interface StreamingSyncCheckpointPartiallyComplete {
+  partial_checkpoint_complete: {
+    priority: number;
+    last_op_id: OpId;
+  };
+}
+
+export interface StreamingSyncKeepalive {
+  /** If specified, token expires in this many seconds. */
+  token_expires_in: number;
+}
+
+export type StreamingSyncLine =
+  | StreamingSyncDataJSON
+  | StreamingSyncCheckpoint
+  | StreamingSyncCheckpointDiff
+  | StreamingSyncCheckpointComplete
+  | StreamingSyncCheckpointPartiallyComplete
+  | StreamingSyncKeepalive;
+
+export type CrudUploadNotification = { crud_upload_completed: null };
+
+export type StreamingSyncLineOrCrudUploadComplete = StreamingSyncLine | CrudUploadNotification;
+
+export interface BucketRequest {
+  name: string;
+
+  /**
+   * Base-10 number. Sync all data from this bucket with op_id > after.
+   */
+  after: OpId;
+}
+
+export function isStreamingSyncData(line: StreamingSyncLine): line is StreamingSyncDataJSON {
+  return (line as StreamingSyncDataJSON).data != null;
+}
+
+export function isStreamingKeepalive(line: StreamingSyncLine): line is StreamingSyncKeepalive {
+  return (line as StreamingSyncKeepalive).token_expires_in != null;
+}
+
+export function isStreamingSyncCheckpoint(line: StreamingSyncLine): line is StreamingSyncCheckpoint {
+  return (line as StreamingSyncCheckpoint).checkpoint != null;
+}
+
+export function isStreamingSyncCheckpointComplete(line: StreamingSyncLine): line is StreamingSyncCheckpointComplete {
+  return (line as StreamingSyncCheckpointComplete).checkpoint_complete != null;
+}
+
+export function isStreamingSyncCheckpointPartiallyComplete(
+  line: StreamingSyncLine
+): line is StreamingSyncCheckpointPartiallyComplete {
+  return (line as StreamingSyncCheckpointPartiallyComplete).partial_checkpoint_complete != null;
+}
+
+export function isStreamingSyncCheckpointDiff(line: StreamingSyncLine): line is StreamingSyncCheckpointDiff {
+  return (line as StreamingSyncCheckpointDiff).checkpoint_diff != null;
+}
+
+export function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest {
+  return (
+    Array.isArray((request as ContinueCheckpointRequest).buckets) &&
+    typeof (request as ContinueCheckpointRequest).checkpoint_token == 'string'
+  );
+}
+
+export function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest {
+  return typeof (request as SyncNewCheckpointRequest).request_checkpoint == 'object';
+}
+
+/**
+ * For crud.json
+ */
+export interface CrudRequest {
+  data: CrudEntry[];
+}
+
+export interface CrudResponse {
+  /**
+   * A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
+   *
+   * Any earlier checkpoint may or may not contain these changes.
+   *
+   * May be empty when the request contains no ops.
+   */
+  checkpoint?: OpId;
+}

package/src/client/sync/sync-streams.ts

@@ -0,0 +1,107 @@
+import { AbstractPowerSyncDatabase } from '../AbstractPowerSyncDatabase.js';
+
+/**
+ * A description of a sync stream, consisting of its {@link name} and the {@link parameters} used when subscribing.
+ */
+export interface SyncStreamDescription {
+  /**
+   * The name of the stream as it appears in the stream definition for the PowerSync service.
+   */
+  name: string;
+
+  /**
+   * The parameters used to subscribe to the stream, if any.
+   *
+   * The same stream can be subscribed to multiple times with different parameters.
+   */
+  parameters: Record<string, any> | null;
+}
+
+/**
+ * Information about a subscribed sync stream.
+ *
+ * This includes the {@link SyncStreamDescription}, along with information about the current sync status.
+ */
+export interface SyncSubscriptionDescription extends SyncStreamDescription {
+  active: boolean;
+  /**
+   * Whether this stream subscription is included by default, regardless of whether the stream has explicitly been
+   * subscribed to or not.
+   *
+   * It's possible for both {@link isDefault} and {@link hasExplicitSubscription} to be true at the same time - this
+   * happens when a default stream was subscribed explicitly.
+   */
+  isDefault: boolean;
+  /**
+   * Whether this stream has been subscribed to explicitly.
+   *
+   * It's possible for both {@link isDefault} and {@link hasExplicitSubscription} to be true at the same time - this
+   * happens when a default stream was subscribed explicitly.
+   */
+  hasExplicitSubscription: boolean;
+  /**
+   * For sync streams that have a time-to-live, the current time at which the stream would expire if not subscribed to
+   * again.
+   */
+  expiresAt: Date | null;
+  /**
+   * Whether this stream subscription has been synced at least once.
+   */
+  hasSynced: boolean;
+  /**
+   * If {@link hasSynced} is true, the last time data from this stream has been synced.
+   */
+  lastSyncedAt: Date | null;
+}
+
+export interface SyncStreamSubscribeOptions {
+  /**
+   * A "time to live" for this stream subscription, in seconds.
+   *
+   * The TTL control when a stream gets evicted after not having an active {@link SyncStreamSubscription} object
+   * attached to it.
+   */
+  ttl?: number;
+  /**
+   * A priority to assign to this subscription. This overrides the default priority that may have been set on streams.
+   *
+   * For details on priorities, see [priotized sync](https://docs.powersync.com/usage/use-case-examples/prioritized-sync).
+   */
+  priority?: 0 | 1 | 2 | 3;
+}
+
+/**
+ * A handle to a {@link SyncStreamDescription} that allows subscribing to the stream.
+ *
+ * To obtain an instance of {@link SyncStream}, call {@link AbstractPowerSyncDatabase.syncStream}.
+ */
+export interface SyncStream extends SyncStreamDescription {
+  /**
+   * Adds a subscription to this stream, requesting it to be included when connecting to the sync service.
+   *
+   * You should keep a reference to the returned {@link SyncStreamSubscription} object along as you need data for that
+   * stream. As soon as {@link SyncStreamSubscription.unsubscribe} is called for all subscriptions on this stream
+   * (including subscriptions created on other tabs), the {@link SyncStreamSubscribeOptions.ttl} starts ticking and will
+   * eventually evict the stream (unless {@link subscribe} is called again).
+   */
+  subscribe(options?: SyncStreamSubscribeOptions): Promise<SyncStreamSubscription>;
+
+  /**
+   * Clears all subscriptions attached to this stream and resets the TTL for the stream.
+   *
+   * This is a potentially dangerous operations, as it interferes with other stream subscriptions.
+   */
+  unsubscribeAll(): Promise<void>;
+}
+
+export interface SyncStreamSubscription extends SyncStreamDescription {
+  /**
+   * A promise that resolves once data from in this sync stream has been synced and applied.
+   */
+  waitForFirstSync(abort?: AbortSignal): Promise<void>;
+
+  /**
+   * Removes this stream subscription.
+   */
+  unsubscribe(): void;
+}