@powersync/common 1.51.0 → 1.53.0

package/src/client/sync/stream/streaming-sync-types.ts

@@ -1,210 +0,0 @@
-import { BucketChecksum, Checkpoint } from '../bucket/BucketStorageAdapter.js';
-import { CrudEntry, OpId } from '../bucket/CrudEntry.js';
-import { SyncDataBucketJSON } from '../bucket/SyncDataBucket.js';
-
-/**
- * For sync2.json
- */
-export interface ContinueCheckpointRequest {
-  /**
-   * Existing bucket states. Only these buckets are synchronized.
-   */
-  buckets: BucketRequest[];
-
-  checkpoint_token: string;
-
-  limit?: number;
-}
-
-export interface SyncNewCheckpointRequest {
-  /**
-   * Existing bucket states. Used if include_data is specified.
-   */
-  buckets?: BucketRequest[];
-
-  request_checkpoint: {
-    /**
-     * Whether or not to include an initial data request.
-     */
-    include_data: boolean;
-
-    /**
-     * Whether or not to compute a checksum.
-     */
-    include_checksum: boolean;
-  };
-
-  limit?: number;
-}
-
-export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
-
-export interface SyncResponse {
-  /**
-   * Data for the buckets returned. May not have an an entry for each bucket in the request.
-   */
-  data?: SyncDataBucketJSON[];
-
-  /**
-   * True if the response limit has been reached, and another request must be made.
-   */
-  has_more: boolean;
-
-  checkpoint_token?: string;
-
-  checkpoint?: Checkpoint;
-}
-
-type JSONValue = string | number | boolean | null | undefined | JSONObject | JSONArray;
-
-interface JSONObject {
-  [key: string]: JSONValue;
-}
-type JSONArray = JSONValue[];
-
-export type StreamingSyncRequestParameterType = JSONValue;
-
-export interface StreamingSyncRequest {
-  /**
-   * Existing bucket states.
-   */
-  buckets?: BucketRequest[];
-
-  /**
-   * If specified, limit the response to only include these buckets.
-   */
-  only?: string[];
-
-  /**
-   * Whether or not to compute a checksum for each checkpoint
-   */
-  include_checksum: boolean;
-
-  /**
-   * Changes the response to stringified data in each OplogEntry
-   */
-  raw_data: boolean;
-
-  /**
-   * Client parameters to be passed to the sync rules.
-   */
-  parameters?: Record<string, StreamingSyncRequestParameterType>;
-
-  /**
-   * Application metadata to be included in service logs.
-   */
-  app_metadata?: Record<string, string>;
-
-  client_id?: string;
-}
-
-export interface StreamingSyncCheckpoint {
-  checkpoint: Checkpoint;
-}
-
-export interface StreamingSyncCheckpointDiff {
-  checkpoint_diff: {
-    last_op_id: OpId;
-    updated_buckets: BucketChecksum[];
-    removed_buckets: string[];
-    write_checkpoint?: string;
-  };
-}
-
-export interface StreamingSyncDataJSON {
-  data: SyncDataBucketJSON;
-}
-
-export interface StreamingSyncCheckpointComplete {
-  checkpoint_complete: {
-    last_op_id: OpId;
-  };
-}
-
-export interface StreamingSyncCheckpointPartiallyComplete {
-  partial_checkpoint_complete: {
-    priority: number;
-    last_op_id: OpId;
-  };
-}
-
-export interface StreamingSyncKeepalive {
-  /** If specified, token expires in this many seconds. */
-  token_expires_in: number;
-}
-
-export type StreamingSyncLine =
-  | StreamingSyncDataJSON
-  | StreamingSyncCheckpoint
-  | StreamingSyncCheckpointDiff
-  | StreamingSyncCheckpointComplete
-  | StreamingSyncCheckpointPartiallyComplete
-  | StreamingSyncKeepalive;
-
-export type CrudUploadNotification = { crud_upload_completed: null };
-
-export type StreamingSyncLineOrCrudUploadComplete = StreamingSyncLine | CrudUploadNotification;
-
-export interface BucketRequest {
-  name: string;
-
-  /**
-   * Base-10 number. Sync all data from this bucket with op_id > after.
-   */
-  after: OpId;
-}
-
-export function isStreamingSyncData(line: StreamingSyncLine): line is StreamingSyncDataJSON {
-  return (line as StreamingSyncDataJSON).data != null;
-}
-
-export function isStreamingKeepalive(line: StreamingSyncLine): line is StreamingSyncKeepalive {
-  return (line as StreamingSyncKeepalive).token_expires_in != null;
-}
-
-export function isStreamingSyncCheckpoint(line: StreamingSyncLine): line is StreamingSyncCheckpoint {
-  return (line as StreamingSyncCheckpoint).checkpoint != null;
-}
-
-export function isStreamingSyncCheckpointComplete(line: StreamingSyncLine): line is StreamingSyncCheckpointComplete {
-  return (line as StreamingSyncCheckpointComplete).checkpoint_complete != null;
-}
-
-export function isStreamingSyncCheckpointPartiallyComplete(
-  line: StreamingSyncLine
-): line is StreamingSyncCheckpointPartiallyComplete {
-  return (line as StreamingSyncCheckpointPartiallyComplete).partial_checkpoint_complete != null;
-}
-
-export function isStreamingSyncCheckpointDiff(line: StreamingSyncLine): line is StreamingSyncCheckpointDiff {
-  return (line as StreamingSyncCheckpointDiff).checkpoint_diff != null;
-}
-
-export function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest {
-  return (
-    Array.isArray((request as ContinueCheckpointRequest).buckets) &&
-    typeof (request as ContinueCheckpointRequest).checkpoint_token == 'string'
-  );
-}
-
-export function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest {
-  return typeof (request as SyncNewCheckpointRequest).request_checkpoint == 'object';
-}
-
-/**
- * For crud.json
- */
-export interface CrudRequest {
-  data: CrudEntry[];
-}
-
-export interface CrudResponse {
-  /**
-   * A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
-   *
-   * Any earlier checkpoint may or may not contain these changes.
-   *
-   * May be empty when the request contains no ops.
-   */
-  checkpoint?: OpId;
-}

package/src/utils/DataStream.ts

@@ -1,222 +0,0 @@
-import Logger, { ILogger } from 'js-logger';
-import { BaseListener, BaseObserver } from './BaseObserver.js';
-
-export type DataStreamOptions<ParsedData, SourceData> = {
-  mapLine?: (line: SourceData) => ParsedData;
-
-  /**
-   * Close the stream if any consumer throws an error
-   */
-  closeOnError?: boolean;
-  pressure?: {
-    highWaterMark?: number;
-    lowWaterMark?: number;
-  };
-  logger?: ILogger;
-};
-
-export type DataStreamCallback<Data extends any = any> = (data: Data) => Promise<void>;
-
-export interface DataStreamListener<Data extends any = any> extends BaseListener {
-  data: (data: Data) => Promise<void>;
-  closed: () => void;
-  error: (error: Error) => void;
-  highWater: () => Promise<void>;
-  lowWater: () => Promise<void>;
-}
-
-export const DEFAULT_PRESSURE_LIMITS = {
-  highWater: 10,
-  lowWater: 0
-};
-
-/**
- * A very basic implementation of a data stream with backpressure support which does not use
- * native JS streams or async iterators.
- * This is handy for environments such as React Native which need polyfills for the above.
- */
-export class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
-  dataQueue: SourceData[];
-
-  protected isClosed: boolean;
-
-  protected processingPromise: Promise<void> | null;
-  protected notifyDataAdded: (() => void) | null;
-
-  protected logger: ILogger;
-
-  protected mapLine: (line: SourceData) => ParsedData;
-
-  constructor(protected options?: DataStreamOptions<ParsedData, SourceData>) {
-    super();
-    this.processingPromise = null;
-    this.isClosed = false;
-    this.dataQueue = [];
-    this.mapLine = options?.mapLine ?? ((line) => line as any);
-
-    this.logger = options?.logger ?? Logger.get('DataStream');
-
-    if (options?.closeOnError) {
-      const l = this.registerListener({
-        error: (ex) => {
-          l?.();
-          this.close();
-        }
-      });
-    }
-  }
-
-  get highWatermark() {
-    return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
-  }
-
-  get lowWatermark() {
-    return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
-  }
-
-  get closed() {
-    return this.isClosed;
-  }
-
-  async close() {
-    this.isClosed = true;
-    await this.processingPromise;
-    this.iterateListeners((l) => l.closed?.());
-    // Discard any data in the queue
-    this.dataQueue = [];
-    this.listeners.clear();
-  }
-
-  /**
-   * Enqueues data for the consumers to read
-   */
-  enqueueData(data: SourceData) {
-    if (this.isClosed) {
-      throw new Error('Cannot enqueue data into closed stream.');
-    }
-
-    this.dataQueue.push(data);
-    this.notifyDataAdded?.();
-
-    this.processQueue();
-  }
-
-  /**
-   * Reads data once from the data stream
-   * @returns a Data payload or Null if the stream closed.
-   */
-  async read(): Promise<ParsedData | null> {
-    if (this.closed) {
-      return null;
-    }
-
-    // Wait for any pending processing to complete first.
-    // This ensures we register our listener before calling processQueue(),
-    // avoiding a race where processQueue() sees no reader and returns early.
-    if (this.processingPromise) {
-      await this.processingPromise;
-    }
-
-    // Re-check after await - stream may have closed while we were waiting
-    if (this.closed) {
-      return null;
-    }
-
-    return new Promise((resolve, reject) => {
-      const l = this.registerListener({
-        data: async (data) => {
-          resolve(data);
-          // Remove the listener
-          l?.();
-        },
-        closed: () => {
-          resolve(null);
-          l?.();
-        },
-        error: (ex) => {
-          reject(ex);
-          l?.();
-        }
-      });
-
-      this.processQueue();
-    });
-  }
-
-  /**
-   * Executes a callback for each data item in the stream
-   */
-  forEach(callback: DataStreamCallback<ParsedData>) {
-    if (this.dataQueue.length <= this.lowWatermark) {
-      this.iterateAsyncErrored(async (l) => l.lowWater?.());
-    }
-
-    return this.registerListener({
-      data: callback
-    });
-  }
-
-  protected processQueue() {
-    if (this.processingPromise) {
-      return;
-    }
-
-    const promise = (this.processingPromise = this._processQueue());
-    promise.finally(() => {
-      this.processingPromise = null;
-    });
-    return promise;
-  }
-
-  protected hasDataReader() {
-    return Array.from(this.listeners.values()).some((l) => !!l.data);
-  }
-
-  protected async _processQueue() {
-    /**
-     * Allow listeners to mutate the queue before processing.
-     * This allows for operations such as dropping or compressing data
-     * on high water or requesting more data on low water.
-     */
-    if (this.dataQueue.length >= this.highWatermark) {
-      await this.iterateAsyncErrored(async (l) => l.highWater?.());
-    }
-
-    if (this.isClosed || !this.hasDataReader()) {
-      return;
-    }
-
-    if (this.dataQueue.length) {
-      const data = this.dataQueue.shift()!;
-      const mapped = this.mapLine(data);
-      await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
-    }
-
-    if (this.dataQueue.length <= this.lowWatermark) {
-      const dataAdded = new Promise<void>((resolve) => {
-        this.notifyDataAdded = resolve;
-      });
-
-      await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
-      this.notifyDataAdded = null;
-    }
-
-    if (this.dataQueue.length > 0) {
-      setTimeout(() => this.processQueue());
-    }
-  }
-
-  protected async iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>) {
-    // Important: We need to copy the listeners, as calling a listener could result in adding another
-    // listener, resulting in infinite loops.
-    const listeners = Array.from(this.listeners.values());
-    for (let i of listeners) {
-      try {
-        await cb(i);
-      } catch (ex) {
-        this.logger.error(ex);
-        this.iterateListeners((l) => l.error?.(ex));
-      }
-    }
-  }
-}