@powersync/common 1.40.0 → 1.41.1

package/src/utils/DataStream.ts

@@ -0,0 +1,211 @@
+import Logger, { ILogger } from 'js-logger';
+import { BaseListener, BaseObserver } from './BaseObserver.js';
+
+export type DataStreamOptions<ParsedData, SourceData> = {
+  mapLine?: (line: SourceData) => ParsedData;
+
+  /**
+   * Close the stream if any consumer throws an error
+   */
+  closeOnError?: boolean;
+  pressure?: {
+    highWaterMark?: number;
+    lowWaterMark?: number;
+  };
+  logger?: ILogger;
+};
+
+export type DataStreamCallback<Data extends any = any> = (data: Data) => Promise<void>;
+
+export interface DataStreamListener<Data extends any = any> extends BaseListener {
+  data: (data: Data) => Promise<void>;
+  closed: () => void;
+  error: (error: Error) => void;
+  highWater: () => Promise<void>;
+  lowWater: () => Promise<void>;
+}
+
+export const DEFAULT_PRESSURE_LIMITS = {
+  highWater: 10,
+  lowWater: 0
+};
+
+/**
+ * A very basic implementation of a data stream with backpressure support which does not use
+ * native JS streams or async iterators.
+ * This is handy for environments such as React Native which need polyfills for the above.
+ */
+export class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
+  dataQueue: SourceData[];
+
+  protected isClosed: boolean;
+
+  protected processingPromise: Promise<void> | null;
+  protected notifyDataAdded: (() => void) | null;
+
+  protected logger: ILogger;
+
+  protected mapLine: (line: SourceData) => ParsedData;
+
+  constructor(protected options?: DataStreamOptions<ParsedData, SourceData>) {
+    super();
+    this.processingPromise = null;
+    this.isClosed = false;
+    this.dataQueue = [];
+    this.mapLine = options?.mapLine ?? ((line) => line as any);
+
+    this.logger = options?.logger ?? Logger.get('DataStream');
+
+    if (options?.closeOnError) {
+      const l = this.registerListener({
+        error: (ex) => {
+          l?.();
+          this.close();
+        }
+      });
+    }
+  }
+
+  get highWatermark() {
+    return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
+  }
+
+  get lowWatermark() {
+    return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
+  }
+
+  get closed() {
+    return this.isClosed;
+  }
+
+  async close() {
+    this.isClosed = true;
+    await this.processingPromise;
+    this.iterateListeners((l) => l.closed?.());
+    // Discard any data in the queue
+    this.dataQueue = [];
+    this.listeners.clear();
+  }
+
+  /**
+   * Enqueues data for the consumers to read
+   */
+  enqueueData(data: SourceData) {
+    if (this.isClosed) {
+      throw new Error('Cannot enqueue data into closed stream.');
+    }
+
+    this.dataQueue.push(data);
+    this.notifyDataAdded?.();
+
+    this.processQueue();
+  }
+
+  /**
+   * Reads data once from the data stream
+   * @returns a Data payload or Null if the stream closed.
+   */
+  async read(): Promise<ParsedData | null> {
+    if (this.closed) {
+      return null;
+    }
+
+    return new Promise((resolve, reject) => {
+      const l = this.registerListener({
+        data: async (data) => {
+          resolve(data);
+          // Remove the listener
+          l?.();
+        },
+        closed: () => {
+          resolve(null);
+          l?.();
+        },
+        error: (ex) => {
+          reject(ex);
+          l?.();
+        }
+      });
+
+      this.processQueue();
+    });
+  }
+
+  /**
+   * Executes a callback for each data item in the stream
+   */
+  forEach(callback: DataStreamCallback<ParsedData>) {
+    if (this.dataQueue.length <= this.lowWatermark) {
+      this.iterateAsyncErrored(async (l) => l.lowWater?.());
+    }
+
+    return this.registerListener({
+      data: callback
+    });
+  }
+
+  protected processQueue() {
+    if (this.processingPromise) {
+      return;
+    }
+
+    const promise = (this.processingPromise = this._processQueue());
+    promise.finally(() => {
+      return (this.processingPromise = null);
+    });
+    return promise;
+  }
+
+  protected hasDataReader() {
+    return Array.from(this.listeners.values()).some((l) => !!l.data);
+  }
+
+  protected async _processQueue() {
+    /**
+     * Allow listeners to mutate the queue before processing.
+     * This allows for operations such as dropping or compressing data
+     * on high water or requesting more data on low water.
+     */
+    if (this.dataQueue.length >= this.highWatermark) {
+      await this.iterateAsyncErrored(async (l) => l.highWater?.());
+    }
+
+    if (this.isClosed || !this.hasDataReader()) {
+      return;
+    }
+
+    if (this.dataQueue.length) {
+      const data = this.dataQueue.shift()!;
+      const mapped = this.mapLine(data);
+      await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
+    }
+
+    if (this.dataQueue.length <= this.lowWatermark) {
+      const dataAdded = new Promise<void>((resolve) => {
+        this.notifyDataAdded = resolve;
+      });
+
+      await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
+      this.notifyDataAdded = null;
+    }
+
+    if (this.dataQueue.length > 0) {
+      // Next tick
+      setTimeout(() => this.processQueue());
+    }
+  }
+
+  protected async iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>) {
+    // Important: We need to copy the listeners, as calling a listener could result in adding another
+    // listener, resulting in infinite loops.
+    const listeners = Array.from(this.listeners.values());
+    for (let i of listeners) {
+      try {
+        await cb(i);
+      } catch (ex) {
+        this.logger.error(ex);
+        this.iterateListeners((l) => l.error?.(ex));
+      }
+    }
+  }
+}

package/src/utils/Logger.ts

@@ -0,0 +1,47 @@
+import Logger, { type ILogger, type ILogLevel } from 'js-logger';
+
+export { GlobalLogger, ILogger, ILoggerOpts, ILogHandler, ILogLevel } from 'js-logger';
+
+const TypedLogger: ILogger = Logger as any;
+
+export const LogLevel = {
+  TRACE: TypedLogger.TRACE,
+  DEBUG: TypedLogger.DEBUG,
+  INFO: TypedLogger.INFO,
+  TIME: TypedLogger.TIME,
+  WARN: TypedLogger.WARN,
+  ERROR: TypedLogger.ERROR,
+  OFF: TypedLogger.OFF
+};
+
+export interface CreateLoggerOptions {
+  logLevel?: ILogLevel;
+}
+
+/**
+ * Retrieves the base (default) logger instance.
+ *
+ * This base logger controls the default logging configuration and is shared
+ * across all loggers created with `createLogger`. Adjusting settings on this
+ * base logger affects all loggers derived from it unless explicitly overridden.
+ *
+ */
+export function createBaseLogger() {
+  return Logger;
+}
+
+/**
+ * Creates and configures a new named logger based on the base logger.
+ *
+ * Named loggers allow specific modules or areas of your application to have
+ * their own logging levels and behaviors. These loggers inherit configuration
+ * from the base logger by default but can override settings independently.
+ */
+export function createLogger(name: string, options: CreateLoggerOptions = {}): ILogger {
+  const logger = Logger.get(name);
+  if (options.logLevel) {
+    logger.setLevel(options.logLevel);
+  }
+
+  return logger;
+}

package/src/utils/MetaBaseObserver.ts

@@ -0,0 +1,81 @@
+import { BaseListener, BaseObserver, BaseObserverInterface } from './BaseObserver.js';
+
+/**
+ * Represents the counts of listeners for each event type in a BaseListener.
+ */
+export type ListenerCounts<Listener extends BaseListener> = Partial<Record<keyof Listener, number>> & {
+  total: number;
+};
+
+/**
+ * Meta listener which reports the counts of listeners for each event type.
+ */
+export interface MetaListener<ParentListener extends BaseListener> extends BaseListener {
+  listenersChanged?: (counts: ListenerCounts<ParentListener>) => void;
+}
+
+export interface ListenerMetaManager<Listener extends BaseListener>
+  extends BaseObserverInterface<MetaListener<Listener>> {
+  counts: ListenerCounts<Listener>;
+}
+
+export interface MetaBaseObserverInterface<Listener extends BaseListener> extends BaseObserverInterface<Listener> {
+  listenerMeta: ListenerMetaManager<Listener>;
+}
+
+/**
+ * A BaseObserver that tracks the counts of listeners for each event type.
+ */
+export class MetaBaseObserver<Listener extends BaseListener>
+  extends BaseObserver<Listener>
+  implements MetaBaseObserverInterface<Listener>
+{
+  protected get listenerCounts(): ListenerCounts<Listener> {
+    const counts = {} as Partial<Record<keyof Listener, number>>;
+    let total = 0;
+    for (const listener of this.listeners) {
+      for (const key in listener) {
+        if (listener[key]) {
+          counts[key] = (counts[key] ?? 0) + 1;
+          total++;
+        }
+      }
+    }
+    return {
+      ...counts,
+      total
+    };
+  }
+
+  get listenerMeta(): ListenerMetaManager<Listener> {
+    return {
+      counts: this.listenerCounts,
+      // Allows registering a meta listener that will be notified of changes in listener counts
+      registerListener: (listener: Partial<MetaListener<Listener>>) => {
+        return this.metaListener.registerListener(listener);
+      }
+    };
+  }
+
+  protected metaListener: BaseObserver<MetaListener<Listener>>;
+
+  constructor() {
+    super();
+    this.metaListener = new BaseObserver<MetaListener<Listener>>();
+  }
+
+  registerListener(listener: Partial<Listener>): () => void {
+    const dispose = super.registerListener(listener);
+    const updatedCount = this.listenerCounts;
+    this.metaListener.iterateListeners((l) => {
+      l.listenersChanged?.(updatedCount);
+    });
+    return () => {
+      dispose();
+      const updatedCount = this.listenerCounts;
+      this.metaListener.iterateListeners((l) => {
+        l.listenersChanged?.(updatedCount);
+      });
+    };
+  }
+}

package/src/utils/async.ts

@@ -0,0 +1,61 @@
+/**
+ * A ponyfill for `Symbol.asyncIterator` that is compatible with the
+ * [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
+ * we recommend for React Native.
+ *
+ * As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
+ * iterators without requiring them.
+ */
+export const symbolAsyncIterator: typeof Symbol.asyncIterator =
+  Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
+
+/**
+ * Throttle a function to be called at most once every "wait" milliseconds,
+ * on the trailing edge.
+ *
+ * Roughly equivalent to lodash/throttle with {leading: false, trailing: true}
+ */
+export function throttleTrailing(func: () => void, wait: number) {
+  let timeoutId: ReturnType<typeof setTimeout> | null = null;
+
+  const later = () => {
+    func();
+    timeoutId = null;
+  };
+
+  return function () {
+    if (timeoutId == null) {
+      timeoutId = setTimeout(later, wait);
+    }
+  };
+}
+
+/**
+ * Throttle a function to be called at most once every "wait" milliseconds,
+ * on the leading and trailing edge.
+ *
+ * Roughly equivalent to lodash/throttle with {leading: true, trailing: true}
+ */
+export function throttleLeadingTrailing(func: () => void, wait: number) {
+  let timeoutId: ReturnType<typeof setTimeout> | null = null;
+  let lastCallTime: number = 0;
+
+  const invokeFunction = () => {
+    func();
+    lastCallTime = Date.now();
+    timeoutId = null;
+  };
+
+  return function () {
+    const now = Date.now();
+    const timeToWait = wait - (now - lastCallTime);
+
+    if (timeToWait <= 0) {
+      // Leading edge: Call the function immediately if enough time has passed
+      invokeFunction();
+    } else if (!timeoutId) {
+      // Set a timeout for the trailing edge if not already set
+      timeoutId = setTimeout(invokeFunction, timeToWait);
+    }
+  };
+}

package/src/utils/mutex.ts

@@ -0,0 +1,34 @@
+import { Mutex } from 'async-mutex';
+
+/**
+ * Wrapper for async-mutex runExclusive, which allows for a timeout on each exclusive lock.
+ */
+export async function mutexRunExclusive<T>(
+  mutex: Mutex,
+  callback: () => Promise<T>,
+  options?: { timeoutMs: number }
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const timeout = options?.timeoutMs;
+    let timedOut = false;
+    const timeoutId = timeout
+      ? setTimeout(() => {
+          timedOut = true;
+          reject(new Error('Timeout waiting for lock'));
+        }, timeout)
+      : undefined;
+
+    mutex.runExclusive(async () => {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+      if (timedOut) return;
+
+      try {
+        resolve(await callback());
+      } catch (ex) {
+        reject(ex);
+      }
+    });
+  });
+}

package/src/utils/parseQuery.ts

@@ -0,0 +1,25 @@
+import type { CompilableQuery } from '../types/types.js';
+
+export interface ParsedQuery {
+  sqlStatement: string;
+  parameters: any[];
+}
+
+export const parseQuery = <T>(query: string | CompilableQuery<T>, parameters: any[]): ParsedQuery => {
+  let sqlStatement: string;
+
+  if (typeof query == 'string') {
+    sqlStatement = query;
+  } else {
+    const hasAdditionalParameters = parameters.length > 0;
+    if (hasAdditionalParameters) {
+      throw new Error('You cannot pass parameters to a compiled query.');
+    }
+
+    const compiled = query.compile();
+    sqlStatement = compiled.sql;
+    parameters = compiled.parameters as any[];
+  }
+
+  return { sqlStatement, parameters: parameters };
+};