@powersync/common 1.41.0 → 1.42.0

package/src/client/watched/processors/AbstractQueryProcessor.ts

@@ -0,0 +1,226 @@
+import { AbstractPowerSyncDatabase } from '../../../client/AbstractPowerSyncDatabase.js';
+import { MetaBaseObserver } from '../../../utils/MetaBaseObserver.js';
+import {
+  WatchedQuery,
+  WatchedQueryListener,
+  WatchedQueryListenerEvent,
+  WatchedQueryOptions,
+  WatchedQueryState
+} from '../WatchedQuery.js';
+
+/**
+ * @internal
+ */
+export interface AbstractQueryProcessorOptions<Data, Settings extends WatchedQueryOptions = WatchedQueryOptions> {
+  db: AbstractPowerSyncDatabase;
+  watchOptions: Settings;
+  placeholderData: Data;
+}
+
+/**
+ * @internal
+ */
+export interface LinkQueryOptions<Data, Settings extends WatchedQueryOptions = WatchedQueryOptions> {
+  abortSignal: AbortSignal;
+  settings: Settings;
+}
+
+type MutableDeep<T> =
+  T extends ReadonlyArray<infer U>
+    ? U[] // convert readonly arrays to mutable arrays
+    : T;
+
+/**
+ * @internal Mutable version of {@link WatchedQueryState}.
+ * This is used internally to allow updates to the state.
+ */
+export type MutableWatchedQueryState<Data> = {
+  -readonly [P in keyof WatchedQueryState<Data>]: MutableDeep<WatchedQueryState<Data>[P]>;
+};
+
+type WatchedQueryProcessorListener<Data> = WatchedQueryListener<Data>;
+
+/**
+ * Performs underlying watching and yields a stream of results.
+ * @internal
+ */
+export abstract class AbstractQueryProcessor<
+    Data = unknown[],
+    Settings extends WatchedQueryOptions = WatchedQueryOptions
+  >
+  extends MetaBaseObserver<WatchedQueryProcessorListener<Data>>
+  implements WatchedQuery<Data, Settings>
+{
+  readonly state: WatchedQueryState<Data>;
+
+  protected abortController: AbortController;
+  protected initialized: Promise<void>;
+  protected _closed: boolean;
+  protected disposeListeners: (() => void) | null;
+
+  get closed() {
+    return this._closed;
+  }
+
+  constructor(protected options: AbstractQueryProcessorOptions<Data, Settings>) {
+    super();
+    this.abortController = new AbortController();
+    this._closed = false;
+    this.state = this.constructInitialState();
+    this.disposeListeners = null;
+    this.initialized = this.init(this.abortController.signal);
+  }
+
+  protected constructInitialState(): WatchedQueryState<Data> {
+    return {
+      isLoading: true,
+      isFetching: this.reportFetching, // Only set to true if we will report updates in future
+      error: null,
+      lastUpdated: null,
+      data: this.options.placeholderData
+    };
+  }
+
+  protected get reportFetching() {
+    return this.options.watchOptions.reportFetching ?? true;
+  }
+
+  protected async updateSettingsInternal(settings: Settings, signal: AbortSignal) {
+    // This may have been aborted while awaiting or if multiple calls to `updateSettings` were made
+    if (this._closed || signal.aborted) {
+      return;
+    }
+
+    this.options.watchOptions = settings;
+
+    this.iterateListeners((l) => l[WatchedQueryListenerEvent.SETTINGS_WILL_UPDATE]?.());
+
+    if (!this.state.isFetching && this.reportFetching) {
+      await this.updateState({
+        isFetching: true
+      });
+    }
+
+    await this.runWithReporting(() =>
+      this.linkQuery({
+        abortSignal: signal,
+        settings
+      })
+    );
+  }
+
+  /**
+   * Updates the underlying query.
+   */
+  async updateSettings(settings: Settings) {
+    // Abort the previous request
+    this.abortController.abort();
+
+    // Keep track of this controller's abort status
+    const abortController = new AbortController();
+    // Allow this to be aborted externally
+    this.abortController = abortController;
+
+    await this.initialized;
+    return this.updateSettingsInternal(settings, abortController.signal);
+  }
+
+  /**
+   * This method is used to link a query to the subscribers of this listener class.
+   * This method should perform actual query watching and report results via {@link updateState} method.
+   */
+  protected abstract linkQuery(options: LinkQueryOptions<Data>): Promise<void>;
+
+  protected async updateState(update: Partial<MutableWatchedQueryState<Data>>) {
+    if (this._closed) {
+      return;
+    }
+
+    if (typeof update.error !== 'undefined') {
+      await this.iterateAsyncListenersWithError(async (l) => l.onError?.(update.error!));
+      // An error always stops for the current fetching state
+      update.isFetching = false;
+      update.isLoading = false;
+    }
+
+    Object.assign(this.state, { lastUpdated: new Date() } satisfies Partial<WatchedQueryState<Data>>, update);
+
+    if (typeof update.data !== 'undefined') {
+      await this.iterateAsyncListenersWithError(async (l) => l.onData?.(this.state.data));
+    }
+    await this.iterateAsyncListenersWithError(async (l) => l.onStateChange?.(this.state));
+  }
+
+  /**
+   * Configures base DB listeners and links the query to listeners.
+   */
+  protected async init(signal: AbortSignal) {
+    const { db } = this.options;
+
+    const disposeCloseListener = db.registerListener({
+      closing: async () => {
+        await this.close();
+      }
+    });
+
+    // Wait for the schema to be set before listening to changes
+    await db.waitForReady();
+    const disposeSchemaListener = db.registerListener({
+      schemaChanged: async () => {
+        await this.runWithReporting(async () => {
+          await this.updateSettings(this.options.watchOptions);
+        });
+      }
+    });
+
+    this.disposeListeners = () => {
+      disposeCloseListener();
+      disposeSchemaListener();
+    };
+
+    // Initial setup
+    await this.runWithReporting(async () => {
+      await this.updateSettingsInternal(this.options.watchOptions, signal);
+    });
+  }
+
+  async close() {
+    this._closed = true;
+    this.abortController.abort();
+    this.disposeListeners?.();
+    this.disposeListeners = null;
+    this.iterateListeners((l) => l.closed?.());
+    this.listeners.clear();
+  }
+
+  /**
+   * Runs a callback and reports errors to the error listeners.
+   */
+  protected async runWithReporting<T>(callback: () => Promise<T>): Promise<void> {
+    try {
+      await callback();
+    } catch (error) {
+      // This will update the error on the state and iterate error listeners
+      await this.updateState({ error });
+    }
+  }
+
+  /**
+   * Iterate listeners and reports errors to onError handlers.
+   */
+  protected async iterateAsyncListenersWithError(
+    callback: (listener: Partial<WatchedQueryProcessorListener<Data>>) => Promise<void> | void
+  ) {
+    try {
+      await this.iterateAsyncListeners(async (l) => callback(l));
+    } catch (error) {
+      try {
+        await this.iterateAsyncListeners(async (l) => l.onError?.(error));
+      } catch (error) {
+        // Errors here are ignored
+        // since we are already in an error state
+        this.options.db.logger.error('Watched query error handler threw an Error', error);
+      }
+    }
+  }
+}

package/src/client/watched/processors/DifferentialQueryProcessor.ts

@@ -0,0 +1,305 @@
+import { WatchCompatibleQuery, WatchedQuery, WatchedQueryListener, WatchedQueryOptions } from '../WatchedQuery.js';
+import {
+  AbstractQueryProcessor,
+  AbstractQueryProcessorOptions,
+  LinkQueryOptions,
+  MutableWatchedQueryState
+} from './AbstractQueryProcessor.js';
+
+/**
+ * Represents an updated row in a differential watched query.
+ * It contains both the current and previous state of the row.
+ */
+export interface WatchedQueryRowDifferential<RowType> {
+  readonly current: RowType;
+  readonly previous: RowType;
+}
+
+/**
+ * Represents the result of a watched query that has been diffed.
+ * {@link DifferentialWatchedQueryState#diff} is of the {@link WatchedQueryDifferential} form.
+ */
+export interface WatchedQueryDifferential<RowType> {
+  readonly added: ReadonlyArray<Readonly<RowType>>;
+  /**
+   * The entire current result set.
+   * Array item object references are preserved between updates if the item is unchanged.
+   *
+   * e.g. In the query
+   * ```sql
+   *  SELECT name, make FROM assets ORDER BY make ASC;
+   * ```
+   *
+   * If a previous result set contains an item (A) `{name: 'pc', make: 'Cool PC'}` and
+   * an update has been made which adds another item (B) to the result set (the item A is unchanged) - then
+   * the updated result set will be contain the same object reference, to item A, as the previous result set.
+   * This is regardless of the item A's position in the updated result set.
+   */
+  readonly all: ReadonlyArray<Readonly<RowType>>;
+  readonly removed: ReadonlyArray<Readonly<RowType>>;
+  readonly updated: ReadonlyArray<WatchedQueryRowDifferential<Readonly<RowType>>>;
+  readonly unchanged: ReadonlyArray<Readonly<RowType>>;
+}
+
+/**
+ * Row comparator for differentially watched queries which keys and compares items in the result set.
+ */
+export interface DifferentialWatchedQueryComparator<RowType> {
+  /**
+   * Generates a unique key for the item.
+   */
+  keyBy: (item: RowType) => string;
+  /**
+   * Generates a token for comparing items with matching keys.
+   */
+  compareBy: (item: RowType) => string;
+}
+
+/**
+ * Options for building a differential watched query with the {@link Query} builder.
+ */
+export interface DifferentialWatchedQueryOptions<RowType> extends WatchedQueryOptions {
+  /**
+   * Initial result data which is presented while the initial loading is executing.
+   */
+  placeholderData?: RowType[];
+
+  /**
+   * Row comparator used to identify and compare rows in the result set.
+   * If not provided, the default comparator will be used which keys items by their `id` property if available,
+   * otherwise it uses JSON stringification of the entire item for keying and comparison.
+   * @defaultValue {@link DEFAULT_ROW_COMPARATOR}
+   */
+  rowComparator?: DifferentialWatchedQueryComparator<RowType>;
+}
+
+/**
+ * Settings for differential incremental watched queries using.
+ */
+export interface DifferentialWatchedQuerySettings<RowType> extends DifferentialWatchedQueryOptions<RowType> {
+  /**
+   * The query here must return an array of items that can be differentiated.
+   */
+  query: WatchCompatibleQuery<RowType[]>;
+}
+
+export interface DifferentialWatchedQueryListener<RowType>
+  extends WatchedQueryListener<ReadonlyArray<Readonly<RowType>>> {
+  onDiff?: (diff: WatchedQueryDifferential<RowType>) => void | Promise<void>;
+}
+
+export type DifferentialWatchedQuery<RowType> = WatchedQuery<
+  ReadonlyArray<Readonly<RowType>>,
+  DifferentialWatchedQuerySettings<RowType>,
+  DifferentialWatchedQueryListener<RowType>
+>;
+
+/**
+ * @internal
+ */
+export interface DifferentialQueryProcessorOptions<RowType>
+  extends AbstractQueryProcessorOptions<RowType[], DifferentialWatchedQuerySettings<RowType>> {
+  rowComparator?: DifferentialWatchedQueryComparator<RowType>;
+}
+
+type DataHashMap<RowType> = Map<string, { hash: string; item: RowType }>;
+
+/**
+ * An empty differential result set.
+ * This is used as the initial state for differential incrementally watched queries.
+ */
+export const EMPTY_DIFFERENTIAL = {
+  added: [],
+  all: [],
+  removed: [],
+  updated: [],
+  unchanged: []
+};
+
+/**
+ * Default implementation of the {@link DifferentialWatchedQueryComparator} for watched queries.
+ * It keys items by their `id` property if available, alternatively it uses JSON stringification
+ * of the entire item for the key and comparison.
+ */
+export const DEFAULT_ROW_COMPARATOR: DifferentialWatchedQueryComparator<any> = {
+  keyBy: (item) => {
+    if (item && typeof item == 'object' && typeof item['id'] == 'string') {
+      return item['id'];
+    }
+    return JSON.stringify(item);
+  },
+  compareBy: (item) => JSON.stringify(item)
+};
+
+/**
+ * Uses the PowerSync onChange event to trigger watched queries.
+ * Results are emitted on every change of the relevant tables.
+ * @internal
+ */
+export class DifferentialQueryProcessor<RowType>
+  extends AbstractQueryProcessor<ReadonlyArray<Readonly<RowType>>, DifferentialWatchedQuerySettings<RowType>>
+  implements DifferentialWatchedQuery<RowType>
+{
+  protected comparator: DifferentialWatchedQueryComparator<RowType>;
+
+  constructor(protected options: DifferentialQueryProcessorOptions<RowType>) {
+    super(options);
+    this.comparator = options.rowComparator ?? DEFAULT_ROW_COMPARATOR;
+  }
+
+  /*
+   * @returns If the sets are equal
+   */
+  protected differentiate(
+    current: RowType[],
+    previousMap: DataHashMap<RowType>
+  ): { diff: WatchedQueryDifferential<RowType>; map: DataHashMap<RowType>; hasChanged: boolean } {
+    const { keyBy, compareBy } = this.comparator;
+
+    let hasChanged = false;
+    const currentMap = new Map<string, { hash: string; item: RowType }>();
+    const removedTracker = new Set(previousMap.keys());
+
+    // Allow mutating to populate the data temporarily.
+    const diff = {
+      all: [] as RowType[],
+      added: [] as RowType[],
+      removed: [] as RowType[],
+      updated: [] as WatchedQueryRowDifferential<RowType>[],
+      unchanged: [] as RowType[]
+    };
+
+    /**
+     * Looping over the current result set array is important to preserve
+     * the ordering of the result set.
+     * We can replace items in the current array with previous object references if they are equal.
+     */
+    for (const item of current) {
+      const key = keyBy(item);
+      const hash = compareBy(item);
+      currentMap.set(key, { hash, item });
+
+      const previousItem = previousMap.get(key);
+      if (!previousItem) {
+        // New item
+        hasChanged = true;
+        diff.added.push(item);
+        diff.all.push(item);
+      } else {
+        // Existing item
+        if (hash == previousItem.hash) {
+          diff.unchanged.push(previousItem.item);
+          // Use the previous object reference
+          diff.all.push(previousItem.item);
+          // update the map to preserve the reference
+          currentMap.set(key, previousItem);
+        } else {
+          hasChanged = true;
+          diff.updated.push({ current: item, previous: previousItem.item });
+          // Use the new reference
+          diff.all.push(item);
+        }
+      }
+      // The item is present, we don't consider it removed
+      removedTracker.delete(key);
+    }
+
+    diff.removed = Array.from(removedTracker).map((key) => previousMap.get(key)!.item);
+    hasChanged = hasChanged || diff.removed.length > 0;
+
+    return {
+      diff,
+      hasChanged,
+      map: currentMap
+    };
+  }
+
+  protected async linkQuery(options: LinkQueryOptions<WatchedQueryDifferential<RowType>>): Promise<void> {
+    const { db, watchOptions } = this.options;
+    const { abortSignal } = options;
+
+    const compiledQuery = watchOptions.query.compile();
+    const tables = await db.resolveTables(compiledQuery.sql, compiledQuery.parameters as any[], {
+      tables: options.settings.triggerOnTables
+    });
+
+    let currentMap: DataHashMap<RowType> = new Map();
+
+    // populate the currentMap from the placeholder data
+    this.state.data.forEach((item) => {
+      currentMap.set(this.comparator.keyBy(item), {
+        hash: this.comparator.compareBy(item),
+        item
+      });
+    });
+
+    db.onChangeWithCallback(
+      {
+        onChange: async () => {
+          if (this.closed || abortSignal.aborted) {
+            return;
+          }
+          // This fires for each change of the relevant tables
+          try {
+            if (this.reportFetching && !this.state.isFetching) {
+              await this.updateState({ isFetching: true });
+            }
+
+            const partialStateUpdate: Partial<MutableWatchedQueryState<RowType[]>> = {};
+
+            // Always run the query if an underlying table has changed
+            const result = await watchOptions.query.execute({
+              sql: compiledQuery.sql,
+              // Allows casting from ReadOnlyArray[unknown] to Array<unknown>
+              // This allows simpler compatibility with PowerSync queries
+              parameters: [...compiledQuery.parameters],
+              db: this.options.db
+            });
+
+            if (abortSignal.aborted) {
+              return;
+            }
+
+            if (this.reportFetching) {
+              partialStateUpdate.isFetching = false;
+            }
+
+            if (this.state.isLoading) {
+              partialStateUpdate.isLoading = false;
+            }
+
+            const { diff, hasChanged, map } = this.differentiate(result, currentMap);
+            // Update for future comparisons
+            currentMap = map;
+
+            if (hasChanged) {
+              await this.iterateAsyncListenersWithError((l) => l.onDiff?.(diff));
+              Object.assign(partialStateUpdate, {
+                data: diff.all
+              });
+            }
+
+            if (this.state.error) {
+              partialStateUpdate.error = null;
+            }
+
+            if (Object.keys(partialStateUpdate).length > 0) {
+              await this.updateState(partialStateUpdate);
+            }
+          } catch (error) {
+            await this.updateState({ error });
+          }
+        },
+        onError: async (error) => {
+          await this.updateState({ error });
+        }
+      },
+      {
+        signal: abortSignal,
+        tables,
+        throttleMs: watchOptions.throttleMs,
+        triggerImmediate: true // used to emit the initial state
+      }
+    );
+  }
+}

package/src/client/watched/processors/OnChangeQueryProcessor.ts

@@ -0,0 +1,122 @@
+import { WatchCompatibleQuery, WatchedQuery, WatchedQueryOptions } from '../WatchedQuery.js';
+import {
+  AbstractQueryProcessor,
+  AbstractQueryProcessorOptions,
+  LinkQueryOptions,
+  MutableWatchedQueryState
+} from './AbstractQueryProcessor.js';
+import { WatchedQueryComparator } from './comparators.js';
+
+/**
+ * Settings for {@link WatchedQuery} instances created via {@link Query#watch}.
+ */
+export interface WatchedQuerySettings<DataType> extends WatchedQueryOptions {
+  query: WatchCompatibleQuery<DataType>;
+}
+
+/**
+ * {@link WatchedQuery} returned from {@link Query#watch}.
+ */
+export type StandardWatchedQuery<DataType> = WatchedQuery<DataType, WatchedQuerySettings<DataType>>;
+
+/**
+ * @internal
+ */
+export interface OnChangeQueryProcessorOptions<Data>
+  extends AbstractQueryProcessorOptions<Data, WatchedQuerySettings<Data>> {
+  comparator?: WatchedQueryComparator<Data>;
+}
+
+/**
+ * Uses the PowerSync onChange event to trigger watched queries.
+ * Results are emitted on every change of the relevant tables.
+ * @internal
+ */
+export class OnChangeQueryProcessor<Data> extends AbstractQueryProcessor<Data, WatchedQuerySettings<Data>> {
+  constructor(protected options: OnChangeQueryProcessorOptions<Data>) {
+    super(options);
+  }
+
+  /**
+   * @returns If the sets are equal
+   */
+  protected checkEquality(current: Data, previous: Data): boolean {
+    // Use the provided comparator if available. Assume values are unique if not available.
+    return this.options.comparator?.checkEquality?.(current, previous) ?? false;
+  }
+
+  protected async linkQuery(options: LinkQueryOptions<Data>): Promise<void> {
+    const { db, watchOptions } = this.options;
+    const { abortSignal } = options;
+
+    const compiledQuery = watchOptions.query.compile();
+    const tables = await db.resolveTables(compiledQuery.sql, compiledQuery.parameters as any[], {
+      tables: options.settings.triggerOnTables
+    });
+
+    db.onChangeWithCallback(
+      {
+        onChange: async () => {
+          if (this.closed || abortSignal.aborted) {
+            return;
+          }
+          // This fires for each change of the relevant tables
+          try {
+            if (this.reportFetching && !this.state.isFetching) {
+              await this.updateState({ isFetching: true });
+            }
+
+            const partialStateUpdate: Partial<MutableWatchedQueryState<Data>> & { data?: Data } = {};
+
+            // Always run the query if an underlying table has changed
+            const result = await watchOptions.query.execute({
+              sql: compiledQuery.sql,
+              // Allows casting from ReadOnlyArray[unknown] to Array<unknown>
+              // This allows simpler compatibility with PowerSync queries
+              parameters: [...compiledQuery.parameters],
+              db: this.options.db
+            });
+
+            if (abortSignal.aborted) {
+              return;
+            }
+
+            if (this.reportFetching) {
+              partialStateUpdate.isFetching = false;
+            }
+
+            if (this.state.isLoading) {
+              partialStateUpdate.isLoading = false;
+            }
+
+            // Check if the result has changed
+            if (!this.checkEquality(result, this.state.data)) {
+              Object.assign(partialStateUpdate, {
+                data: result
+              });
+            }
+
+            if (this.state.error) {
+              partialStateUpdate.error = null;
+            }
+
+            if (Object.keys(partialStateUpdate).length > 0) {
+              await this.updateState(partialStateUpdate);
+            }
+          } catch (error) {
+            await this.updateState({ error });
+          }
+        },
+        onError: async (error) => {
+          await this.updateState({ error });
+        }
+      },
+      {
+        signal: abortSignal,
+        tables,
+        throttleMs: watchOptions.throttleMs,
+        triggerImmediate: true // used to emit the initial state
+      }
+    );
+  }
+}

package/src/client/watched/processors/comparators.ts

@@ -0,0 +1,57 @@
+/**
+ * A basic comparator for incrementally watched queries. This performs a single comparison which
+ * determines if the result set has changed. The {@link WatchedQuery} will only emit the new result
+ * if a change has been detected.
+ */
+export interface WatchedQueryComparator<Data> {
+  checkEquality: (current: Data, previous: Data) => boolean;
+}
+
+/**
+ * Options for {@link ArrayComparator}
+ */
+export type ArrayComparatorOptions<ItemType> = {
+  /**
+   * Returns a string to uniquely identify an item in the array.
+   */
+  compareBy: (item: ItemType) => string;
+};
+
+/**
+ * An efficient comparator for {@link WatchedQuery} created with {@link Query#watch}. This has the ability to determine if a query
+ * result has changes without necessarily processing all items in the result.
+ */
+export class ArrayComparator<ItemType> implements WatchedQueryComparator<ItemType[]> {
+  constructor(protected options: ArrayComparatorOptions<ItemType>) {}
+
+  checkEquality(current: ItemType[], previous: ItemType[]) {
+    if (current.length === 0 && previous.length === 0) {
+      return true;
+    }
+
+    if (current.length !== previous.length) {
+      return false;
+    }
+
+    const { compareBy } = this.options;
+
+    // At this point the lengths are equal
+    for (let i = 0; i < current.length; i++) {
+      const currentItem = compareBy(current[i]);
+      const previousItem = compareBy(previous[i]);
+
+      if (currentItem !== previousItem) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+}
+
+/**
+ * Watched query comparator that always reports changed result sets.
+ */
+export const FalsyComparator: WatchedQueryComparator<unknown> = {
+  checkEquality: () => false // Default comparator that always returns false
+};