@powersync/common 1.40.0 → 1.41.1

package/src/client/watched/processors/DifferentialQueryProcessor.ts

@@ -0,0 +1,305 @@
+import { WatchCompatibleQuery, WatchedQuery, WatchedQueryListener, WatchedQueryOptions } from '../WatchedQuery.js';
+import {
+  AbstractQueryProcessor,
+  AbstractQueryProcessorOptions,
+  LinkQueryOptions,
+  MutableWatchedQueryState
+} from './AbstractQueryProcessor.js';
+
+/**
+ * Represents an updated row in a differential watched query.
+ * It contains both the current and previous state of the row.
+ */
+export interface WatchedQueryRowDifferential<RowType> {
+  readonly current: RowType;
+  readonly previous: RowType;
+}
+
+/**
+ * Represents the result of a watched query that has been diffed.
+ * {@link DifferentialWatchedQueryState#diff} is of the {@link WatchedQueryDifferential} form.
+ */
+export interface WatchedQueryDifferential<RowType> {
+  readonly added: ReadonlyArray<Readonly<RowType>>;
+  /**
+   * The entire current result set.
+   * Array item object references are preserved between updates if the item is unchanged.
+   *
+   * e.g. In the query
+   * ```sql
+   *  SELECT name, make FROM assets ORDER BY make ASC;
+   * ```
+   *
+   * If a previous result set contains an item (A) `{name: 'pc', make: 'Cool PC'}` and
+   * an update has been made which adds another item (B) to the result set (the item A is unchanged) - then
+   * the updated result set will be contain the same object reference, to item A, as the previous result set.
+   * This is regardless of the item A's position in the updated result set.
+   */
+  readonly all: ReadonlyArray<Readonly<RowType>>;
+  readonly removed: ReadonlyArray<Readonly<RowType>>;
+  readonly updated: ReadonlyArray<WatchedQueryRowDifferential<Readonly<RowType>>>;
+  readonly unchanged: ReadonlyArray<Readonly<RowType>>;
+}
+
+/**
+ * Row comparator for differentially watched queries which keys and compares items in the result set.
+ */
+export interface DifferentialWatchedQueryComparator<RowType> {
+  /**
+   * Generates a unique key for the item.
+   */
+  keyBy: (item: RowType) => string;
+  /**
+   * Generates a token for comparing items with matching keys.
+   */
+  compareBy: (item: RowType) => string;
+}
+
+/**
+ * Options for building a differential watched query with the {@link Query} builder.
+ */
+export interface DifferentialWatchedQueryOptions<RowType> extends WatchedQueryOptions {
+  /**
+   * Initial result data which is presented while the initial loading is executing.
+   */
+  placeholderData?: RowType[];
+
+  /**
+   * Row comparator used to identify and compare rows in the result set.
+   * If not provided, the default comparator will be used which keys items by their `id` property if available,
+   * otherwise it uses JSON stringification of the entire item for keying and comparison.
+   * @defaultValue {@link DEFAULT_ROW_COMPARATOR}
+   */
+  rowComparator?: DifferentialWatchedQueryComparator<RowType>;
+}
+
+/**
+ * Settings for differential incremental watched queries using.
+ */
+export interface DifferentialWatchedQuerySettings<RowType> extends DifferentialWatchedQueryOptions<RowType> {
+  /**
+   * The query here must return an array of items that can be differentiated.
+   */
+  query: WatchCompatibleQuery<RowType[]>;
+}
+
+export interface DifferentialWatchedQueryListener<RowType>
+  extends WatchedQueryListener<ReadonlyArray<Readonly<RowType>>> {
+  onDiff?: (diff: WatchedQueryDifferential<RowType>) => void | Promise<void>;
+}
+
+export type DifferentialWatchedQuery<RowType> = WatchedQuery<
+  ReadonlyArray<Readonly<RowType>>,
+  DifferentialWatchedQuerySettings<RowType>,
+  DifferentialWatchedQueryListener<RowType>
+>;
+
+/**
+ * @internal
+ */
+export interface DifferentialQueryProcessorOptions<RowType>
+  extends AbstractQueryProcessorOptions<RowType[], DifferentialWatchedQuerySettings<RowType>> {
+  rowComparator?: DifferentialWatchedQueryComparator<RowType>;
+}
+
+type DataHashMap<RowType> = Map<string, { hash: string; item: RowType }>;
+
+/**
+ * An empty differential result set.
+ * This is used as the initial state for differential incrementally watched queries.
+ */
+export const EMPTY_DIFFERENTIAL = {
+  added: [],
+  all: [],
+  removed: [],
+  updated: [],
+  unchanged: []
+};
+
+/**
+ * Default implementation of the {@link DifferentialWatchedQueryComparator} for watched queries.
+ * It keys items by their `id` property if available, alternatively it uses JSON stringification
+ * of the entire item for the key and comparison.
+ */
+export const DEFAULT_ROW_COMPARATOR: DifferentialWatchedQueryComparator<any> = {
+  keyBy: (item) => {
+    if (item && typeof item == 'object' && typeof item['id'] == 'string') {
+      return item['id'];
+    }
+    return JSON.stringify(item);
+  },
+  compareBy: (item) => JSON.stringify(item)
+};
+
+/**
+ * Uses the PowerSync onChange event to trigger watched queries.
+ * Results are emitted on every change of the relevant tables.
+ * @internal
+ */
+export class DifferentialQueryProcessor<RowType>
+  extends AbstractQueryProcessor<ReadonlyArray<Readonly<RowType>>, DifferentialWatchedQuerySettings<RowType>>
+  implements DifferentialWatchedQuery<RowType>
+{
+  protected comparator: DifferentialWatchedQueryComparator<RowType>;
+
+  constructor(protected options: DifferentialQueryProcessorOptions<RowType>) {
+    super(options);
+    this.comparator = options.rowComparator ?? DEFAULT_ROW_COMPARATOR;
+  }
+
+  /*
+   * @returns If the sets are equal
+   */
+  protected differentiate(
+    current: RowType[],
+    previousMap: DataHashMap<RowType>
+  ): { diff: WatchedQueryDifferential<RowType>; map: DataHashMap<RowType>; hasChanged: boolean } {
+    const { keyBy, compareBy } = this.comparator;
+
+    let hasChanged = false;
+    const currentMap = new Map<string, { hash: string; item: RowType }>();
+    const removedTracker = new Set(previousMap.keys());
+
+    // Allow mutating to populate the data temporarily.
+    const diff = {
+      all: [] as RowType[],
+      added: [] as RowType[],
+      removed: [] as RowType[],
+      updated: [] as WatchedQueryRowDifferential<RowType>[],
+      unchanged: [] as RowType[]
+    };
+
+    /**
+     * Looping over the current result set array is important to preserve
+     * the ordering of the result set.
+     * We can replace items in the current array with previous object references if they are equal.
+     */
+    for (const item of current) {
+      const key = keyBy(item);
+      const hash = compareBy(item);
+      currentMap.set(key, { hash, item });
+
+      const previousItem = previousMap.get(key);
+      if (!previousItem) {
+        // New item
+        hasChanged = true;
+        diff.added.push(item);
+        diff.all.push(item);
+      } else {
+        // Existing item
+        if (hash == previousItem.hash) {
+          diff.unchanged.push(previousItem.item);
+          // Use the previous object reference
+          diff.all.push(previousItem.item);
+          // update the map to preserve the reference
+          currentMap.set(key, previousItem);
+        } else {
+          hasChanged = true;
+          diff.updated.push({ current: item, previous: previousItem.item });
+          // Use the new reference
+          diff.all.push(item);
+        }
+      }
+      // The item is present, we don't consider it removed
+      removedTracker.delete(key);
+    }
+
+    diff.removed = Array.from(removedTracker).map((key) => previousMap.get(key)!.item);
+    hasChanged = hasChanged || diff.removed.length > 0;
+
+    return {
+      diff,
+      hasChanged,
+      map: currentMap
+    };
+  }
+
+  protected async linkQuery(options: LinkQueryOptions<WatchedQueryDifferential<RowType>>): Promise<void> {
+    const { db, watchOptions } = this.options;
+    const { abortSignal } = options;
+
+    const compiledQuery = watchOptions.query.compile();
+    const tables = await db.resolveTables(compiledQuery.sql, compiledQuery.parameters as any[], {
+      tables: options.settings.triggerOnTables
+    });
+
+    let currentMap: DataHashMap<RowType> = new Map();
+
+    // populate the currentMap from the placeholder data
+    this.state.data.forEach((item) => {
+      currentMap.set(this.comparator.keyBy(item), {
+        hash: this.comparator.compareBy(item),
+        item
+      });
+    });
+
+    db.onChangeWithCallback(
+      {
+        onChange: async () => {
+          if (this.closed || abortSignal.aborted) {
+            return;
+          }
+          // This fires for each change of the relevant tables
+          try {
+            if (this.reportFetching && !this.state.isFetching) {
+              await this.updateState({ isFetching: true });
+            }
+
+            const partialStateUpdate: Partial<MutableWatchedQueryState<RowType[]>> = {};
+
+            // Always run the query if an underlying table has changed
+            const result = await watchOptions.query.execute({
+              sql: compiledQuery.sql,
+              // Allows casting from ReadOnlyArray[unknown] to Array<unknown>
+              // This allows simpler compatibility with PowerSync queries
+              parameters: [...compiledQuery.parameters],
+              db: this.options.db
+            });
+
+            if (abortSignal.aborted) {
+              return;
+            }
+
+            if (this.reportFetching) {
+              partialStateUpdate.isFetching = false;
+            }
+
+            if (this.state.isLoading) {
+              partialStateUpdate.isLoading = false;
+            }
+
+            const { diff, hasChanged, map } = this.differentiate(result, currentMap);
+            // Update for future comparisons
+            currentMap = map;
+
+            if (hasChanged) {
+              await this.iterateAsyncListenersWithError((l) => l.onDiff?.(diff));
+              Object.assign(partialStateUpdate, {
+                data: diff.all
+              });
+            }
+
+            if (this.state.error) {
+              partialStateUpdate.error = null;
+            }
+
+            if (Object.keys(partialStateUpdate).length > 0) {
+              await this.updateState(partialStateUpdate);
+            }
+          } catch (error) {
+            await this.updateState({ error });
+          }
+        },
+        onError: async (error) => {
+          await this.updateState({ error });
+        }
+      },
+      {
+        signal: abortSignal,
+        tables,
+        throttleMs: watchOptions.throttleMs,
+        triggerImmediate: true // used to emit the initial state
+      }
+    );
+  }
+}

package/src/client/watched/processors/OnChangeQueryProcessor.ts

@@ -0,0 +1,122 @@
+import { WatchCompatibleQuery, WatchedQuery, WatchedQueryOptions } from '../WatchedQuery.js';
+import {
+  AbstractQueryProcessor,
+  AbstractQueryProcessorOptions,
+  LinkQueryOptions,
+  MutableWatchedQueryState
+} from './AbstractQueryProcessor.js';
+import { WatchedQueryComparator } from './comparators.js';
+
+/**
+ * Settings for {@link WatchedQuery} instances created via {@link Query#watch}.
+ */
+export interface WatchedQuerySettings<DataType> extends WatchedQueryOptions {
+  query: WatchCompatibleQuery<DataType>;
+}
+
+/**
+ * {@link WatchedQuery} returned from {@link Query#watch}.
+ */
+export type StandardWatchedQuery<DataType> = WatchedQuery<DataType, WatchedQuerySettings<DataType>>;
+
+/**
+ * @internal
+ */
+export interface OnChangeQueryProcessorOptions<Data>
+  extends AbstractQueryProcessorOptions<Data, WatchedQuerySettings<Data>> {
+  comparator?: WatchedQueryComparator<Data>;
+}
+
+/**
+ * Uses the PowerSync onChange event to trigger watched queries.
+ * Results are emitted on every change of the relevant tables.
+ * @internal
+ */
+export class OnChangeQueryProcessor<Data> extends AbstractQueryProcessor<Data, WatchedQuerySettings<Data>> {
+  constructor(protected options: OnChangeQueryProcessorOptions<Data>) {
+    super(options);
+  }
+
+  /**
+   * @returns If the sets are equal
+   */
+  protected checkEquality(current: Data, previous: Data): boolean {
+    // Use the provided comparator if available. Assume values are unique if not available.
+    return this.options.comparator?.checkEquality?.(current, previous) ?? false;
+  }
+
+  protected async linkQuery(options: LinkQueryOptions<Data>): Promise<void> {
+    const { db, watchOptions } = this.options;
+    const { abortSignal } = options;
+
+    const compiledQuery = watchOptions.query.compile();
+    const tables = await db.resolveTables(compiledQuery.sql, compiledQuery.parameters as any[], {
+      tables: options.settings.triggerOnTables
+    });
+
+    db.onChangeWithCallback(
+      {
+        onChange: async () => {
+          if (this.closed || abortSignal.aborted) {
+            return;
+          }
+          // This fires for each change of the relevant tables
+          try {
+            if (this.reportFetching && !this.state.isFetching) {
+              await this.updateState({ isFetching: true });
+            }
+
+            const partialStateUpdate: Partial<MutableWatchedQueryState<Data>> & { data?: Data } = {};
+
+            // Always run the query if an underlying table has changed
+            const result = await watchOptions.query.execute({
+              sql: compiledQuery.sql,
+              // Allows casting from ReadOnlyArray[unknown] to Array<unknown>
+              // This allows simpler compatibility with PowerSync queries
+              parameters: [...compiledQuery.parameters],
+              db: this.options.db
+            });
+
+            if (abortSignal.aborted) {
+              return;
+            }
+
+            if (this.reportFetching) {
+              partialStateUpdate.isFetching = false;
+            }
+
+            if (this.state.isLoading) {
+              partialStateUpdate.isLoading = false;
+            }
+
+            // Check if the result has changed
+            if (!this.checkEquality(result, this.state.data)) {
+              Object.assign(partialStateUpdate, {
+                data: result
+              });
+            }
+
+            if (this.state.error) {
+              partialStateUpdate.error = null;
+            }
+
+            if (Object.keys(partialStateUpdate).length > 0) {
+              await this.updateState(partialStateUpdate);
+            }
+          } catch (error) {
+            await this.updateState({ error });
+          }
+        },
+        onError: async (error) => {
+          await this.updateState({ error });
+        }
+      },
+      {
+        signal: abortSignal,
+        tables,
+        throttleMs: watchOptions.throttleMs,
+        triggerImmediate: true // used to emit the initial state
+      }
+    );
+  }
+}

package/src/client/watched/processors/comparators.ts

@@ -0,0 +1,57 @@
+/**
+ * A basic comparator for incrementally watched queries. This performs a single comparison which
+ * determines if the result set has changed. The {@link WatchedQuery} will only emit the new result
+ * if a change has been detected.
+ */
+export interface WatchedQueryComparator<Data> {
+  checkEquality: (current: Data, previous: Data) => boolean;
+}
+
+/**
+ * Options for {@link ArrayComparator}
+ */
+export type ArrayComparatorOptions<ItemType> = {
+  /**
+   * Returns a string to uniquely identify an item in the array.
+   */
+  compareBy: (item: ItemType) => string;
+};
+
+/**
+ * An efficient comparator for {@link WatchedQuery} created with {@link Query#watch}. This has the ability to determine if a query
+ * result has changes without necessarily processing all items in the result.
+ */
+export class ArrayComparator<ItemType> implements WatchedQueryComparator<ItemType[]> {
+  constructor(protected options: ArrayComparatorOptions<ItemType>) {}
+
+  checkEquality(current: ItemType[], previous: ItemType[]) {
+    if (current.length === 0 && previous.length === 0) {
+      return true;
+    }
+
+    if (current.length !== previous.length) {
+      return false;
+    }
+
+    const { compareBy } = this.options;
+
+    // At this point the lengths are equal
+    for (let i = 0; i < current.length; i++) {
+      const currentItem = compareBy(current[i]);
+      const previousItem = compareBy(previous[i]);
+
+      if (currentItem !== previousItem) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+}
+
+/**
+ * Watched query comparator that always reports changed result sets.
+ */
+export const FalsyComparator: WatchedQueryComparator<unknown> = {
+  checkEquality: () => false // Default comparator that always returns false
+};

package/src/db/DBAdapter.ts

@@ -0,0 +1,134 @@
+/**
+ * Set of generic interfaces to allow PowerSync compatibility with
+ * different SQLite DB implementations.
+ */
+
+import { BaseListener, BaseObserverInterface } from '../utils/BaseObserver.js';
+
+/**
+ * TODO most of these types could be exported to a common `types` package
+ * which is used by the DB adapter libraries as well.
+ */
+
+/**
+ * Object returned by SQL Query executions.
+ */
+export type QueryResult = {
+  /** Represents the auto-generated row id if applicable. */
+  insertId?: number;
+  /** Number of affected rows if result of a update query. */
+  rowsAffected: number;
+  /** if status is undefined or 0 this object will contain the query results */
+  rows?: {
+    /** Raw array with all dataset */
+    _array: any[];
+    /** The length of the dataset */
+    length: number;
+    /** A convenience function to acess the index based the row object
+     * @param idx the row index
+     * @returns the row structure identified by column names
+     */
+    item: (idx: number) => any;
+  };
+};
+
+export interface DBGetUtils {
+  /** Execute a read-only query and return results. */
+  getAll<T>(sql: string, parameters?: any[]): Promise<T[]>;
+  /** Execute a read-only query and return the first result, or null if the ResultSet is empty. */
+  getOptional<T>(sql: string, parameters?: any[]): Promise<T | null>;
+  /** Execute a read-only query and return the first result, error if the ResultSet is empty. */
+  get<T>(sql: string, parameters?: any[]): Promise<T>;
+}
+
+export interface LockContext extends DBGetUtils {
+  /** Execute a single write statement. */
+  execute: (query: string, params?: any[] | undefined) => Promise<QueryResult>;
+  /**
+   * Execute a single write statement and return raw results.
+   * Unlike `execute`, which returns an object with structured key-value pairs,
+   * `executeRaw` returns a nested array of raw values, where each row is
+   * represented as an array of column values without field names.
+   *
+   * Example result:
+   *
+   * ```[ [ '1', 'list 1', '33', 'Post content', '1' ] ]```
+   *
+   * Where as `execute`'s `rows._array` would have been:
+   *
+   * ```[ { id: '33', name: 'list 1', content: 'Post content', list_id: '1' } ]```
+   */
+  executeRaw: (query: string, params?: any[] | undefined) => Promise<any[][]>;
+}
+
+export interface Transaction extends LockContext {
+  /** Commit multiple changes to the local DB using the Transaction context. */
+  commit: () => Promise<QueryResult>;
+  /** Roll back multiple attempted changes using the Transaction context. */
+  rollback: () => Promise<QueryResult>;
+}
+
+/**
+ * Update table operation numbers from SQLite
+ */
+export enum RowUpdateType {
+  SQLITE_INSERT = 18,
+  SQLITE_DELETE = 9,
+  SQLITE_UPDATE = 23
+}
+export interface TableUpdateOperation {
+  opType: RowUpdateType;
+  rowId: number;
+}
+/**
+ * Notification of an update to one or more tables, for the purpose of realtime change notifications.
+ */
+export interface UpdateNotification extends TableUpdateOperation {
+  table: string;
+}
+
+export interface BatchedUpdateNotification {
+  rawUpdates: UpdateNotification[];
+  tables: string[];
+  groupedUpdates: Record<string, TableUpdateOperation[]>;
+}
+
+export interface DBAdapterListener extends BaseListener {
+  /**
+   * Listener for table updates.
+   * Allows for single table updates in order to maintain API compatibility
+   * without the need for a major version bump
+   * The DB adapter can also batch update notifications if supported.
+   */
+  tablesUpdated: (updateNotification: BatchedUpdateNotification | UpdateNotification) => void;
+}
+
+export interface DBLockOptions {
+  timeoutMs?: number;
+}
+
+export interface DBAdapter extends BaseObserverInterface<DBAdapterListener>, DBGetUtils {
+  close: () => void | Promise<void>;
+  execute: (query: string, params?: any[]) => Promise<QueryResult>;
+  executeRaw: (query: string, params?: any[]) => Promise<any[][]>;
+  executeBatch: (query: string, params?: any[][]) => Promise<QueryResult>;
+  name: string;
+  readLock: <T>(fn: (tx: LockContext) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+  readTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+  writeLock: <T>(fn: (tx: LockContext) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+  writeTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+  /**
+   * This method refreshes the schema information across all connections. This is for advanced use cases, and should generally not be needed.
+   */
+  refreshSchema: () => Promise<void>;
+}
+
+export function isBatchedUpdateNotification(
+  update: BatchedUpdateNotification | UpdateNotification
+): update is BatchedUpdateNotification {
+  return 'tables' in update;
+}
+
+export function extractTableUpdates(update: BatchedUpdateNotification | UpdateNotification) {
+  return isBatchedUpdateNotification(update) ? update.tables : [update.table];
+}