@powersync/common 1.41.0 → 1.42.0

package/src/client/compilableQueryWatch.ts

@@ -0,0 +1,55 @@
+import { CompilableQuery } from './../types/types.js';
+import { AbstractPowerSyncDatabase, SQLWatchOptions } from './AbstractPowerSyncDatabase.js';
+import { runOnSchemaChange } from './runOnSchemaChange.js';
+
+export interface CompilableQueryWatchHandler<T> {
+  onResult: (results: T[]) => void;
+  onError?: (error: Error) => void;
+}
+
+export function compilableQueryWatch<T>(
+  db: AbstractPowerSyncDatabase,
+  query: CompilableQuery<T>,
+  handler: CompilableQueryWatchHandler<T>,
+  options?: SQLWatchOptions
+): void {
+  const { onResult, onError = (e: Error) => {} } = handler ?? {};
+  if (!onResult) {
+    throw new Error('onResult is required');
+  }
+
+  const watchQuery = async (abortSignal: AbortSignal) => {
+    try {
+      const toSql = query.compile();
+      const resolvedTables = await db.resolveTables(toSql.sql, toSql.parameters as [], options);
+
+      // Fetch initial data
+      const result = await query.execute();
+      onResult(result);
+
+      db.onChangeWithCallback(
+        {
+          onChange: async () => {
+            try {
+              const result = await query.execute();
+              onResult(result);
+            } catch (error: any) {
+              onError(error);
+            }
+          },
+          onError
+        },
+        {
+          ...(options ?? {}),
+          tables: resolvedTables,
+          // Override the abort signal since we intercept it
+          signal: abortSignal
+        }
+      );
+    } catch (error: any) {
+      onError(error);
+    }
+  };
+
+  runOnSchemaChange(watchQuery, db, options);
+}

package/src/client/connection/PowerSyncBackendConnector.ts

@@ -0,0 +1,25 @@
+import { PowerSyncCredentials } from './PowerSyncCredentials.js';
+import type { AbstractPowerSyncDatabase } from '../AbstractPowerSyncDatabase.js';
+
+export interface PowerSyncBackendConnector {
+  /** Allows the PowerSync client to retrieve an authentication token from your backend
+   * which is used to authenticate against the PowerSync service.
+   *
+   * This should always fetch a fresh set of credentials - don't use cached
+   * values.
+   *
+   * Return null if the user is not signed in. Throw an error if credentials
+   * cannot be fetched due to a network error or other temporary error.
+   *
+   * This token is kept for the duration of a sync connection.
+   */
+  fetchCredentials: () => Promise<PowerSyncCredentials | null>;
+
+  /** Upload local changes to the app backend.
+   *
+   * Use {@link AbstractPowerSyncDatabase.getCrudBatch} to get a batch of changes to upload.
+   *
+   * Any thrown errors will result in a retry after the configured wait period (default: 5 seconds).
+   */
+  uploadData: (database: AbstractPowerSyncDatabase) => Promise<void>;
+}

package/src/client/connection/PowerSyncCredentials.ts

@@ -0,0 +1,5 @@
+export interface PowerSyncCredentials {
+  endpoint: string;
+  token: string;
+  expiresAt?: Date;
+}

package/src/client/constants.ts

@@ -0,0 +1 @@
+export const MAX_OP_ID = '9223372036854775807';

package/src/client/runOnSchemaChange.ts

@@ -0,0 +1,31 @@
+import { AbstractPowerSyncDatabase, SQLWatchOptions } from './AbstractPowerSyncDatabase.js';
+
+export function runOnSchemaChange(
+  callback: (signal: AbortSignal) => void,
+  db: AbstractPowerSyncDatabase,
+  options?: SQLWatchOptions
+): void {
+  const triggerWatchedQuery = () => {
+    const abortController = new AbortController();
+    let disposeSchemaListener: (() => void) | null = null;
+    const stopWatching = () => {
+      abortController.abort('Abort triggered');
+      disposeSchemaListener?.();
+      disposeSchemaListener = null;
+      // Stop listening to upstream abort for this watch
+      options?.signal?.removeEventListener('abort', stopWatching);
+    };
+
+    options?.signal?.addEventListener('abort', stopWatching);
+    disposeSchemaListener = db.registerListener({
+      schemaChanged: async () => {
+        stopWatching();
+        // Re trigger the watched query (recursively), setTimeout ensures that we don't modify the list of listeners while iterating through them
+        setTimeout(() => triggerWatchedQuery(), 0);
+      }
+    });
+    callback(abortController.signal);
+  };
+
+  triggerWatchedQuery();
+}

package/src/client/sync/bucket/BucketStorageAdapter.ts

@@ -0,0 +1,118 @@
+import { BaseListener, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
+import { CrudBatch } from './CrudBatch.js';
+import { CrudEntry, OpId } from './CrudEntry.js';
+import { SyncDataBatch } from './SyncDataBatch.js';
+
+export interface BucketDescription {
+  name: string;
+  priority: number;
+}
+
+export interface Checkpoint {
+  last_op_id: OpId;
+  buckets: BucketChecksum[];
+  write_checkpoint?: string;
+  streams?: any[];
+}
+
+export interface BucketState {
+  bucket: string;
+  op_id: string;
+}
+
+export interface ChecksumCache {
+  checksums: Map<string, { checksum: BucketChecksum; last_op_id: OpId }>;
+  lastOpId: OpId;
+}
+
+export interface SyncLocalDatabaseResult {
+  ready: boolean;
+  checkpointValid: boolean;
+  checkpointFailures?: string[];
+}
+
+export type SavedProgress = {
+  atLast: number;
+  sinceLast: number;
+};
+
+export type BucketOperationProgress = Record<string, SavedProgress>;
+
+export interface BucketChecksum {
+  bucket: string;
+  priority?: number;
+  /**
+   * 32-bit unsigned hash.
+   */
+  checksum: number;
+
+  /**
+   * Count of operations - informational only.
+   */
+  count?: number;
+  /**
+   * The JavaScript client does not use this field, which is why it's defined to be `any`. We rely on the structure of
+   * this interface to pass custom `BucketChecksum`s to the Rust client in unit tests, which so all fields need to be
+   * present.
+   */
+  subscriptions?: any;
+}
+
+export enum PSInternalTable {
+  DATA = 'ps_data',
+  CRUD = 'ps_crud',
+  BUCKETS = 'ps_buckets',
+  OPLOG = 'ps_oplog',
+  UNTYPED = 'ps_untyped'
+}
+
+export enum PowerSyncControlCommand {
+  PROCESS_TEXT_LINE = 'line_text',
+  PROCESS_BSON_LINE = 'line_binary',
+  STOP = 'stop',
+  START = 'start',
+  NOTIFY_TOKEN_REFRESHED = 'refreshed_token',
+  NOTIFY_CRUD_UPLOAD_COMPLETED = 'completed_upload',
+  UPDATE_SUBSCRIPTIONS = 'update_subscriptions'
+}
+
+export interface BucketStorageListener extends BaseListener {
+  crudUpdate: () => void;
+}
+
+export interface BucketStorageAdapter extends BaseObserverInterface<BucketStorageListener>, Disposable {
+  init(): Promise<void>;
+  saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
+  removeBuckets(buckets: string[]): Promise<void>;
+  setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
+
+  startSession(): void;
+
+  getBucketStates(): Promise<BucketState[]>;
+  getBucketOperationProgress(): Promise<BucketOperationProgress>;
+  hasMigratedSubkeys(): Promise<boolean>;
+  migrateToFixedSubkeys(): Promise<void>;
+
+  syncLocalDatabase(
+    checkpoint: Checkpoint,
+    priority?: number
+  ): Promise<{ checkpointValid: boolean; ready: boolean; failures?: any[] }>;
+
+  nextCrudItem(): Promise<CrudEntry | undefined>;
+  hasCrud(): Promise<boolean>;
+  getCrudBatch(limit?: number): Promise<CrudBatch | null>;
+
+  hasCompletedSync(): Promise<boolean>;
+  updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
+  getMaxOpId(): string;
+
+  /**
+   * Get an unique client id.
+   */
+  getClientId(): Promise<string>;
+
+  /**
+   * Invokes the `powersync_control` function for the sync client.
+   */
+  control(op: PowerSyncControlCommand, payload: string | Uint8Array | null): Promise<string>;
+}

package/src/client/sync/bucket/CrudBatch.ts

@@ -0,0 +1,21 @@
+import { CrudEntry } from './CrudEntry.js';
+
+/**
+ * A batch of client-side changes.
+ */
+export class CrudBatch {
+  constructor(
+    /**
+     * List of client-side changes.
+     */
+    public crud: CrudEntry[],
+    /**
+     * true if there are more changes in the local queue.
+     */
+    public haveMore: boolean,
+    /**
+     * Call to remove the changes from the local queue, once successfully uploaded.
+     */
+    public complete: (writeCheckpoint?: string) => Promise<void>
+  ) {}
+}

package/src/client/sync/bucket/CrudEntry.ts

@@ -0,0 +1,172 @@
+/**
+ * 64-bit unsigned integer stored as a string in base-10.
+ *
+ * Not sortable as a string.
+ */
+export type OpId = string;
+
+/**
+ * Type of local change.
+ */
+export enum UpdateType {
+  /** Insert or replace existing row. All non-null columns are included in the data. Generated by INSERT statements. */
+  PUT = 'PUT',
+  /** Update existing row. Contains the id, and value of each changed column. Generated by UPDATE statements. */
+  PATCH = 'PATCH',
+  /** Delete existing row. Contains the id. Generated by DELETE statements. */
+  DELETE = 'DELETE'
+}
+
+export type CrudEntryJSON = {
+  id: string;
+  data: string;
+  tx_id?: number;
+};
+
+type CrudEntryDataJSON = {
+  data: Record<string, any>;
+  old?: Record<string, any>;
+  op: UpdateType;
+  type: string;
+  id: string;
+  metadata?: string;
+};
+
+/**
+ * The output JSON seems to be a third type of JSON, not the same as the input JSON.
+ */
+type CrudEntryOutputJSON = {
+  op_id: number;
+  op: UpdateType;
+  type: string;
+  id: string;
+  tx_id?: number;
+  data?: Record<string, any>;
+  old?: Record<string, any>;
+  metadata?: string;
+};
+
+/**
+ * A single client-side change.
+ */
+export class CrudEntry {
+  /**
+   * Auto-incrementing client-side id.
+   */
+  clientId: number;
+  /**
+   * ID of the changed row.
+   */
+  id: string;
+  /**
+   * Type of change.
+   */
+  op: UpdateType;
+  /**
+   * Data associated with the change.
+   */
+  opData?: Record<string, any>;
+
+  /**
+   * For tables where the `trackPreviousValues` option has been enabled, this tracks previous values for
+   * `UPDATE` and `DELETE` statements.
+   */
+  previousValues?: Record<string, any>;
+
+  /**
+   * Table that contained the change.
+   */
+  table: string;
+  /**
+   * Auto-incrementing transaction id. This is the same for all operations within the same transaction.
+   */
+  transactionId?: number;
+
+  /**
+   * Client-side metadata attached with this write.
+   *
+   * This field is only available when the `trackMetadata` option was set to `true` when creating a table
+   * and the insert or update statement set the `_metadata` column.
+   */
+  metadata?: string;
+
+  static fromRow(dbRow: CrudEntryJSON) {
+    const data: CrudEntryDataJSON = JSON.parse(dbRow.data);
+    return new CrudEntry(
+      parseInt(dbRow.id),
+      data.op,
+      data.type,
+      data.id,
+      dbRow.tx_id,
+      data.data,
+      data.old,
+      data.metadata
+    );
+  }
+
+  constructor(
+    clientId: number,
+    op: UpdateType,
+    table: string,
+    id: string,
+    transactionId?: number,
+    opData?: Record<string, any>,
+    previousValues?: Record<string, any>,
+    metadata?: string
+  ) {
+    this.clientId = clientId;
+    this.id = id;
+    this.op = op;
+    this.opData = opData;
+    this.table = table;
+    this.transactionId = transactionId;
+    this.previousValues = previousValues;
+    this.metadata = metadata;
+  }
+
+  /**
+   * Converts the change to JSON format.
+   */
+  toJSON(): CrudEntryOutputJSON {
+    return {
+      op_id: this.clientId,
+      op: this.op,
+      type: this.table,
+      id: this.id,
+      tx_id: this.transactionId,
+      data: this.opData,
+      old: this.previousValues,
+      metadata: this.metadata
+    };
+  }
+
+  equals(entry: CrudEntry) {
+    return JSON.stringify(this.toComparisonArray()) == JSON.stringify(entry.toComparisonArray());
+  }
+
+  /**
+   * The hash code for this object.
+   * @deprecated This should not be necessary in the JS SDK.
+   * Use the  @see CrudEntry#equals method instead.
+   * TODO remove in the next major release.
+   */
+  hashCode() {
+    return JSON.stringify(this.toComparisonArray());
+  }
+
+  /**
+   * Generates an array for use in deep comparison operations
+   */
+  toComparisonArray() {
+    return [
+      this.transactionId,
+      this.clientId,
+      this.op,
+      this.table,
+      this.id,
+      this.opData,
+      this.previousValues,
+      this.metadata
+    ];
+  }
+}

package/src/client/sync/bucket/CrudTransaction.ts

@@ -0,0 +1,21 @@
+import { CrudBatch } from './CrudBatch.js';
+import { CrudEntry } from './CrudEntry.js';
+
+export class CrudTransaction extends CrudBatch {
+  constructor(
+    /**
+     * List of client-side changes.
+     */
+    public crud: CrudEntry[],
+    /**
+     * Call to remove the changes from the local queue, once successfully uploaded.
+     */
+    public complete: (checkpoint?: string) => Promise<void>,
+    /**
+     * If null, this contains a list of changes recorded without an explicit transaction associated.
+     */
+    public transactionId?: number
+  ) {
+    super(crud, false, complete);
+  }
+}

package/src/client/sync/bucket/OpType.ts

@@ -0,0 +1,23 @@
+export enum OpTypeEnum {
+  CLEAR = 1,
+  MOVE = 2,
+  PUT = 3,
+  REMOVE = 4
+}
+
+export type OpTypeJSON = string;
+
+/**
+ * Used internally for sync buckets.
+ */
+export class OpType {
+  static fromJSON(jsonValue: OpTypeJSON) {
+    return new OpType(OpTypeEnum[jsonValue]);
+  }
+
+  constructor(public value: OpTypeEnum) {}
+
+  toJSON() {
+    return Object.entries(OpTypeEnum).find(([, value]) => value === this.value)![0];
+  }
+}

package/src/client/sync/bucket/OplogEntry.ts

@@ -0,0 +1,50 @@
+import { OpId } from './CrudEntry.js';
+import { OpType, OpTypeJSON } from './OpType.js';
+
+export interface OplogEntryJSON {
+  checksum: number;
+  data?: string;
+  object_id?: string;
+  object_type?: string;
+  op_id: string;
+  op: OpTypeJSON;
+  subkey?: string;
+}
+
+export class OplogEntry {
+  static fromRow(row: OplogEntryJSON) {
+    return new OplogEntry(
+      row.op_id,
+      OpType.fromJSON(row.op),
+      row.checksum,
+      row.subkey,
+      row.object_type,
+      row.object_id,
+      row.data
+    );
+  }
+
+  constructor(
+    public op_id: OpId,
+    public op: OpType,
+    public checksum: number,
+    public subkey?: string,
+    public object_type?: string,
+    public object_id?: string,
+    public data?: string
+  ) {}
+
+  toJSON(fixedKeyEncoding = false): OplogEntryJSON {
+    return {
+      op_id: this.op_id,
+      op: this.op.toJSON(),
+      object_type: this.object_type,
+      object_id: this.object_id,
+      checksum: this.checksum,
+      data: this.data,
+      // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
+      // but we need to migrate gradually to not break existing databases.
+      subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
+    };
+  }
+}