@powersync/common 0.0.0-dev-20260311103504 → 0.0.0-dev-20260503073249

package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts

@@ -4,7 +4,6 @@ import { InternalProgressInformation } from '../../../db/crud/SyncProgress.js';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
-import { DataStream } from '../../../utils/DataStream.js';
 import { throttleLeadingTrailing } from '../../../utils/async.js';
 import {
   BucketChecksum,
@@ -19,7 +18,6 @@ import { AbstractRemote, FetchStrategy, SyncStreamOptions } from './AbstractRemo
 import { EstablishSyncStream, Instruction, coreStatusToJs } from './core-instruction.js';
 import {
   BucketRequest,
-  CrudUploadNotification,
   StreamingSyncLine,
   StreamingSyncLineOrCrudUploadComplete,
   StreamingSyncRequestParameterType,
@@ -30,6 +28,8 @@ import {
   isStreamingSyncCheckpointPartiallyComplete,
   isStreamingSyncData
 } from './streaming-sync-types.js';
+import { injectable, InjectableIterator, map, SimpleAsyncIterator } from '../../../utils/stream_transform.js';
+import type { BSON } from 'bson';
 
 export enum LockType {
   CRUD = 'crud',
@@ -209,6 +209,7 @@ export interface StreamingSyncImplementation
   waitForStatus(status: SyncStatusOptions): Promise<void>;
   waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
   updateSubscriptions(subscriptions: SubscribedStream[]): void;
+  markConnectionMayHaveChanged(): void;
 }
 
 export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
@@ -256,6 +257,7 @@ export abstract class AbstractStreamingSyncImplementation
   protected streamingSyncPromise?: Promise<void>;
   protected logger: ILogger;
   private activeStreams: SubscribedStream[];
+  private connectionMayHaveChanged = false;
 
   private isUploadingCrud: boolean = false;
   private notifyCompletedUploads?: () => void;
@@ -572,6 +574,10 @@ The next upload iteration will be delayed.`);
           this.logger.warn(ex);
           shouldDelayRetry = false;
           // A disconnect was requested, we should not delay since there is no explicit retry
+        } else if (this.connectionMayHaveChanged && (ex as Error).message?.indexOf('No iteration is active') >= 0) {
+          this.connectionMayHaveChanged = false;
+          this.logger.info('Sync error after changed connection, retrying immediately');
+          shouldDelayRetry = false;
         } else {
           this.logger.error(ex);
         }
@@ -607,6 +613,17 @@ The next upload iteration will be delayed.`);
     this.updateSyncStatus({ connected: false, connecting: false });
   }
 
+  markConnectionMayHaveChanged() {
+    // By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
+    // an active sync iteration on the connection in use.
+    this.connectionMayHaveChanged = true;
+
+    // This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
+    // make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
+    // iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
+    this.handleActiveStreamsChange?.();
+  }
+
   private async collectLocalBucketState(): Promise<[BucketRequest[], Map<string, BucketDescription | null>]> {
     const bucketEntries = await this.options.adapter.getBucketStates();
     const req: BucketRequest[] = bucketEntries.map((entry) => ({
@@ -682,6 +699,27 @@ The next upload iteration will be delayed.`);
     });
   }
 
+  private async receiveSyncLines(data: {
+    options: SyncStreamOptions;
+    connection: RequiredPowerSyncConnectionOptions;
+    bson?: typeof BSON;
+  }): Promise<SimpleAsyncIterator<Uint8Array | string>> {
+    const { options, connection, bson } = data;
+    const remote = this.options.remote;
+
+    if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+      return await remote.fetchStream(options);
+    } else {
+      return await this.options.remote.socketStreamRaw(
+        {
+          ...options,
+          ...{ fetchStrategy: connection.fetchStrategy }
+        },
+        bson
+      );
+    }
+  }
+
   private async legacyStreamingSyncIteration(signal: AbortSignal, resolvedOptions: RequiredPowerSyncConnectionOptions) {
     const rawTables = resolvedOptions.serializedSchema?.raw_tables;
     if (rawTables != null && rawTables.length) {
@@ -717,46 +755,31 @@ The next upload iteration will be delayed.`);
       }
     };
 
-    let stream: DataStream<StreamingSyncLineOrCrudUploadComplete>;
-    if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-      stream = await this.options.remote.postStreamRaw(syncOptions, (line: string | CrudUploadNotification) => {
+    const bson = await this.options.remote.getBSON();
+    const source = await this.receiveSyncLines({
+      options: syncOptions,
+      connection: resolvedOptions,
+      bson
+    });
+    const stream: InjectableIterator<StreamingSyncLineOrCrudUploadComplete> = injectable(
+      map(source, (line) => {
         if (typeof line == 'string') {
           return JSON.parse(line) as StreamingSyncLine;
         } else {
-          // Directly enqueued by us
-          return line;
+          return bson.deserialize(line) as StreamingSyncLine;
         }
-      });
-    } else {
-      const bson = await this.options.remote.getBSON();
-      stream = await this.options.remote.socketStreamRaw(
-        {
-          ...syncOptions,
-          ...{ fetchStrategy: resolvedOptions.fetchStrategy }
-        },
-        (payload: Uint8Array | CrudUploadNotification) => {
-          if (payload instanceof Uint8Array) {
-            return bson.deserialize(payload) as StreamingSyncLine;
-          } else {
-            // Directly enqueued by us
-            return payload;
-          }
-        },
-        bson
-      );
-    }
+      })
+    );
 
     this.logger.debug('Stream established. Processing events');
 
     this.notifyCompletedUploads = () => {
-      if (!stream.closed) {
-        stream.enqueueData({ crud_upload_completed: null });
-      }
+      stream.inject({ crud_upload_completed: null });
     };
 
-    while (!stream.closed) {
-      const line = await stream.read();
-      if (!line) {
+    while (true) {
+      const { value: line, done } = await stream.next();
+      if (done) {
         // The stream has closed while waiting
         return;
       }
@@ -942,6 +965,11 @@ The next upload iteration will be delayed.`);
     const syncImplementation = this;
     const adapter = this.options.adapter;
     const remote = this.options.remote;
+    const controller = new AbortController();
+    const abort = () => {
+      return controller.abort(signal.reason);
+    };
+    signal.addEventListener('abort', abort);
     let receivingLines: Promise<void> | null = null;
     let hadSyncLine = false;
     let hideDisconnectOnRestart = false;
@@ -949,64 +977,53 @@ The next upload iteration will be delayed.`);
     if (signal.aborted) {
       throw new AbortOperation('Connection request has been aborted');
     }
-    const abortController = new AbortController();
-    signal.addEventListener('abort', () => abortController.abort());
 
     // Pending sync lines received from the service, as well as local events that trigger a powersync_control
     // invocation (local events include refreshed tokens and completed uploads).
     // This is a single data stream so that we can handle all control calls from a single place.
-    let controlInvocations: DataStream<EnqueuedCommand, Uint8Array | EnqueuedCommand> | null = null;
+    let controlInvocations: InjectableIterator<EnqueuedCommand> | null = null;
 
     async function connect(instr: EstablishSyncStream) {
       const syncOptions: SyncStreamOptions = {
         path: '/sync/stream',
-        abortSignal: abortController.signal,
+        abortSignal: controller.signal,
         data: instr.request
       };
 
-      if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-        controlInvocations = await remote.postStreamRaw(syncOptions, (line: string | EnqueuedCommand) => {
-          if (typeof line == 'string') {
-            return {
-              command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
-              payload: line
-            };
-          } else {
-            // Directly enqueued by us
-            return line;
-          }
-        });
-      } else {
-        controlInvocations = await remote.socketStreamRaw(
-          {
-            ...syncOptions,
-            fetchStrategy: resolvedOptions.fetchStrategy
-          },
-          (payload: Uint8Array | EnqueuedCommand) => {
-            if (payload instanceof Uint8Array) {
+      controlInvocations = injectable(
+        map(
+          await syncImplementation.receiveSyncLines({
+            options: syncOptions,
+            connection: resolvedOptions
+          }),
+          (line) => {
+            if (typeof line == 'string') {
               return {
-                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
-                payload: payload
+                command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
+                payload: line
               };
             } else {
-              // Directly enqueued by us
-              return payload;
+              return {
+                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
+                payload: line
+              };
             }
           }
-        );
-      }
+        )
+      );
 
       // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
       // we're already connected here and can report that.
       syncImplementation.updateSyncStatus({ connected: true });
 
       try {
-        while (!controlInvocations.closed) {
-          const line = await controlInvocations.read();
-          if (line == null) {
-            return;
+        while (true) {
+          let event = await controlInvocations.next();
+          if (event.done) {
+            break;
           }
 
+          const line = event.value;
           await control(line.command, line.payload);
 
           if (!hadSyncLine) {
@@ -1015,12 +1032,8 @@ The next upload iteration will be delayed.`);
           }
         }
       } finally {
-        const activeInstructions = controlInvocations;
-        // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
-        // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
-        // to null.
-        controlInvocations = null;
-        await activeInstructions.close();
+        abort();
+        signal.removeEventListener('abort', abort);
       }
     }
 
@@ -1038,6 +1051,10 @@ The next upload iteration will be delayed.`);
         rawResponse
       );
 
+      if (op != PowerSyncControlCommand.STOP) {
+        // Evidently we have a working connection here, otherwise powersync_control would have failed.
+        syncImplementation.connectionMayHaveChanged = false;
+      }
       await handleInstructions(JSON.parse(rawResponse));
     }
 
@@ -1072,7 +1089,7 @@ The next upload iteration will be delayed.`);
           // Restart iteration after the credentials have been refreshed.
           remote.fetchCredentials().then(
             (_) => {
-              controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
+              controlInvocations?.inject({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
             },
             (err) => {
               syncImplementation.logger.warn('Could not prefetch credentials', err);
@@ -1080,7 +1097,7 @@ The next upload iteration will be delayed.`);
           );
         }
       } else if ('CloseSyncStream' in instruction) {
-        abortController.abort();
+        controller.abort();
         hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
       } else if ('FlushFileSystem' in instruction) {
         // Not necessary on JS platforms.
@@ -1113,17 +1130,13 @@ The next upload iteration will be delayed.`);
       await control(PowerSyncControlCommand.START, JSON.stringify(options));
 
       this.notifyCompletedUploads = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
-        }
+        controlInvocations?.inject({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
       };
       this.handleActiveStreamsChange = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({
-            command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
-            payload: JSON.stringify(this.activeStreams)
-          });
-        }
+        controlInvocations?.inject({
+          command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
+          payload: JSON.stringify(this.activeStreams)
+        });
       };
       await receivingLines;
     } finally {

package/src/db/DBAdapter.ts

@@ -16,7 +16,13 @@ import { BaseListener, BaseObserverInterface } from '../utils/BaseObserver.js';
 export type QueryResult = {
   /** Represents the auto-generated row id if applicable. */
   insertId?: number;
-  /** Number of affected rows if result of a update query. */
+  /**
+   * Number of affected rows reported by SQLite for a write query.
+   *
+   * When using the default client-side [JSON-based view system](https://docs.powersync.com/architecture/client-architecture#client-side-schema-and-sqlite-database-structure),
+   * `rowsAffected` may be `0` for successful `UPDATE` and `DELETE` statements.
+   * Use a `RETURNING` clause and inspect `rows` when you need to confirm which rows changed.
+   */
   rowsAffected: number;
   /** if status is undefined or 0 this object will contain the query results */
   rows?: {
@@ -41,7 +47,7 @@ export interface DBGetUtils {
   get<T>(sql: string, parameters?: any[]): Promise<T>;
 }
 
-export interface LockContext extends DBGetUtils {
+export interface SqlExecutor {
   /** Execute a single write statement. */
   execute: (query: string, params?: any[] | undefined) => Promise<QueryResult>;
   /**
@@ -59,6 +65,61 @@ export interface LockContext extends DBGetUtils {
    * ```[ { id: '33', name: 'list 1', content: 'Post content', list_id: '1' } ]```
    */
   executeRaw: (query: string, params?: any[] | undefined) => Promise<any[][]>;
+
+  executeBatch: (query: string, params?: any[][]) => Promise<QueryResult>;
+}
+
+export interface LockContext extends SqlExecutor, DBGetUtils {}
+
+/**
+ * Implements {@link DBGetUtils} on a {@link SqlRunner}.
+ */
+export function DBGetUtilsDefaultMixin<TBase extends new (...args: any[]) => Omit<SqlExecutor, 'executeBatch'>>(
+  Base: TBase
+) {
+  return class extends Base implements DBGetUtils, SqlExecutor {
+    async getAll<T>(sql: string, parameters?: any[]): Promise<T[]> {
+      const res = await this.execute(sql, parameters);
+      return res.rows?._array ?? [];
+    }
+
+    async getOptional<T>(sql: string, parameters?: any[]): Promise<T | null> {
+      const res = await this.execute(sql, parameters);
+      return res.rows?.item(0) ?? null;
+    }
+
+    async get<T>(sql: string, parameters?: any[]): Promise<T> {
+      const res = await this.execute(sql, parameters);
+      const first = res.rows?.item(0);
+      if (!first) {
+        throw new Error('Result set is empty');
+      }
+      return first;
+    }
+
+    async executeBatch(query: string, params: any[][] = []): Promise<QueryResult> {
+      // If this context can run batch statements natively, use that.
+      // @ts-ignore
+      if (super.executeBatch) {
+        // @ts-ignore
+        return super.executeBatch(query, params);
+      }
+
+      // Emulate executeBatch by running statements individually.
+      let lastInsertId: number | undefined;
+      let rowsAffected = 0;
+      for (const set of params) {
+        const result = await this.execute(query, set);
+        lastInsertId = result.insertId;
+        rowsAffected += result.rowsAffected;
+      }
+
+      return {
+        rowsAffected,
+        insertId: lastInsertId
+      };
+    }
+  };
 }
 
 export interface Transaction extends LockContext {
@@ -107,22 +168,119 @@ export interface DBLockOptions {
   timeoutMs?: number;
 }
 
-export interface DBAdapter extends BaseObserverInterface<DBAdapterListener>, DBGetUtils {
-  close: () => void | Promise<void>;
-  execute: (query: string, params?: any[]) => Promise<QueryResult>;
-  executeRaw: (query: string, params?: any[]) => Promise<any[][]>;
-  executeBatch: (query: string, params?: any[][]) => Promise<QueryResult>;
+export interface ConnectionPool extends BaseObserverInterface<DBAdapterListener> {
   name: string;
+  close: () => void | Promise<void>;
   readLock: <T>(fn: (tx: LockContext) => Promise<T>, options?: DBLockOptions) => Promise<T>;
-  readTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
   writeLock: <T>(fn: (tx: LockContext) => Promise<T>, options?: DBLockOptions) => Promise<T>;
-  writeTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+
   /**
    * This method refreshes the schema information across all connections. This is for advanced use cases, and should generally not be needed.
    */
   refreshSchema: () => Promise<void>;
 }
 
+export interface DBAdapter extends ConnectionPool, SqlExecutor, DBGetUtils {
+  readTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+  writeTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+}
+
+/**
+ * A mixin to implement {@link DBAdapter} by delegating to {@link ConnectionPool.readLock} and
+ * {@link ConnectionPool.writeLock}.
+ */
+export function DBAdapterDefaultMixin<TBase extends new (...args: any[]) => ConnectionPool>(Base: TBase) {
+  return class extends Base implements DBAdapter {
+    readTransaction<T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions): Promise<T> {
+      return this.readLock((ctx) => TransactionImplementation.runWith(ctx, fn), options);
+    }
+
+    writeTransaction<T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions): Promise<T> {
+      return this.writeLock((ctx) => TransactionImplementation.runWith(ctx, fn), options);
+    }
+
+    getAll<T>(sql: string, parameters?: any[]): Promise<T[]> {
+      return this.readLock((ctx) => ctx.getAll(sql, parameters));
+    }
+
+    getOptional<T>(sql: string, parameters?: any[]): Promise<T | null> {
+      return this.readLock((ctx) => ctx.getOptional(sql, parameters));
+    }
+
+    get<T>(sql: string, parameters?: any[]): Promise<T> {
+      return this.readLock((ctx) => ctx.get(sql, parameters));
+    }
+
+    execute(query: string, params?: any[]): Promise<QueryResult> {
+      return this.writeLock((ctx) => ctx.execute(query, params));
+    }
+
+    executeRaw(query: string, params?: any[]): Promise<any[][]> {
+      return this.writeLock((ctx) => ctx.executeRaw(query, params));
+    }
+
+    executeBatch(query: string, params?: any[][]): Promise<QueryResult> {
+      return this.writeTransaction((tx) => tx.executeBatch(query, params));
+    }
+  };
+}
+
+class BaseTransaction implements SqlExecutor {
+  private finalized = false;
+
+  constructor(private inner: SqlExecutor) {}
+
+  async commit(): Promise<QueryResult> {
+    if (this.finalized) {
+      return { rowsAffected: 0 };
+    }
+    this.finalized = true;
+    return this.inner.execute('COMMIT');
+  }
+
+  async rollback(): Promise<QueryResult> {
+    if (this.finalized) {
+      return { rowsAffected: 0 };
+    }
+    this.finalized = true;
+    return this.inner.execute('ROLLBACK');
+  }
+
+  execute(query: string, params?: any[] | undefined): Promise<QueryResult> {
+    return this.inner.execute(query, params);
+  }
+
+  executeRaw(query: string, params?: any[] | undefined): Promise<any[][]> {
+    return this.inner.executeRaw(query, params);
+  }
+
+  executeBatch(query: string, params?: any[][]): Promise<QueryResult> {
+    return this.inner.executeBatch(query, params);
+  }
+}
+
+class TransactionImplementation extends DBGetUtilsDefaultMixin(BaseTransaction) {
+  static async runWith<T>(ctx: LockContext, fn: (tx: Transaction) => Promise<T>): Promise<T> {
+    let tx = new TransactionImplementation(ctx);
+
+    try {
+      await ctx.execute('BEGIN IMMEDIATE');
+
+      const result = await fn(tx);
+      await tx.commit();
+      return result;
+    } catch (ex) {
+      try {
+        await tx.rollback();
+      } catch (ex2) {
+        // In rare cases, a rollback may fail.
+        // Safe to ignore.
+      }
+      throw ex;
+    }
+  }
+}
+
 export function isBatchedUpdateNotification(
   update: BatchedUpdateNotification | UpdateNotification
 ): update is BatchedUpdateNotification {

package/src/db/crud/SyncStatus.ts

@@ -122,8 +122,6 @@ export class SyncStatus {
    *
    * This returns null when the database is currently being opened and we don't have reliable information about all
    * included streams yet.
-   *
-   * @experimental Sync streams are currently in alpha.
    */
   get syncStreams(): SyncStreamStatus[] | undefined {
     return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
@@ -131,8 +129,6 @@ export class SyncStatus {
 
   /**
    * If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
-   *
-   * @experimental Sync streams are currently in alpha.
    */
   forStream(stream: SyncStreamDescription): SyncStreamStatus | undefined {
     const asJson = JSON.stringify(stream.parameters);

package/src/db/schema/RawTable.ts

@@ -9,11 +9,6 @@ import { TableOrRawTableOptions } from './Table.js';
  * To collect local writes to raw tables with PowerSync, custom triggers are required. See
  * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
  * using raw tables.
- *
- * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
- *
- * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
- * stability guarantees.
  */
 export type RawTableType = RawTableTypeWithStatements | InferredRawTableType;
 

package/src/db/schema/Schema.ts

@@ -51,10 +51,8 @@ export class Schema<S extends SchemaType = SchemaType> {
    * developer instead of automatically by PowerSync.
    * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
    * using client-side table and column constraints.
-   * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
    *
    * @param tables An object of (table name, raw table definition) entries.
-   * @experimental Note that the raw tables API is still experimental and may change in the future.
    */
   withRawTables(tables: Record<string, RawTableType>) {
     for (const [name, rawTableDefinition] of Object.entries(tables)) {

package/src/index.ts

@@ -59,9 +59,9 @@ export * from './client/watched/WatchedQuery.js';
 export * from './utils/AbortOperation.js';
 export * from './utils/BaseObserver.js';
 export * from './utils/ControlledExecutor.js';
-export * from './utils/DataStream.js';
 export * from './utils/Logger.js';
 export * from './utils/mutex.js';
 export * from './utils/parseQuery.js';
+export type { SimpleAsyncIterator } from './utils/stream_transform.js';
 
 export * from './types/types.js';

package/src/utils/async.ts

@@ -1,14 +1,3 @@
-/**
- * A ponyfill for `Symbol.asyncIterator` that is compatible with the
- * [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
- * we recommend for React Native.
- *
- * As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
- * iterators without requiring them.
- */
-export const symbolAsyncIterator: typeof Symbol.asyncIterator =
-  Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
-
 /**
  * Throttle a function to be called at most once every "wait" milliseconds,
  * on the trailing edge.