@powersync/common 0.0.0-dev-20251106124255 → 0.0.0-dev-20251201150812

package/src/client/ConnectionManager.ts

@@ -1,4 +1,5 @@
 import { ILogger } from 'js-logger';
+import { SyncStatus } from '../db/crud/SyncStatus.js';
 import { BaseListener, BaseObserver } from '../utils/BaseObserver.js';
 import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnector.js';
 import {
@@ -13,7 +14,6 @@ import {
   SyncStreamSubscribeOptions,
   SyncStreamSubscription
 } from './sync/sync-streams.js';
-import { SyncStatus } from '../db/crud/SyncStatus.js';
 
 /**
  * @internal

package/src/client/sync/bucket/SqliteBucketStorage.ts

@@ -255,7 +255,7 @@ export class SqliteBucketStorage extends BaseObserver<BucketStorageListener> imp
       // Nothing to update
       return false;
     }
-    const rs = await this.db.getAll<{ seq: number }>("SELECT seq FROM sqlite_sequence WHERE name = 'ps_crud'");
+    const rs = await this.db.getAll<{ seq: number }>("SELECT seq FROM main.sqlite_sequence WHERE name = 'ps_crud'");
     if (!rs.length) {
       // Nothing to update
       return false;
@@ -273,7 +273,7 @@ export class SqliteBucketStorage extends BaseObserver<BucketStorageListener> imp
         return false;
       }
 
-      const rs = await tx.execute("SELECT seq FROM sqlite_sequence WHERE name = 'ps_crud'");
+      const rs = await tx.execute("SELECT seq FROM main.sqlite_sequence WHERE name = 'ps_crud'");
       if (!rs.rows?.length) {
         // assert isNotEmpty
         throw new Error('SQLite Sequence should not be empty');

package/src/client/sync/stream/AbstractRemote.ts

@@ -6,8 +6,9 @@ import PACKAGE from '../../../../package.json' with { type: 'json' };
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { DataStream } from '../../../utils/DataStream.js';
 import { PowerSyncCredentials } from '../../connection/PowerSyncCredentials.js';
-import { StreamingSyncRequest } from './streaming-sync-types.js';
 import { WebsocketClientTransport } from './WebsocketClientTransport.js';
+import { StreamingSyncRequest } from './streaming-sync-types.js';
+
 
 export type BSONImplementation = typeof BSON;
 
@@ -557,9 +558,11 @@ export abstract class AbstractRemote {
     // Create a new stream splitting the response at line endings while also handling cancellations
     // by closing the reader.
     const reader = res.body.getReader();
+    let readerReleased = false;
     // This will close the network request and read stream
     const closeReader = async () => {
       try {
+        readerReleased = true;
         await reader.cancel();
       } catch (ex) {
         // an error will throw if the reader hasn't been used yet
@@ -567,20 +570,27 @@ export abstract class AbstractRemote {
       reader.releaseLock();
     };
 
+
+    const stream = new DataStream<T, string>({
+      logger: this.logger,
+      mapLine: mapLine
+    });
+
     abortSignal?.addEventListener('abort', () => {
       closeReader();
+      stream.close();
     });
 
     const decoder = this.createTextDecoder();
     let buffer = '';
 
-    const stream = new DataStream<T, string>({
-      logger: this.logger,
-      mapLine: mapLine
-    });
+
 
     const l = stream.registerListener({
       lowWater: async () => {
+        if (stream.closed || abortSignal?.aborted || readerReleased) {
+          return
+        } 
         try {
           let didCompleteLine = false;
           while (!didCompleteLine) {

package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts

@@ -16,7 +16,7 @@ import {
 import { CrudEntry } from '../bucket/CrudEntry.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { AbstractRemote, FetchStrategy, SyncStreamOptions } from './AbstractRemote.js';
-import { coreStatusToJs, EstablishSyncStream, Instruction, SyncPriorityStatus } from './core-instruction.js';
+import { EstablishSyncStream, Instruction, coreStatusToJs } from './core-instruction.js';
 import {
   BucketRequest,
   CrudUploadNotification,
@@ -429,7 +429,7 @@ The next upload iteration will be delayed.`);
                 uploadError: ex
               }
             });
-            await this.delayRetry(controller.signal);
+            await this.delayRetry(controller.signal, this.options.crudUploadThrottleMs);
             if (!this.isConnected) {
               // Exit the upload loop if the sync stream is no longer connected
               break;
@@ -1216,15 +1216,14 @@ The next upload iteration will be delayed.`);
     this.iterateListeners((cb) => cb.statusUpdated?.(options));
   }
 
-  private async delayRetry(signal?: AbortSignal): Promise<void> {
+  private async delayRetry(signal?: AbortSignal, delayMs?: number): Promise<void> {
     return new Promise((resolve) => {
       if (signal?.aborted) {
         // If the signal is already aborted, resolve immediately
         resolve();
         return;
       }
-
-      const { retryDelayMs } = this.options;
+      const delay = delayMs ?? this.options.retryDelayMs;
 
       let timeoutId: ReturnType<typeof setTimeout> | undefined;
 
@@ -1238,7 +1237,7 @@ The next upload iteration will be delayed.`);
       };
 
       signal?.addEventListener('abort', endDelay, { once: true });
-      timeoutId = setTimeout(endDelay, retryDelayMs);
+      timeoutId = setTimeout(endDelay, delay);
     });
   }
 

package/src/client/triggers/TriggerManager.ts

@@ -14,8 +14,11 @@ export enum DiffTriggerOperation {
  * @experimental
  * Diffs created by {@link TriggerManager#createDiffTrigger} are stored in a temporary table.
  * This is the base record structure for all diff records.
+ *
+ * @template TOperationId - The type for `operation_id`. Defaults to `number` as returned by default SQLite database queries.
+ * Use `string` for full 64-bit precision when using `{ castOperationIdAsText: true }` option.
  */
-export interface BaseTriggerDiffRecord {
+export interface BaseTriggerDiffRecord<TOperationId extends string | number = number> {
   /**
    * The modified row's `id` column value.
    */
@@ -24,6 +27,13 @@ export interface BaseTriggerDiffRecord {
    * The operation performed which created this record.
    */
   operation: DiffTriggerOperation;
+  /**
+   * Auto-incrementing primary key for the operation.
+   * Defaults to number as returned by database queries (wa-sqlite returns lower 32 bits).
+   * Can be string for full 64-bit precision when using `{ castOperationIdAsText: true }` option.
+   */
+  operation_id: TOperationId;
+
   /**
    * Time the change operation was recorded.
    * This is in ISO 8601 format, e.g. `2023-10-01T12:00:00.000Z`.
@@ -37,7 +47,8 @@ export interface BaseTriggerDiffRecord {
  * This record contains the new value and optionally the previous value.
  * Values are stored as JSON strings.
  */
-export interface TriggerDiffUpdateRecord extends BaseTriggerDiffRecord {
+export interface TriggerDiffUpdateRecord<TOperationId extends string | number = number>
+  extends BaseTriggerDiffRecord<TOperationId> {
   operation: DiffTriggerOperation.UPDATE;
   /**
    * The updated state of the row in JSON string format.
@@ -54,7 +65,8 @@ export interface TriggerDiffUpdateRecord extends BaseTriggerDiffRecord {
  * Represents a diff record for a SQLite INSERT operation.
  * This record contains the new value represented as a JSON string.
  */
-export interface TriggerDiffInsertRecord extends BaseTriggerDiffRecord {
+export interface TriggerDiffInsertRecord<TOperationId extends string | number = number>
+  extends BaseTriggerDiffRecord<TOperationId> {
   operation: DiffTriggerOperation.INSERT;
   /**
    * The value of the row, at the time of INSERT, in JSON string format.
@@ -67,7 +79,8 @@ export interface TriggerDiffInsertRecord extends BaseTriggerDiffRecord {
  * Represents a diff record for a SQLite DELETE operation.
  * This record contains the new value represented as a JSON string.
  */
-export interface TriggerDiffDeleteRecord extends BaseTriggerDiffRecord {
+export interface TriggerDiffDeleteRecord<TOperationId extends string | number = number>
+  extends BaseTriggerDiffRecord<TOperationId> {
   operation: DiffTriggerOperation.DELETE;
   /**
    * The value of the row, before the DELETE operation, in JSON string format.
@@ -82,27 +95,53 @@ export interface TriggerDiffDeleteRecord extends BaseTriggerDiffRecord {
  *
  * Querying the DIFF table directly with {@link TriggerDiffHandlerContext#withDiff} will return records
  * with the structure of this type.
+ *
+ * @template TOperationId - The type for `operation_id`. Defaults to `number` as returned by database queries.
+ * Use `string` for full 64-bit precision when using `{ castOperationIdAsText: true }` option.
+ *
  * @example
  * ```typescript
+ * // Default: operation_id is number
  * const diffs = await context.withDiff<TriggerDiffRecord>('SELECT * FROM DIFF');
- * diff.forEach(diff => console.log(diff.operation, diff.timestamp, JSON.parse(diff.value)))
+ *
+ * // With string operation_id for full precision
+ * const diffsWithString = await context.withDiff<TriggerDiffRecord<string>>(
+ *   'SELECT * FROM DIFF',
+ *   undefined,
+ *   { castOperationIdAsText: true }
+ * );
  * ```
  */
-export type TriggerDiffRecord = TriggerDiffUpdateRecord | TriggerDiffInsertRecord | TriggerDiffDeleteRecord;
+export type TriggerDiffRecord<TOperationId extends string | number = number> =
+  | TriggerDiffUpdateRecord<TOperationId>
+  | TriggerDiffInsertRecord<TOperationId>
+  | TriggerDiffDeleteRecord<TOperationId>;
 
 /**
  * @experimental
  * Querying the DIFF table directly with {@link TriggerDiffHandlerContext#withExtractedDiff} will return records
  * with the tracked columns extracted from the JSON value.
  * This type represents the structure of such records.
+ *
+ * @template T - The type for the extracted columns from the tracked JSON value.
+ * @template TOperationId - The type for `operation_id`. Defaults to `number` as returned by database queries.
+ * Use `string` for full 64-bit precision when using `{ castOperationIdAsText: true }` option.
+ *
  * @example
  * ```typescript
+ * // Default: operation_id is number
  * const diffs = await context.withExtractedDiff<ExtractedTriggerDiffRecord<{id: string, name: string}>>('SELECT * FROM DIFF');
- * diff.forEach(diff => console.log(diff.__operation, diff.__timestamp, diff.columnName))
+ *
+ * // With string operation_id for full precision
+ * const diffsWithString = await context.withExtractedDiff<ExtractedTriggerDiffRecord<{id: string, name: string}, string>>(
+ *   'SELECT * FROM DIFF',
+ *   undefined,
+ *   { castOperationIdAsText: true }
+ * );
  * ```
  */
-export type ExtractedTriggerDiffRecord<T> = T & {
-  [K in keyof Omit<BaseTriggerDiffRecord, 'id'> as `__${string & K}`]: TriggerDiffRecord[K];
+export type ExtractedTriggerDiffRecord<T, TOperationId extends string | number = number> = T & {
+  [K in keyof Omit<BaseTriggerDiffRecord<TOperationId>, 'id'> as `__${string & K}`]: TriggerDiffRecord<TOperationId>[K];
 } & {
   __previous_value?: string;
 };
@@ -183,6 +222,21 @@ export interface CreateDiffTriggerOptions extends BaseCreateDiffTriggerOptions {
  */
 export type TriggerRemoveCallback = () => Promise<void>;
 
+/**
+ * @experimental
+ * Options for {@link TriggerDiffHandlerContext#withDiff}.
+ */
+export interface WithDiffOptions {
+  /**
+   * If true, casts `operation_id` as TEXT in the internal CTE to preserve full 64-bit precision.
+   * Use this when you need to ensure `operation_id` is treated as a string to avoid precision loss
+   * for values exceeding JavaScript's Number.MAX_SAFE_INTEGER.
+   *
+   * When enabled, use {@link TriggerDiffRecord}<string> to type the result correctly.
+   */
+  castOperationIdAsText?: boolean;
+}
+
 /**
  * @experimental
  * Context for the `onChange` handler provided to {@link TriggerManager#trackTableDiff}.
@@ -200,9 +254,10 @@ export interface TriggerDiffHandlerContext extends LockContext {
    * The `DIFF` table is of the form described in {@link TriggerManager#createDiffTrigger}
    * ```sql
    * CREATE TEMP DIFF (
+   *       operation_id INTEGER PRIMARY KEY AUTOINCREMENT,
    *       id TEXT,
    *       operation TEXT,
-   *       timestamp TEXT
+   *       timestamp TEXT,
    *       value TEXT,
    *       previous_value TEXT
    *     );
@@ -222,8 +277,19 @@ export interface TriggerDiffHandlerContext extends LockContext {
    * JOIN todos ON DIFF.id = todos.id
    * WHERE json_extract(DIFF.value, '$.status') = 'active'
    * ```
+   *
+   * @example
+   * ```typescript
+   * // With operation_id cast as TEXT for full precision
+   * const diffs = await context.withDiff<TriggerDiffRecord<string>>(
+   *   'SELECT * FROM DIFF',
+   *   undefined,
+   *   { castOperationIdAsText: true }
+   * );
+   * // diffs[0].operation_id is now typed as string
+   * ```
    */
-  withDiff: <T = any>(query: string, params?: ReadonlyArray<Readonly<any>>) => Promise<T[]>;
+  withDiff: <T = any>(query: string, params?: ReadonlyArray<Readonly<any>>, options?: WithDiffOptions) => Promise<T[]>;
 
   /**
    * Allows querying the database with access to the table containing diff records.
@@ -292,9 +358,10 @@ export interface TriggerManager {
    *
    * ```sql
    * CREATE TEMP TABLE ${destination} (
+   *       operation_id INTEGER PRIMARY KEY AUTOINCREMENT,
    *       id TEXT,
    *       operation TEXT,
-   *       timestamp TEXT
+   *       timestamp TEXT,
    *       value TEXT,
    *       previous_value TEXT
    *     );

package/src/client/triggers/TriggerManagerImpl.ts

@@ -7,7 +7,8 @@ import {
   DiffTriggerOperation,
   TrackDiffOptions,
   TriggerManager,
-  TriggerRemoveCallback
+  TriggerRemoveCallback,
+  WithDiffOptions
 } from './TriggerManager.js';
 
 export type TriggerManagerImplOptions = {
@@ -117,6 +118,7 @@ export class TriggerManagerImpl implements TriggerManager {
       await hooks?.beforeCreate?.(tx);
       await tx.execute(/* sql */ `
         CREATE TEMP TABLE ${destination} (
+          operation_id INTEGER PRIMARY KEY AUTOINCREMENT,
           id TEXT,
           operation TEXT,
           timestamp TEXT,
@@ -243,17 +245,20 @@ export class TriggerManagerImpl implements TriggerManager {
             const callbackResult = await options.onChange({
               ...tx,
               destinationTable: destination,
-              withDiff: async <T>(query, params) => {
+              withDiff: async <T>(query, params, options?: WithDiffOptions) => {
                 // Wrap the query to expose the destination table
+                const operationIdSelect = options?.castOperationIdAsText
+                  ? 'id, operation, CAST(operation_id AS TEXT) as operation_id, timestamp, value, previous_value'
+                  : '*';
                 const wrappedQuery = /* sql */ `
                   WITH
                     DIFF AS (
                       SELECT
-                        *
+                        ${operationIdSelect}
                       FROM
                         ${destination}
                       ORDER BY
-                        timestamp ASC
+                        operation_id ASC
                     ) ${query}
                 `;
                 return tx.getAll<T>(wrappedQuery, params);
@@ -267,13 +272,14 @@ export class TriggerManagerImpl implements TriggerManager {
                         id,
                         ${contextColumns.length > 0
                     ? `${contextColumns.map((col) => `json_extract(value, '$.${col}') as ${col}`).join(', ')},`
-                    : ''} operation as __operation,
+                    : ''} operation_id as __operation_id,
+                        operation as __operation,
                         timestamp as __timestamp,
                         previous_value as __previous_value
                       FROM
                         ${destination}
                       ORDER BY
-                        __timestamp ASC
+                        __operation_id ASC
                     ) ${query}
                 `;
                 return tx.getAll<T>(wrappedQuery, params);

package/src/db/crud/SyncStatus.ts

@@ -1,7 +1,7 @@
-import { CoreStreamSubscription } from '../../client/sync/stream/core-instruction.js';
 import { SyncClientImplementation } from '../../client/sync/stream/AbstractStreamingSyncImplementation.js';
-import { InternalProgressInformation, ProgressWithOperations, SyncProgress } from './SyncProgress.js';
+import { CoreStreamSubscription } from '../../client/sync/stream/core-instruction.js';
 import { SyncStreamDescription, SyncSubscriptionDescription } from '../../client/sync/sync-streams.js';
+import { InternalProgressInformation, ProgressWithOperations, SyncProgress } from './SyncProgress.js';
 
 export type SyncDataFlowStatus = Partial<{
   downloading: boolean;
@@ -250,13 +250,28 @@ export class SyncStatus {
     return {
       connected: this.connected,
       connecting: this.connecting,
-      dataFlow: this.dataFlowStatus,
+      dataFlow: {
+        ...this.dataFlowStatus,
+        uploadError: this.serializeError(this.dataFlowStatus.uploadError),
+        downloadError: this.serializeError(this.dataFlowStatus.downloadError)
+      },
       lastSyncedAt: this.lastSyncedAt,
       hasSynced: this.hasSynced,
       priorityStatusEntries: this.priorityStatusEntries
     };
   }
 
+  protected serializeError(error?: Error) {
+    if (typeof error == 'undefined') {
+      return undefined;
+    }
+    return {
+      name: error.name,
+      message: error.message,
+      stack: error.stack
+    };
+  }
+
   private static comparePriorities(a: SyncPriorityStatus, b: SyncPriorityStatus) {
     return b.priority - a.priority; // Reverse because higher priorities have lower numbers
   }