@powersync/common 0.0.0-dev-20260630144038 → 0.0.0-dev-20261015114518

package/src/attachments/AttachmentQueue.ts

@@ -1,10 +1,13 @@
 import { DifferentialWatchedQuery } from '../client/watched/processors/DifferentialQueryProcessor.js';
+import { Mutex } from '../utils/mutex.js';
 import { LogLevels, PowerSyncLogger } from '../utils/Logger.js';
 import { Transaction } from '../db/DBAdapter.js';
 import { AttachmentContext } from './AttachmentContext.js';
 import { AttachmentErrorHandler } from './AttachmentErrorHandler.js';
 import { AttachmentService } from './AttachmentService.js';
-import { AttachmentData, LocalStorageAdapter } from './LocalStorageAdapter.js';
+import { AttachmentTransportAdapter } from './AttachmentTransportAdapter.js';
+import { BufferedAttachmentTransport } from './BufferedAttachmentTransport.js';
+import { AttachmentData, LocalStorageAdapter, StreamingLocalStorageAdapter } from './LocalStorageAdapter.js';
 import { RemoteStorageAdapter } from './RemoteStorageAdapter.js';
 import { ATTACHMENT_TABLE, AttachmentRecord, AttachmentState } from './Schema.js';
 import { SyncingService } from './SyncingService.js';
@@ -12,24 +15,22 @@ import { WatchedAttachmentItem } from './WatchedAttachmentItem.js';
 import { CommonPowerSyncDatabase } from '../client/CommonPowerSyncDatabase.js';
 
 /**
- * Configuration options for {@link AttachmentQueue}.
+ * Fields common to every {@link AttachmentQueueOptions} variant.
  *
  * @experimental
  * @alpha This is currently experimental and may change without a major version bump.
  */
-export interface AttachmentQueueOptions {
+export interface BaseAttachmentQueueOptions<TLocal extends LocalStorageAdapter = LocalStorageAdapter> {
   /**
    * PowerSync database instance
    */
   db: CommonPowerSyncDatabase;
   /**
-   * Remote storage adapter for upload/download operations
-   */
-  remoteStorage: RemoteStorageAdapter;
-  /**
-   * Local storage adapter for file persistence
+   * Local storage adapter for file persistence. Its type determines whether
+   * {@link AttachmentQueue.saveFileFromUri} is available (a
+   * {@link StreamingLocalStorageAdapter} enables it).
    */
-  localStorage: LocalStorageAdapter;
+  localStorage: TLocal;
   /**
    * Callback for monitoring attachment changes in your data model
    */
@@ -63,6 +64,52 @@ export interface AttachmentQueueOptions {
   errorHandler?: AttachmentErrorHandler;
 }
 
+/**
+ * Configuration options for {@link AttachmentQueue}.
+ *
+ * Provide **exactly one** remote mechanism:
+ * - `remoteStorage` — a {@link RemoteStorageAdapter}, wrapped in the default
+ *   default buffered transport that delegates upload/download/delete to it.
+ * - `transportAdapter` — an {@link AttachmentTransportAdapter} that owns all remote
+ *   operations directly (e.g. a native file-URI implementation for buffer-free
+ *   transfer of large files). No `remoteStorage` is needed in this case.
+ *
+ * Supplying both, or neither, is a type error.
+ *
+ * @experimental
+ * @alpha This is currently experimental and may change without a major version bump.
+ */
+export type AttachmentQueueOptions<TLocal extends LocalStorageAdapter = LocalStorageAdapter> =
+  BaseAttachmentQueueOptions<TLocal> &
+    (
+      | { remoteStorage: RemoteStorageAdapter; transportAdapter?: never }
+      | { transportAdapter: AttachmentTransportAdapter; remoteStorage?: never }
+    );
+
+/**
+ * Fields shared by {@link AttachmentQueue.saveFile} and {@link AttachmentQueue.saveFileFromUri}.
+ *
+ * @alpha
+ */
+export interface SaveAttachmentOptions {
+  /** File extension (e.g., 'jpg', 'pdf') */
+  fileExtension: string;
+  /** MIME type of the file (e.g., 'image/jpeg') */
+  mediaType?: string;
+  /** Optional metadata to associate with the attachment */
+  metaData?: string;
+  /** Optional custom ID. If not provided, a UUID will be generated */
+  id?: string;
+  /**
+   * Optional callback to execute additional database operations within the same transaction as the
+   * attachment creation.
+   */
+  updateHook?: (transaction: Transaction, attachment: AttachmentRecord) => Promise<void>;
+}
+
+/** How the file bytes reach managed storage when creating an upload attachment. */
+type AttachmentSource = { kind: 'data'; data: AttachmentData } | { kind: 'uri'; localUri: string };
+
 /**
  * AttachmentQueue manages the lifecycle and synchronization of attachments
  * between local and remote storage.
@@ -72,7 +119,7 @@ export interface AttachmentQueueOptions {
  * @experimental
  * @alpha This is currently experimental and may change without a major version bump.
  */
-export class AttachmentQueue {
+export class AttachmentQueue<TLocal extends LocalStorageAdapter = LocalStorageAdapter> {
   /** Timer for periodic synchronization operations */
   private periodicSyncTimer?: ReturnType<typeof setInterval>;
 
@@ -80,10 +127,7 @@ export class AttachmentQueue {
   private readonly syncingService: SyncingService;
 
   /** Adapter for local file storage operations */
-  readonly localStorage: LocalStorageAdapter;
-
-  /** Adapter for remote file storage operations */
-  readonly remoteStorage: RemoteStorageAdapter;
+  readonly localStorage: TLocal;
 
   /**
    * Callback function to watch for changes in attachment references in your data model.
@@ -134,6 +178,21 @@ export class AttachmentQueue {
 
   private watchAttachmentsAbortController!: AbortController;
 
+  /**
+   * Serializes concurrent `syncStorage()` triggers (periodic timer, watch-onDiff,
+   * status-changed). Held across the whole batch, but only contended by other
+   * sync triggers — foreground `saveFile` / `deleteFile` / watched-attachment
+   * processing don't take this lock and proceed in parallel via the
+   * `AttachmentService` mutex, which is acquired only briefly per row.
+   */
+  private syncLoopMutex: Mutex;
+
+  /**
+   * Aborted by `stopSync()` to interrupt an in-flight batch within one
+   * attachment's processing time. Polled between rows by `SyncingService`.
+   */
+  private syncAbortController?: AbortController;
+
   /**
    * Creates a new AttachmentQueue instance.
    *
@@ -143,6 +202,7 @@ export class AttachmentQueue {
     db,
     localStorage,
     remoteStorage,
+    transportAdapter,
     watchAttachments,
     logger,
     tableName = ATTACHMENT_TABLE,
@@ -151,9 +211,9 @@ export class AttachmentQueue {
     downloadAttachments = true,
     archivedCacheLimit = 100,
     errorHandler
-  }: AttachmentQueueOptions) {
+  }: AttachmentQueueOptions<TLocal>) {
     this.db = db;
-    this.remoteStorage = remoteStorage;
+    this.syncLoopMutex = db.createMutex();
     this.localStorage = localStorage;
     this.watchAttachments = watchAttachments;
     this.tableName = tableName;
@@ -163,10 +223,13 @@ export class AttachmentQueue {
     this.downloadAttachments = downloadAttachments;
     this.logger = logger ?? db.logger;
     this.attachmentService = new AttachmentService(db, this.logger, tableName, archivedCacheLimit);
+
+    const transport = transportAdapter ?? new BufferedAttachmentTransport(localStorage, remoteStorage!);
+
     this.syncingService = new SyncingService(
       this.attachmentService,
       localStorage,
-      remoteStorage,
+      transport,
       this.logger,
       errorHandler
     );
@@ -192,17 +255,24 @@ export class AttachmentQueue {
    * - Handles state transitions for archived and new attachments
    */
   async startSync(): Promise<void> {
-    await this.stopSync();
-
-    this.watchActiveAttachments = this.attachmentService.watchActiveAttachments({
-      throttleMs: this.syncThrottleDuration
-    });
+    this.syncAbortController?.abort();
+    const syncAbortController = (this.syncAbortController = new AbortController());
+    await this.stopSyncAfterAbort();
 
     // immediately invoke the sync storage to initialize local storage
     await this.localStorage.initialize();
 
     await this.verifyAttachments();
 
+    // Skip rest of setup if a stopSync() aborted this call during the async setup steps.
+    if (syncAbortController.signal.aborted) {
+      return;
+    }
+
+    this.watchActiveAttachments = this.attachmentService.watchActiveAttachments({
+      throttleMs: this.syncThrottleDuration
+    });
+
     // Sync storage periodically
     this.periodicSyncTimer = setInterval(async () => {
       await this.syncStorage();
@@ -324,22 +394,48 @@ export class AttachmentQueue {
    *
    * This is called automatically at regular intervals when sync is started,
    * but can also be called manually to trigger an immediate sync.
+   *
+   * Concurrent invocations are serialized via `syncLoopMutex`.
    */
   async syncStorage(): Promise<void> {
-    await this.attachmentService.withContext(async (ctx) => {
-      const activeAttachments = await ctx.getActiveAttachments();
-      await this.localStorage.initialize();
-      await this.syncingService.processAttachments(activeAttachments, ctx);
-      await this.syncingService.deleteArchivedAttachments(ctx);
-    });
+    const signal = this.syncAbortController?.signal;
+    // We have a signal from startSync() to stopSync(), so treat the absence of one like an aborted sync.
+    if (signal == null || signal?.aborted) return;
+
+    try {
+      await this.syncLoopMutex.runExclusive(async () => {
+        const activeAttachments = await this.attachmentService.withContext((ctx) => ctx.getActiveAttachments());
+        await this.localStorage.initialize();
+
+        await this.syncingService.processAttachments(activeAttachments, { signal });
+
+        if (signal.aborted) return;
+
+        await this.attachmentService.withContext((ctx) => this.syncingService.deleteArchivedAttachments(ctx));
+      }, signal);
+    } catch (error) {
+      // A queued batch's acquire rejects when `stopSync` aborts — expected, not an error.
+      if (signal.aborted) return;
+      throw error;
+    }
   }
 
   /**
    * Stops the attachment synchronization process.
    *
-   * Clears the periodic sync timer and closes all active attachment watchers.
+   * Clears the periodic sync timer, closes all active attachment watchers, and
+   * aborts any in-flight `syncStorage()` call so it exits within one
+   * attachment's processing time instead of running the batch to completion.
    */
   async stopSync(): Promise<void> {
+    if (this.syncAbortController) {
+      this.syncAbortController.abort();
+      this.syncAbortController = undefined;
+    }
+    return this.stopSyncAfterAbort();
+  }
+
+  private async stopSyncAfterAbort(): Promise<void> {
     clearInterval(this.periodicSyncTimer);
     this.periodicSyncTimer = undefined;
     if (this.watchActiveAttachments) await this.watchActiveAttachments.close();
@@ -367,49 +463,30 @@ export class AttachmentQueue {
     return this.attachmentService.withContext(callback);
   }
   /**
-   * Saves a file to local storage and queues it for upload to remote storage.
-   *
-   * @param options - File save options
-   * @returns Promise resolving to the created attachment record
+   * Creates a `QUEUED_UPLOAD` attachment record, placing the file at the managed
+   * `localUri` from the given `source`, and persists the record
+   * alongside the caller's `updateHook` in a single transaction.
    */
-  async saveFile({
-    data,
-    fileExtension,
-    mediaType,
-    metaData,
-    id,
-    updateHook
-  }: {
-    /**
-     * The file data as ArrayBuffer, Blob, or base64 string
-     */
-    data: AttachmentData;
-    /**
-     * File extension (e.g., 'jpg', 'pdf')
-     */
-    fileExtension: string;
-    /**
-     * MIME type of the file (e.g., 'image/jpeg')
-     */
-    mediaType?: string;
-    /**
-     * Optional metadata to associate with the attachment
-     */
-    metaData?: string;
-    /**
-     * Optional custom ID. If not provided, a UUID will be generated
-     */
-    id?: string;
-    /**
-     * Optional callback to execute additional database operations within the same transaction as the attachment
-     * creation.
-     */
-    updateHook?: (transaction: Transaction, attachment: AttachmentRecord) => Promise<void>;
-  }): Promise<AttachmentRecord> {
+  private async createUploadAttachment(
+    { fileExtension, mediaType, metaData, id, updateHook }: SaveAttachmentOptions,
+    source: AttachmentSource
+  ): Promise<AttachmentRecord> {
     const resolvedId = id ?? (await this.generateAttachmentId());
     const filename = `${resolvedId}.${fileExtension}`;
     const localUri = this.localStorage.getLocalUri(filename);
-    const size = await this.localStorage.saveFile(localUri, data);
+
+    let size: number;
+    if (source.kind === 'data') {
+      size = await this.localStorage.saveFile(localUri, source.data);
+    } else {
+      // saveFileFromUri is only exposed for streaming-capable local adapters; guard at
+      // runtime too for plain-JS callers.
+      const localStorage = this.localStorage as Partial<StreamingLocalStorageAdapter>;
+      if (!localStorage.moveFile) {
+        throw new Error('The configured local storage adapter does not support moveFile, required by saveFileFromUri.');
+      }
+      size = await localStorage.moveFile(source.localUri, localUri);
+    }
 
     const attachment: AttachmentRecord = {
       id: resolvedId,
@@ -433,6 +510,37 @@ export class AttachmentQueue {
     return attachment;
   }
 
+  /**
+   * Saves in-memory file data to local storage and queues it for upload.
+   *
+   * @param options - File data plus {@link SaveAttachmentOptions}
+   * @returns Promise resolving to the created attachment record
+   */
+  async saveFile(options: SaveAttachmentOptions & { data: AttachmentData }): Promise<AttachmentRecord> {
+    return this.createUploadAttachment(options, { kind: 'data', data: options.data });
+  }
+
+  /**
+   * Registers a file that already exists on disk and queues it for upload, moving it
+   * into managed storage without loading it into memory.
+   *
+   * Prefer this over {@link AttachmentQueue.saveFile} for large, app-originated files
+   * (recordings, videos): it avoids reading the file into an `ArrayBuffer` just to write
+   * it back to disk. Requires the local storage adapter to implement `moveFile`.
+   *
+   * Only available when the queue is configured with a {@link StreamingLocalStorageAdapter}
+   * (one that implements `moveFile`).
+   *
+   * @param options - The existing file's `localUri` plus {@link SaveAttachmentOptions}
+   * @returns Promise resolving to the created attachment record
+   */
+  async saveFileFromUri(
+    this: AttachmentQueue<StreamingLocalStorageAdapter>,
+    options: SaveAttachmentOptions & { localUri: string }
+  ): Promise<AttachmentRecord> {
+    return this.createUploadAttachment(options, { kind: 'uri', localUri: options.localUri });
+  }
+
   async deleteFile({
     id,
     updateHook

package/src/attachments/AttachmentTransportAdapter.ts

@@ -0,0 +1,49 @@
+import { AttachmentRecord } from './Schema.js';
+
+/**
+ * An {@link AttachmentRecord} that is guaranteed to have a `localUri`.
+ *
+ * The syncing service assigns `localUri` before invoking a transport download,
+ * so implementations always receive both the metadata and the destination path.
+ *
+ * @alpha
+ */
+export type LocatedAttachmentRecord = AttachmentRecord & { localUri: string };
+
+/**
+ * AttachmentTransportAdapter owns all remote-side operations for an attachment —
+ * transfer (upload/download) and delete — as single operations.
+ *
+ * A transport owns the entire transfer, letting implementations pick the most
+ * efficient mechanism available (buffer, stream, or a platform-native file-URI
+ * upload/download API). On platforms like React Native this allows large files to
+ * be transferred without ever materializing them in the JS heap.
+ *
+ * The default transport composes the local and remote storage adapters. Provide a custom transport (via
+ * `AttachmentQueue`'s `transportAdapter` option) to own the whole remote side; in
+ * that case a separate `remoteStorage` is not required.
+ *
+ * @experimental
+ * @alpha This is currently experimental and may change without a major version bump.
+ */
+export interface AttachmentTransportAdapter {
+  /**
+   * Uploads the attachment's local file to remote storage.
+   * @param attachment - The attachment to upload. `localUri` points at the source file.
+   */
+  upload(attachment: LocatedAttachmentRecord): Promise<void>;
+
+  /**
+   * Downloads the remote file into `attachment.localUri`.
+   * @param attachment - The attachment to download. `localUri` is the destination path,
+   *                      assigned by the syncing service before this call.
+   */
+  download(attachment: LocatedAttachmentRecord): Promise<void>;
+
+  /**
+   * Deletes the attachment's file from remote storage. Local file removal is handled
+   * separately by the syncing service.
+   * @param attachment - The attachment to delete.
+   */
+  delete(attachment: AttachmentRecord): Promise<void>;
+}

package/src/attachments/BufferedAttachmentTransport.ts

@@ -0,0 +1,37 @@
+import { AttachmentTransportAdapter, LocatedAttachmentRecord } from './AttachmentTransportAdapter.js';
+import { LocalStorageAdapter } from './LocalStorageAdapter.js';
+import { RemoteStorageAdapter } from './RemoteStorageAdapter.js';
+import { AttachmentRecord } from './Schema.js';
+
+/**
+ * Default {@link AttachmentTransportAdapter}, composing the local and remote
+ * storage adapters.
+ *
+ * The full file body is materialized as an `ArrayBuffer` in JS memory between the
+ * two adapter calls. This is fine for small files but can cause memory pressure on
+ * large ones — environments that support native transfer should provide a custom
+ * transport instead.
+ *
+ * @experimental
+ * @alpha This is currently experimental and may change without a major version bump.
+ */
+export class BufferedAttachmentTransport implements AttachmentTransportAdapter {
+  constructor(
+    private localStorage: LocalStorageAdapter,
+    private remoteStorage: RemoteStorageAdapter
+  ) {}
+
+  async upload(attachment: LocatedAttachmentRecord): Promise<void> {
+    const fileData = await this.localStorage.readFile(attachment.localUri);
+    await this.remoteStorage.uploadFile(fileData, attachment);
+  }
+
+  async download(attachment: LocatedAttachmentRecord): Promise<void> {
+    const fileData = await this.remoteStorage.downloadFile(attachment);
+    await this.localStorage.saveFile(attachment.localUri, fileData);
+  }
+
+  async delete(attachment: AttachmentRecord): Promise<void> {
+    await this.remoteStorage.deleteFile(attachment);
+  }
+}

package/src/attachments/LocalStorageAdapter.ts

@@ -76,3 +76,23 @@ export interface LocalStorageAdapter {
    */
   getLocalUri(filename: string): string;
 }
+
+/**
+ * A {@link LocalStorageAdapter} that can relocate a file into managed storage without
+ * loading it into memory. Required for {@link AttachmentQueue.saveFileFromUri}; only
+ * queues configured with a streaming-capable local adapter expose that method.
+ *
+ * @experimental
+ * @alpha This is currently experimental and may change without a major version bump.
+ */
+export interface StreamingLocalStorageAdapter extends LocalStorageAdapter {
+  /**
+   * Moves a file into managed storage without loading it into memory.
+   * Overwrites any existing file at the target. When source and target are the same
+   * path, this is a no-op that just reports the size.
+   * @param sourceUri - Path of the existing file
+   * @param targetUri - Destination path within managed storage
+   * @returns Number of bytes in the moved file
+   */
+  moveFile(sourceUri: string, targetUri: string): Promise<number>;
+}

package/src/attachments/Schema.ts

@@ -1,5 +1,5 @@
 import { column } from '../db/schema/Column.js';
-import { Table } from '../db/schema/Table.js';
+import { RowType, Table } from '../db/schema/Table.js';
 import { TableOptions } from '../db/schema/Table.js';
 
 /**
@@ -66,30 +66,39 @@ export enum AttachmentState {
  */
 export interface AttachmentTableOptions extends Omit<TableOptions, 'name' | 'columns'> {}
 
+/**
+ * @alpha
+ */
+export const ATTACHMENT_TABLE_COLUMNS = {
+  filename: column.text,
+  local_uri: column.text,
+  timestamp: column.integer,
+  size: column.integer,
+  media_type: column.text,
+  state: column.integer, // Corresponds to AttachmentState
+  has_synced: column.integer,
+  meta_data: column.text
+};
+
 /**
  * AttachmentTable defines the schema for the attachment queue table.
  *
  * @alpha
  */
-export class AttachmentTable extends Table {
+export class AttachmentTable extends Table<typeof ATTACHMENT_TABLE_COLUMNS> {
   constructor(options?: AttachmentTableOptions) {
-    super(
-      {
-        filename: column.text,
-        local_uri: column.text,
-        timestamp: column.integer,
-        size: column.integer,
-        media_type: column.text,
-        state: column.integer, // Corresponds to AttachmentState
-        has_synced: column.integer,
-        meta_data: column.text
-      },
-      {
-        ...options,
-        viewName: options?.viewName ?? ATTACHMENT_TABLE,
-        localOnly: true,
-        insertOnly: false
-      }
-    );
+    super(ATTACHMENT_TABLE_COLUMNS, {
+      ...options,
+      viewName: options?.viewName ?? ATTACHMENT_TABLE,
+      localOnly: true,
+      insertOnly: false
+    });
   }
 }
+
+/**
+ * AttachmentTableRecord represents the row type of the attachment table.
+ *
+ * @alpha
+ */
+export type AttachmentTableRecord = RowType<AttachmentTable>;