@powersync/common 1.46.0 → 1.48.0

package/src/attachments/RemoteStorageAdapter.ts

@@ -0,0 +1,30 @@
+import { AttachmentRecord } from './Schema.js';
+
+/**
+ * RemoteStorageAdapter defines the interface for remote storage operations.
+ * Implementations handle uploading, downloading, and deleting files from remote storage.
+ *
+ * @experimental
+ * @alpha This is currently experimental and may change without a major version bump.
+ */
+export interface RemoteStorageAdapter {
+  /**
+   * Uploads a file to remote storage.
+   * @param fileData The binary content of the file to upload
+   * @param attachment The associated attachment metadata
+   */
+  uploadFile(fileData: ArrayBuffer, attachment: AttachmentRecord): Promise<void>;
+
+  /**
+   * Downloads a file from remote storage.
+   * @param attachment The attachment describing the file to download
+   * @returns The binary data of the downloaded file
+   */
+  downloadFile(attachment: AttachmentRecord): Promise<ArrayBuffer>;
+
+  /**
+   * Deletes a file from remote storage.
+   * @param attachment The attachment describing the file to delete
+   */
+  deleteFile(attachment: AttachmentRecord): Promise<void>;
+}

package/src/attachments/Schema.ts

@@ -0,0 +1,87 @@
+import { column } from '../db/schema/Column.js';
+import { Table } from '../db/schema/Table.js';
+import { TableV2Options } from '../db/schema/Table.js';
+
+export const ATTACHMENT_TABLE = 'attachments';
+
+/**
+ * AttachmentRecord represents an attachment in the local database.
+ *
+ * @experimental
+ */
+export interface AttachmentRecord {
+  id: string;
+  filename: string;
+  localUri?: string;
+  size?: number;
+  mediaType?: string;
+  timestamp?: number;
+  metaData?: string;
+  hasSynced?: boolean;
+  state: AttachmentState;
+}
+
+/**
+ * Maps a database row to an AttachmentRecord.
+ *
+ * @param row - The database row object
+ * @returns The corresponding AttachmentRecord
+ *
+ * @experimental
+ */
+export function attachmentFromSql(row: any): AttachmentRecord {
+  return {
+    id: row.id,
+    filename: row.filename,
+    localUri: row.local_uri,
+    size: row.size,
+    mediaType: row.media_type,
+    timestamp: row.timestamp,
+    metaData: row.meta_data,
+    hasSynced: row.has_synced === 1,
+    state: row.state
+  };
+}
+
+/**
+ * AttachmentState represents the current synchronization state of an attachment.
+ *
+ * @experimental
+ */
+export enum AttachmentState {
+  QUEUED_UPLOAD = 0, // Attachment to be uploaded
+  QUEUED_DOWNLOAD = 1, // Attachment to be downloaded
+  QUEUED_DELETE = 2, // Attachment to be deleted
+  SYNCED = 3, // Attachment has been synced
+  ARCHIVED = 4 // Attachment has been orphaned, i.e. the associated record has been deleted
+}
+
+export interface AttachmentTableOptions extends Omit<TableV2Options, 'name' | 'columns'> {}
+
+/**
+ * AttachmentTable defines the schema for the attachment queue table.
+ *
+ * @internal
+ */
+export class AttachmentTable extends Table {
+  constructor(options?: AttachmentTableOptions) {
+    super(
+      {
+        filename: column.text,
+        local_uri: column.text,
+        timestamp: column.integer,
+        size: column.integer,
+        media_type: column.text,
+        state: column.integer, // Corresponds to AttachmentState
+        has_synced: column.integer,
+        meta_data: column.text
+      },
+      {
+        ...options,
+        viewName: options?.viewName ?? ATTACHMENT_TABLE,
+        localOnly: true,
+        insertOnly: false
+      }
+    );
+  }
+}

package/src/attachments/SyncingService.ts

@@ -0,0 +1,193 @@
+import { ILogger } from '../utils/Logger.js';
+import { AttachmentService } from './AttachmentService.js';
+import { LocalStorageAdapter } from './LocalStorageAdapter.js';
+import { RemoteStorageAdapter } from './RemoteStorageAdapter.js';
+import { AttachmentRecord, AttachmentState } from './Schema.js';
+import { AttachmentErrorHandler } from './AttachmentErrorHandler.js';
+import { AttachmentContext } from './AttachmentContext.js';
+
+/**
+ * Orchestrates attachment synchronization between local and remote storage.
+ * Handles uploads, downloads, deletions, and state transitions.
+ *
+ * @internal
+ */
+export class SyncingService {
+  private attachmentService: AttachmentService;
+  private localStorage: LocalStorageAdapter;
+  private remoteStorage: RemoteStorageAdapter;
+  private logger: ILogger;
+  private errorHandler?: AttachmentErrorHandler;
+
+  constructor(
+    attachmentService: AttachmentService,
+    localStorage: LocalStorageAdapter,
+    remoteStorage: RemoteStorageAdapter,
+    logger: ILogger,
+    errorHandler?: AttachmentErrorHandler
+  ) {
+    this.attachmentService = attachmentService;
+    this.localStorage = localStorage;
+    this.remoteStorage = remoteStorage;
+    this.logger = logger;
+    this.errorHandler = errorHandler;
+  }
+
+  /**
+   * Processes attachments based on their state (upload, download, or delete).
+   * All updates are saved in a single batch after processing.
+   *
+   * @param attachments - Array of attachment records to process
+   * @param context - Attachment context for database operations
+   * @returns Promise that resolves when all attachments have been processed and saved
+   */
+  async processAttachments(attachments: AttachmentRecord[], context: AttachmentContext): Promise<void> {
+    const updatedAttachments: AttachmentRecord[] = [];
+    for (const attachment of attachments) {
+      switch (attachment.state) {
+        case AttachmentState.QUEUED_UPLOAD:
+          const uploaded = await this.uploadAttachment(attachment);
+          updatedAttachments.push(uploaded);
+          break;
+        case AttachmentState.QUEUED_DOWNLOAD:
+          const downloaded = await this.downloadAttachment(attachment);
+          updatedAttachments.push(downloaded);
+          break;
+        case AttachmentState.QUEUED_DELETE:
+          const deleted = await this.deleteAttachment(attachment);
+          updatedAttachments.push(deleted);
+          break;
+
+        default:
+          break;
+      }
+    }
+
+    await context.saveAttachments(updatedAttachments);
+  }
+
+  /**
+   * Uploads an attachment from local storage to remote storage.
+   * On success, marks as SYNCED. On failure, defers to error handler or archives.
+   *
+   * @param attachment - The attachment record to upload
+   * @returns Updated attachment record with new state
+   * @throws Error if the attachment has no localUri
+   */
+  async uploadAttachment(attachment: AttachmentRecord): Promise<AttachmentRecord> {
+    this.logger.info(`Uploading attachment ${attachment.filename}`);
+    try {
+      if (attachment.localUri == null) {
+        throw new Error(`No localUri for attachment ${attachment.id}`);
+      }
+
+      const fileBlob = await this.localStorage.readFile(attachment.localUri);
+      await this.remoteStorage.uploadFile(fileBlob, attachment);
+
+      return {
+        ...attachment,
+        state: AttachmentState.SYNCED,
+        hasSynced: true
+      };
+    } catch (error) {
+      const shouldRetry = (await this.errorHandler?.onUploadError(attachment, error)) ?? true;
+      if (!shouldRetry) {
+        return {
+          ...attachment,
+          state: AttachmentState.ARCHIVED
+        };
+      }
+
+      return attachment;
+    }
+  }
+
+  /**
+   * Downloads an attachment from remote storage to local storage.
+   * Retrieves the file, converts to base64, and saves locally.
+   * On success, marks as SYNCED. On failure, defers to error handler or archives.
+   *
+   * @param attachment - The attachment record to download
+   * @returns Updated attachment record with local URI and new state
+   */
+  async downloadAttachment(attachment: AttachmentRecord): Promise<AttachmentRecord> {
+    this.logger.info(`Downloading attachment ${attachment.filename}`);
+    try {
+      const fileData = await this.remoteStorage.downloadFile(attachment);
+
+      const localUri = this.localStorage.getLocalUri(attachment.filename);
+      await this.localStorage.saveFile(localUri, fileData);
+
+      return {
+        ...attachment,
+        state: AttachmentState.SYNCED,
+        localUri: localUri,
+        hasSynced: true
+      };
+    } catch (error) {
+      const shouldRetry = (await this.errorHandler?.onDownloadError(attachment, error)) ?? true;
+      if (!shouldRetry) {
+        return {
+          ...attachment,
+          state: AttachmentState.ARCHIVED
+        };
+      }
+
+      return attachment;
+    }
+  }
+
+  /**
+   * Deletes an attachment from both remote and local storage.
+   * Removes the remote file, local file (if exists), and the attachment record.
+   * On failure, defers to error handler or archives.
+   *
+   * @param attachment - The attachment record to delete
+   * @returns Updated attachment record
+   */
+  async deleteAttachment(attachment: AttachmentRecord): Promise<AttachmentRecord> {
+    try {
+      await this.remoteStorage.deleteFile(attachment);
+      if (attachment.localUri) {
+        await this.localStorage.deleteFile(attachment.localUri);
+      }
+
+      await this.attachmentService.withContext(async (ctx) => {
+        await ctx.deleteAttachment(attachment.id);
+      });
+
+      return {
+        ...attachment,
+        state: AttachmentState.ARCHIVED
+      };
+    } catch (error) {
+      const shouldRetry = (await this.errorHandler?.onDeleteError(attachment, error)) ?? true;
+      if (!shouldRetry) {
+        return {
+          ...attachment,
+          state: AttachmentState.ARCHIVED
+        };
+      }
+
+      return attachment;
+    }
+  }
+
+  /**
+   * Performs cleanup of archived attachments by removing their local files and records.
+   * Errors during local file deletion are logged but do not prevent record deletion.
+   */
+  async deleteArchivedAttachments(context: AttachmentContext): Promise<boolean> {
+    return await context.deleteArchivedAttachments(async (archivedAttachments) => {
+      for (const attachment of archivedAttachments) {
+        if (attachment.localUri) {
+          try {
+            await this.localStorage.deleteFile(attachment.localUri);
+          } catch (error) {
+            this.logger.error('Error deleting local file for archived attachment', error);
+          }
+        }
+      }
+    });
+  }
+}

package/src/attachments/WatchedAttachmentItem.ts

@@ -0,0 +1,19 @@
+/**
+ * WatchedAttachmentItem represents an attachment reference in your application's data model.
+ * Use either filename OR fileExtension (not both).
+ *
+ * @experimental
+ */
+export type WatchedAttachmentItem =
+  | {
+      id: string;
+      filename: string;
+      fileExtension?: never;
+      metaData?: string;
+    }
+  | {
+      id: string;
+      fileExtension: string;
+      filename?: never;
+      metaData?: string;
+    };

package/src/db/schema/RawTable.ts

@@ -1,8 +1,23 @@
+import { TableOrRawTableOptions } from './Table.js';
+
 /**
- * A pending variant of a {@link RawTable} that doesn't have a name (because it would be inferred when creating the
- * schema).
+ * Instructs PowerSync to sync data into a "raw" table.
+ *
+ * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
+ * using client-side table and column constraints.
+ *
+ * To collect local writes to raw tables with PowerSync, custom triggers are required. See
+ * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
+ * using raw tables.
+ *
+ * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
+ *
+ * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
+ * stability guarantees.
  */
-export type RawTableType = {
+export type RawTableType = RawTableTypeWithStatements | InferredRawTableType;
+
+interface RawTableTypeWithStatements {
   /**
    * The statement to run when PowerSync detects that a row needs to be inserted or updated.
    */
@@ -11,7 +26,44 @@ export type RawTableType = {
    * The statement to run when PowerSync detects that a row needs to be deleted.
    */
   delete: PendingStatement;
-};
+
+  /**
+   * An optional statement to run when `disconnectAndClear()` is called on a PowerSync database.
+   */
+  clear?: string;
+}
+
+/**
+ * The schema of a {@link RawTableType} in the local database.
+ *
+ * This information is optional when declaring raw tables. However, providing it allows the sync client to infer `put`
+ * and `delete` statements automatically.
+ */
+interface RawTableSchema extends TableOrRawTableOptions {
+  /**
+   * The actual name of the raw table in the local schema.
+   *
+   * Unlike {@link RawTable.name}, which describes the name of synced tables to match, this reflects  the SQLite table
+   * name. This is used to infer {@link RawTableType.put} and {@link RawTableType.delete} statements for the sync
+   * client. It can also be used to auto-generate triggers forwarding writes on raw tables into the CRUD upload queue
+   * (using the `powersync_create_raw_table_crud_trigger` SQL function).
+   *
+   * When absent, defaults to {@link RawTable.name}.
+   */
+  tableName?: string;
+
+  /**
+   * An optional filter of columns that should be synced.
+   *
+   * By default, all columns in a raw table are considered for sync. If a filter is specified, PowerSync treats
+   * unmatched columns as local-only and will not attempt to sync them.
+   */
+  syncedColumns?: string[];
+}
+
+interface InferredRawTableType extends Partial<RawTableTypeWithStatements> {
+  schema: RawTableSchema;
+}
 
 /**
  * A parameter to use as part of {@link PendingStatement}.
@@ -21,8 +73,10 @@ export type RawTableType = {
  *
  * For insert and replace operations, the values of columns in the table are available as parameters through
  * `{Column: 'name'}`.
+ * The `"Rest"` parameter gets resolved to a JSON object covering all values from the synced row that haven't been
+ * covered by a `Column` parameter.
  */
-export type PendingStatementParameter = 'Id' | { Column: string };
+export type PendingStatementParameter = 'Id' | { Column: string } | 'Rest';
 
 /**
  * A statement that the PowerSync client should use to insert or delete data into a table managed by the user.
@@ -33,35 +87,16 @@ export type PendingStatement = {
 };
 
 /**
- * Instructs PowerSync to sync data into a "raw" table.
- *
- * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
- * using client-side table and column constraints.
- *
- * To collect local writes to raw tables with PowerSync, custom triggers are required. See
- * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
- * using raw tables.
- *
- * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
- *
- * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
- * stability guarantees.
+ * @internal
  */
-export class RawTable implements RawTableType {
+export type RawTable<T extends RawTableType = RawTableType> = T & {
   /**
    * The name of the table.
    *
-   * This does not have to match the actual table name in the schema - {@link put} and {@link delete} are free to use
-   * another table. Instead, this name is used by the sync client to recognize that operations on this table (as it
-   * appears in the source / backend database) are to be handled specially.
+   * This does not have to match the actual table name in the schema - {@link RawTableType.put} and
+   * {@link RawTableType.delete} are free to use another table. Instead, this name is used by the sync client to
+   * recognize that operations on this table (as it appears in the source / backend database) are to be handled
+   * specially.
    */
   name: string;
-  put: PendingStatement;
-  delete: PendingStatement;
-
-  constructor(name: string, type: RawTableType) {
-    this.name = name;
-    this.put = type.put;
-    this.delete = type.delete;
-  }
-}
+};

package/src/db/schema/Schema.ts

@@ -1,3 +1,4 @@
+import { encodeTableOptions } from './internal.js';
 import { RawTable, RawTableType } from './RawTable.js';
 import { RowType, Table } from './Table.js';
 
@@ -57,7 +58,7 @@ export class Schema<S extends SchemaType = SchemaType> {
    */
   withRawTables(tables: Record<string, RawTableType>) {
     for (const [name, rawTableDefinition] of Object.entries(tables)) {
-      this.rawTables.push(new RawTable(name, rawTableDefinition));
+      this.rawTables.push({ name, ...rawTableDefinition });
     }
   }
 
@@ -70,7 +71,31 @@ export class Schema<S extends SchemaType = SchemaType> {
   toJSON() {
     return {
       tables: this.tables.map((t) => t.toJSON()),
-      raw_tables: this.rawTables
+      raw_tables: this.rawTables.map(Schema.rawTableToJson)
     };
   }
+
+  /**
+   * Returns a representation of the raw table that is understood by the PowerSync SQLite core extension.
+   *
+   * The output of this can be passed through `JSON.serialize` and then used in `powersync_create_raw_table_crud_trigger`
+   * to define triggers for this table.
+   */
+  static rawTableToJson(table: RawTable): unknown {
+    const serialized: any = {
+      name: table.name,
+      put: table.put,
+      delete: table.delete,
+      clear: table.clear
+    };
+    if ('schema' in table) {
+      // We have schema options, those are flattened into the outer JSON object for the core extension.
+      const schema = table.schema;
+      serialized.table_name = schema.tableName ?? table.name;
+      serialized.synced_columns = schema.syncedColumns;
+      Object.assign(serialized, encodeTableOptions(table.schema));
+    }
+
+    return serialized;
+  }
 }

package/src/db/schema/Table.ts

@@ -8,17 +8,24 @@ import {
 } from './Column.js';
 import { Index } from './Index.js';
 import { IndexedColumn } from './IndexedColumn.js';
+import { encodeTableOptions } from './internal.js';
 import { TableV2 } from './TableV2.js';
 
-interface SharedTableOptions {
+/**
+ * Options that apply both to JSON-based tables and raw tables.
+ */
+export interface TableOrRawTableOptions {
   localOnly?: boolean;
   insertOnly?: boolean;
-  viewName?: string;
   trackPrevious?: boolean | TrackPreviousOptions;
   trackMetadata?: boolean;
   ignoreEmptyUpdates?: boolean;
 }
 
+interface SharedTableOptions extends TableOrRawTableOptions {
+  viewName?: string;
+}
+
 /** Whether to include previous column values when PowerSync tracks local changes.
  *
  * Including old values may be helpful for some backend connector implementations, which is
@@ -341,19 +348,12 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
   }
 
   toJSON() {
-    const trackPrevious = this.trackPrevious;
-
     return {
       name: this.name,
       view_name: this.viewName,
-      local_only: this.localOnly,
-      insert_only: this.insertOnly,
-      include_old: trackPrevious && ((trackPrevious as any).columns ?? true),
-      include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true,
-      include_metadata: this.trackMetadata,
-      ignore_empty_update: this.ignoreEmptyUpdates,
       columns: this.columns.map((c) => c.toJSON()),
-      indexes: this.indexes.map((e) => e.toJSON(this))
+      indexes: this.indexes.map((e) => e.toJSON(this)),
+      ...encodeTableOptions(this)
     };
   }
 }

package/src/db/schema/internal.ts

@@ -0,0 +1,17 @@
+import { TableOrRawTableOptions } from './Table.js';
+
+/**
+ * @internal Not exported from `index.ts`.
+ */
+export function encodeTableOptions(options: TableOrRawTableOptions) {
+  const trackPrevious = options.trackPrevious;
+
+  return {
+    local_only: options.localOnly,
+    insert_only: options.insertOnly,
+    include_old: trackPrevious && ((trackPrevious as any).columns ?? true),
+    include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true,
+    include_metadata: options.trackMetadata,
+    ignore_empty_update: options.ignoreEmptyUpdates
+  };
+}

package/src/index.ts

@@ -1,3 +1,13 @@
+export * from './attachments/AttachmentContext.js';
+export * from './attachments/AttachmentErrorHandler.js';
+export * from './attachments/AttachmentQueue.js';
+export * from './attachments/AttachmentService.js';
+export * from './attachments/LocalStorageAdapter.js';
+export * from './attachments/RemoteStorageAdapter.js';
+export * from './attachments/Schema.js';
+export * from './attachments/SyncingService.js';
+export * from './attachments/WatchedAttachmentItem.js';
+
 export * from './client/AbstractPowerSyncDatabase.js';
 export * from './client/AbstractPowerSyncOpenFactory.js';
 export { compilableQueryWatch, CompilableQueryWatchHandler } from './client/compilableQueryWatch.js';
@@ -29,7 +39,7 @@ export * from './db/DBAdapter.js';
 export * from './db/schema/Column.js';
 export * from './db/schema/Index.js';
 export * from './db/schema/IndexedColumn.js';
-export * from './db/schema/RawTable.js';
+export { RawTableType, PendingStatementParameter, PendingStatement } from './db/schema/RawTable.js';
 export * from './db/schema/Schema.js';
 export * from './db/schema/Table.js';
 export * from './db/schema/TableV2.js';
@@ -51,6 +61,7 @@ export * from './utils/BaseObserver.js';
 export * from './utils/ControlledExecutor.js';
 export * from './utils/DataStream.js';
 export * from './utils/Logger.js';
+export * from './utils/mutex.js';
 export * from './utils/parseQuery.js';
 
 export * from './types/types.js';

package/src/utils/mutex.ts

@@ -6,7 +6,7 @@ import { Mutex } from 'async-mutex';
 export async function mutexRunExclusive<T>(
   mutex: Mutex,
   callback: () => Promise<T>,
-  options?: { timeoutMs: number }
+  options?: { timeoutMs?: number }
 ): Promise<T> {
   return new Promise((resolve, reject) => {
     const timeout = options?.timeoutMs;