@powersync/common 1.45.0 → 1.47.0

package/src/client/triggers/TriggerManagerImpl.ts

@@ -1,24 +1,62 @@
 import { LockContext } from '../../db/DBAdapter.js';
 import { Schema } from '../../db/schema/Schema.js';
-import { type AbstractPowerSyncDatabase } from '../AbstractPowerSyncDatabase.js';
+import type { AbstractPowerSyncDatabase } from '../AbstractPowerSyncDatabase.js';
 import { DEFAULT_WATCH_THROTTLE_MS } from '../watched/WatchedQuery.js';
 import {
   CreateDiffTriggerOptions,
   DiffTriggerOperation,
   TrackDiffOptions,
   TriggerManager,
+  TriggerManagerConfig,
   TriggerRemoveCallback,
   WithDiffOptions
 } from './TriggerManager.js';
 
-export type TriggerManagerImplOptions = {
+export type TriggerManagerImplOptions = TriggerManagerConfig & {
   db: AbstractPowerSyncDatabase;
   schema: Schema;
 };
 
+export type TriggerManagerImplConfiguration = {
+  useStorageByDefault: boolean;
+};
+
+export const DEFAULT_TRIGGER_MANAGER_CONFIGURATION: TriggerManagerImplConfiguration = {
+  useStorageByDefault: false
+};
+
+/**
+ * A record of persisted table/trigger information.
+ * This is used for fail-safe cleanup.
+ */
+type TrackedTableRecord = {
+  /**
+   * The id of the trigger. This is used in the SQLite trigger name
+   */
+  id: string;
+  /**
+   * The destination table name for the trigger
+   */
+  table: string;
+  /**
+   * Array of actual trigger names found for this table/id combo
+   */
+  triggerNames: string[];
+};
+
+const TRIGGER_CLEANUP_INTERVAL_MS = 120_000; // 2 minutes
+
+/**
+ * @internal
+ * @experimental
+ */
 export class TriggerManagerImpl implements TriggerManager {
   protected schema: Schema;
 
+  protected defaultConfig: TriggerManagerImplConfiguration;
+  protected cleanupTimeout: ReturnType<typeof setTimeout> | null;
+  protected isDisposed: boolean;
+
   constructor(protected options: TriggerManagerImplOptions) {
     this.schema = options.schema;
     options.db.registerListener({
@@ -26,6 +64,32 @@ export class TriggerManagerImpl implements TriggerManager {
         this.schema = schema;
       }
     });
+    this.isDisposed = false;
+
+    /**
+     * Configure a cleanup to run on an interval.
+     * The interval is configured using setTimeout to take the async
+     * execution time of the callback into account.
+     */
+    this.defaultConfig = DEFAULT_TRIGGER_MANAGER_CONFIGURATION;
+    const cleanupCallback = async () => {
+      this.cleanupTimeout = null;
+      if (this.isDisposed) {
+        return;
+      }
+      try {
+        await this.cleanupResources();
+      } catch (ex) {
+        this.db.logger.error(`Caught error while attempting to cleanup triggers`, ex);
+      } finally {
+        // if not closed, set another timeout
+        if (this.isDisposed) {
+          return;
+        }
+        this.cleanupTimeout = setTimeout(cleanupCallback, TRIGGER_CLEANUP_INTERVAL_MS);
+      }
+    };
+    this.cleanupTimeout = setTimeout(cleanupCallback, TRIGGER_CLEANUP_INTERVAL_MS);
   }
 
   protected get db() {
@@ -48,14 +112,111 @@ export class TriggerManagerImpl implements TriggerManager {
     }
   }
 
+  dispose() {
+    this.isDisposed = true;
+    if (this.cleanupTimeout) {
+      clearTimeout(this.cleanupTimeout);
+    }
+  }
+
+  /**
+   * Updates default config settings for platform specific use-cases.
+   */
+  updateDefaults(config: TriggerManagerImplConfiguration) {
+    this.defaultConfig = {
+      ...this.defaultConfig,
+      ...config
+    };
+  }
+
+  protected generateTriggerName(operation: DiffTriggerOperation, destinationTable: string, triggerId: string) {
+    return `__ps_temp_trigger_${operation.toLowerCase()}__${destinationTable}__${triggerId}`;
+  }
+
+  /**
+   * Cleanup any SQLite triggers or tables that are no longer in use.
+   */
+  async cleanupResources() {
+    // we use the database here since cleanupResources is called during the PowerSyncDatabase initialization
+    await this.db.database.writeLock(async (ctx) => {
+      /**
+       * Note: We only cleanup persisted triggers. These are tracked in the sqlite_master table.
+       * temporary triggers will not be affected by this.
+       * Query all triggers that match our naming pattern
+       */
+      const triggers = await ctx.getAll<{ name: string }>(/* sql */ `
+        SELECT
+          name
+        FROM
+          sqlite_master
+        WHERE
+          type = 'trigger'
+          AND name LIKE '__ps_temp_trigger_%'
+      `);
+
+      /** Use regex to extract table names and IDs from trigger names
+       * Trigger naming convention: __ps_temp_trigger_<operation>__<destination_table>__<id>
+       */
+      const triggerPattern = /^__ps_temp_trigger_(?:insert|update|delete)__(.+)__([a-f0-9_]{36})$/i;
+      const trackedItems = new Map<string, TrackedTableRecord>();
+
+      for (const trigger of triggers) {
+        const match = trigger.name.match(triggerPattern);
+        if (match) {
+          const [, table, id] = match;
+          // Collect all trigger names for each id combo
+          const existing = trackedItems.get(id);
+          if (existing) {
+            existing.triggerNames.push(trigger.name);
+          } else {
+            trackedItems.set(id, { table, id, triggerNames: [trigger.name] });
+          }
+        }
+      }
+
+      for (const trackedItem of trackedItems.values()) {
+        // check if there is anything holding on to this item
+        const hasClaim = await this.options.claimManager.checkClaim(trackedItem.id);
+        if (hasClaim) {
+          // This does not require cleanup
+          continue;
+        }
+
+        this.db.logger.debug(`Clearing resources for trigger ${trackedItem.id} with table ${trackedItem.table}`);
+
+        // We need to delete the triggers and table
+        for (const triggerName of trackedItem.triggerNames) {
+          await ctx.execute(`DROP TRIGGER IF EXISTS ${triggerName}`);
+        }
+        await ctx.execute(`DROP TABLE IF EXISTS ${trackedItem.table}`);
+      }
+    });
+  }
+
   async createDiffTrigger(options: CreateDiffTriggerOptions) {
     await this.db.waitForReady();
-    const { source, destination, columns, when, hooks } = options;
+    const {
+      source,
+      destination,
+      columns,
+      when,
+      hooks,
+      // Fall back to the provided default if not given on this level
+      useStorage = this.defaultConfig.useStorageByDefault
+    } = options;
     const operations = Object.keys(when) as DiffTriggerOperation[];
     if (operations.length == 0) {
       throw new Error('At least one WHEN operation must be specified for the trigger.');
     }
 
+    /**
+     * The clause to use when executing
+     * CREATE ${tableTriggerTypeClause} TABLE
+     * OR
+     * CREATE ${tableTriggerTypeClause} TRIGGER
+     */
+    const tableTriggerTypeClause = !useStorage ? 'TEMP' : '';
+
     const whenClauses = Object.fromEntries(
       Object.entries(when).map(([operation, filter]) => [operation, `WHEN ${filter}`])
     );
@@ -76,6 +237,8 @@ export class TriggerManagerImpl implements TriggerManager {
 
     const id = await this.getUUID();
 
+    const releaseStorageClaim = useStorage ? await this.options.claimManager.obtainClaim(id) : null;
+
     /**
      * We default to replicating all columns if no columns array is provided.
      */
@@ -110,6 +273,7 @@ export class TriggerManagerImpl implements TriggerManager {
       return this.db.writeLock(async (tx) => {
         await this.removeTriggers(tx, triggerIds);
         await tx.execute(/* sql */ `DROP TABLE IF EXISTS ${destination};`);
+        await releaseStorageClaim?.();
       });
     };
 
@@ -117,22 +281,22 @@ export class TriggerManagerImpl implements TriggerManager {
       // Allow user code to execute in this lock context before the trigger is created.
       await hooks?.beforeCreate?.(tx);
       await tx.execute(/* sql */ `
-        CREATE TEMP TABLE ${destination} (
+        CREATE ${tableTriggerTypeClause} TABLE ${destination} (
           operation_id INTEGER PRIMARY KEY AUTOINCREMENT,
           id TEXT,
           operation TEXT,
           timestamp TEXT,
           value TEXT,
           previous_value TEXT
-        );
+        )
       `);
 
       if (operations.includes(DiffTriggerOperation.INSERT)) {
-        const insertTriggerId = `ps_temp_trigger_insert_${id}`;
+        const insertTriggerId = this.generateTriggerName(DiffTriggerOperation.INSERT, destination, id);
         triggerIds.push(insertTriggerId);
 
         await tx.execute(/* sql */ `
-          CREATE TEMP TRIGGER ${insertTriggerId} AFTER INSERT ON ${internalSource} ${whenClauses[
+          CREATE ${tableTriggerTypeClause} TRIGGER ${insertTriggerId} AFTER INSERT ON ${internalSource} ${whenClauses[
             DiffTriggerOperation.INSERT
           ]} BEGIN
           INSERT INTO
@@ -145,16 +309,16 @@ export class TriggerManagerImpl implements TriggerManager {
               ${jsonFragment('NEW')}
             );
 
-          END;
+          END
         `);
       }
 
       if (operations.includes(DiffTriggerOperation.UPDATE)) {
-        const updateTriggerId = `ps_temp_trigger_update_${id}`;
+        const updateTriggerId = this.generateTriggerName(DiffTriggerOperation.UPDATE, destination, id);
         triggerIds.push(updateTriggerId);
 
         await tx.execute(/* sql */ `
-          CREATE TEMP TRIGGER ${updateTriggerId} AFTER
+          CREATE ${tableTriggerTypeClause} TRIGGER ${updateTriggerId} AFTER
           UPDATE ON ${internalSource} ${whenClauses[DiffTriggerOperation.UPDATE]} BEGIN
           INSERT INTO
             ${destination} (id, operation, timestamp, value, previous_value)
@@ -172,12 +336,12 @@ export class TriggerManagerImpl implements TriggerManager {
       }
 
       if (operations.includes(DiffTriggerOperation.DELETE)) {
-        const deleteTriggerId = `ps_temp_trigger_delete_${id}`;
+        const deleteTriggerId = this.generateTriggerName(DiffTriggerOperation.DELETE, destination, id);
         triggerIds.push(deleteTriggerId);
 
         // Create delete trigger for basic JSON
         await tx.execute(/* sql */ `
-          CREATE TEMP TRIGGER ${deleteTriggerId} AFTER DELETE ON ${internalSource} ${whenClauses[
+          CREATE ${tableTriggerTypeClause} TRIGGER ${deleteTriggerId} AFTER DELETE ON ${internalSource} ${whenClauses[
             DiffTriggerOperation.DELETE
           ]} BEGIN
           INSERT INTO
@@ -227,7 +391,7 @@ export class TriggerManagerImpl implements TriggerManager {
     const contextColumns = columns ?? sourceDefinition.columns.map((col) => col.name);
 
     const id = await this.getUUID();
-    const destination = `ps_temp_track_${source}_${id}`;
+    const destination = `__ps_temp_track_${source}_${id}`;
 
     // register an onChange before the trigger is created
     const abortController = new AbortController();

package/src/index.ts

@@ -1,3 +1,13 @@
+export * from './attachments/AttachmentContext.js';
+export * from './attachments/AttachmentErrorHandler.js';
+export * from './attachments/AttachmentQueue.js';
+export * from './attachments/AttachmentService.js';
+export * from './attachments/LocalStorageAdapter.js';
+export * from './attachments/RemoteStorageAdapter.js';
+export * from './attachments/Schema.js';
+export * from './attachments/SyncingService.js';
+export * from './attachments/WatchedAttachmentItem.js';
+
 export * from './client/AbstractPowerSyncDatabase.js';
 export * from './client/AbstractPowerSyncOpenFactory.js';
 export { compilableQueryWatch, CompilableQueryWatchHandler } from './client/compilableQueryWatch.js';
@@ -29,13 +39,16 @@ export * from './db/DBAdapter.js';
 export * from './db/schema/Column.js';
 export * from './db/schema/Index.js';
 export * from './db/schema/IndexedColumn.js';
+export * from './db/schema/RawTable.js';
 export * from './db/schema/Schema.js';
 export * from './db/schema/Table.js';
 export * from './db/schema/TableV2.js';
 
 export * from './client/Query.js';
+export { MEMORY_TRIGGER_CLAIM_MANAGER } from './client/triggers/MemoryTriggerClaimManager.js';
 export * from './client/triggers/sanitizeSQL.js';
 export * from './client/triggers/TriggerManager.js';
+export { TriggerManagerImpl } from './client/triggers/TriggerManagerImpl.js';
 export * from './client/watched/GetAllQuery.js';
 export * from './client/watched/processors/AbstractQueryProcessor.js';
 export * from './client/watched/processors/comparators.js';
@@ -48,6 +61,7 @@ export * from './utils/BaseObserver.js';
 export * from './utils/ControlledExecutor.js';
 export * from './utils/DataStream.js';
 export * from './utils/Logger.js';
+export * from './utils/mutex.js';
 export * from './utils/parseQuery.js';
 
 export * from './types/types.js';

package/src/utils/DataStream.ts

@@ -110,6 +110,18 @@ export class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataS
       return null;
     }
 
+    // Wait for any pending processing to complete first.
+    // This ensures we register our listener before calling processQueue(),
+    // avoiding a race where processQueue() sees no reader and returns early.
+    if (this.processingPromise) {
+      await this.processingPromise;
+    }
+
+    // Re-check after await - stream may have closed while we were waiting
+    if (this.closed) {
+      return null;
+    }
+
     return new Promise((resolve, reject) => {
       const l = this.registerListener({
         data: async (data) => {
@@ -151,7 +163,7 @@ export class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataS
 
     const promise = (this.processingPromise = this._processQueue());
     promise.finally(() => {
-      return (this.processingPromise = null);
+      this.processingPromise = null;
     });
     return promise;
   }
@@ -190,7 +202,6 @@ export class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataS
     }
 
     if (this.dataQueue.length > 0) {
-      // Next tick
       setTimeout(() => this.processQueue());
     }
   }

package/src/utils/mutex.ts

@@ -6,7 +6,7 @@ import { Mutex } from 'async-mutex';
 export async function mutexRunExclusive<T>(
   mutex: Mutex,
   callback: () => Promise<T>,
-  options?: { timeoutMs: number }
+  options?: { timeoutMs?: number }
 ): Promise<T> {
   return new Promise((resolve, reject) => {
     const timeout = options?.timeoutMs;