@powersync/web 0.0.0-dev-20251201150812 → 0.0.0-dev-20251203144301

package/src/worker/db/opfs.ts

@@ -0,0 +1,623 @@
+// Copyright 2024 Roy T. Hashimoto. All Rights Reserved.
+// @ts-nocheck
+import { FacadeVFS } from '@journeyapps/wa-sqlite/src/FacadeVFS.js';
+import * as VFS from '@journeyapps/wa-sqlite/src/VFS.js';
+
+const DEFAULT_TEMPORARY_FILES = 10;
+const LOCK_NOTIFY_INTERVAL = 1000;
+
+const DB_RELATED_FILE_SUFFIXES = ['', '-journal', '-wal'];
+
+const finalizationRegistry = new FinalizationRegistry((releaser) => releaser());
+
+class File {
+  /** @type {string} */ path;
+  /** @type {number} */ flags;
+  /** @type {FileSystemSyncAccessHandle} */ accessHandle;
+
+  /** @type {PersistentFile?} */ persistentFile;
+
+  constructor(path, flags) {
+    this.path = path;
+    this.flags = flags;
+  }
+}
+
+class PersistentFile {
+  /** @type {FileSystemFileHandle} */ fileHandle;
+  /** @type {FileSystemSyncAccessHandle} */ accessHandle = null;
+
+  // The following properties are for main database files.
+
+  /** @type {boolean} */ isLockBusy = false;
+  /** @type {boolean} */ isFileLocked = false;
+  /** @type {boolean} */ isRequestInProgress = false;
+  /** @type {function} */ handleLockReleaser = null;
+
+  /** @type {BroadcastChannel} */ handleRequestChannel;
+  /** @type {boolean} */ isHandleRequested = false;
+
+  constructor(fileHandle) {
+    this.fileHandle = fileHandle;
+  }
+}
+
+export class OPFSCoopSyncVFS extends FacadeVFS {
+  /** @type {Map<number, File>} */ mapIdToFile = new Map();
+
+  lastError = null;
+  log = null; //function(...args) { console.log(`[${contextName}]`, ...args) };
+
+  /** @type {Map<string, PersistentFile>} */ persistentFiles = new Map();
+  /** @type {Map<string, FileSystemSyncAccessHandle>} */ boundAccessHandles = new Map();
+  /** @type {Set<FileSystemSyncAccessHandle>} */ unboundAccessHandles = new Set();
+  /** @type {Set<string>} */ accessiblePaths = new Set();
+  releaser = null;
+
+  static async create(name, module) {
+    const vfs = new OPFSCoopSyncVFS(name, module);
+    await Promise.all([vfs.isReady(), vfs.#initialize(DEFAULT_TEMPORARY_FILES)]);
+    return vfs;
+  }
+
+  constructor(name, module) {
+    super(name, module);
+  }
+
+  async #initialize(nTemporaryFiles) {
+    // Delete temporary directories no longer in use.
+    const root = await navigator.storage.getDirectory();
+    // @ts-ignore
+    for await (const entry of root.values()) {
+      if (entry.kind === 'directory' && entry.name.startsWith('.ahp-')) {
+        // A lock with the same name as the directory protects it from
+        // being deleted.
+        await navigator.locks.request(entry.name, { ifAvailable: true }, async (lock) => {
+          if (lock) {
+            this.log?.(`Deleting temporary directory ${entry.name}`);
+            await root.removeEntry(entry.name, { recursive: true });
+          } else {
+            this.log?.(`Temporary directory ${entry.name} is in use`);
+          }
+        });
+      }
+    }
+
+    // Create our temporary directory.
+    const tmpDirName = `.ahp-${Math.random().toString(36).slice(2)}`;
+    this.releaser = await new Promise((resolve) => {
+      navigator.locks.request(tmpDirName, () => {
+        return new Promise((release) => {
+          resolve(release);
+        });
+      });
+    });
+
+    const releaseHandle = async () => {
+      await Promise.all(
+        this.persistentFiles.values().map(async (file) => {
+          try {
+            await this.#releaseAccessHandle(file);
+          } catch (e) {
+            this.log?.('error releasing access handle', e);
+          } finally {
+            release();
+          }
+        })
+      );
+    };
+
+    finalizationRegistry.register(this, releaseHandle);
+    finalizationRegistry.register(this, this.releaser);
+    const tmpDir = await root.getDirectoryHandle(tmpDirName, { create: true });
+
+    // Populate temporary directory.
+    for (let i = 0; i < nTemporaryFiles; i++) {
+      const tmpFile = await tmpDir.getFileHandle(`${i}.tmp`, { create: true });
+      const tmpAccessHandle = await tmpFile.createSyncAccessHandle();
+      this.unboundAccessHandles.add(tmpAccessHandle);
+    }
+  }
+
+  /**
+   * @param {string?} zName
+   * @param {number} fileId
+   * @param {number} flags
+   * @param {DataView} pOutFlags
+   * @returns {number}
+   */
+  jOpen(zName, fileId, flags, pOutFlags) {
+    try {
+      const url = new URL(zName || Math.random().toString(36).slice(2), 'file://');
+      const path = url.pathname;
+
+      if (flags & VFS.SQLITE_OPEN_MAIN_DB) {
+        const persistentFile = this.persistentFiles.get(path);
+        if (persistentFile?.isRequestInProgress) {
+          // Should not reach here unless SQLite itself retries an open.
+          // Otherwise, asynchronous operations started on a previous
+          // open try should have completed.
+          return VFS.SQLITE_BUSY;
+        } else if (!persistentFile) {
+          // This is the usual starting point for opening a database.
+          // Register a Promise that resolves when the database and related
+          // files are ready to be used.
+          this.log?.(`creating persistent file for ${path}`);
+          const create = !!(flags & VFS.SQLITE_OPEN_CREATE);
+          this._module.retryOps.push(
+            (async () => {
+              try {
+                // Get the path directory handle.
+                let dirHandle = await navigator.storage.getDirectory();
+                const directories = path.split('/').filter((d) => d);
+                const filename = directories.pop();
+                for (const directory of directories) {
+                  dirHandle = await dirHandle.getDirectoryHandle(directory, { create });
+                }
+
+                // Get file handles for the database and related files,
+                // and create persistent file instances.
+                for (const suffix of DB_RELATED_FILE_SUFFIXES) {
+                  const fileHandle = await dirHandle.getFileHandle(filename + suffix, { create });
+                  await this.#createPersistentFile(fileHandle);
+                }
+
+                // Get access handles for the files.
+                const file = new File(path, flags);
+                file.persistentFile = this.persistentFiles.get(path);
+                await this.#requestAccessHandle(file);
+              } catch (e) {
+                // Use an invalid persistent file to signal this error
+                // for the retried open.
+                const persistentFile = new PersistentFile(null);
+                this.persistentFiles.set(path, persistentFile);
+                console.error(e);
+              }
+            })()
+          );
+          return VFS.SQLITE_BUSY;
+        } else if (!persistentFile.fileHandle) {
+          // The asynchronous open operation failed.
+          this.persistentFiles.delete(path);
+          return VFS.SQLITE_CANTOPEN;
+        } else if (!persistentFile.accessHandle) {
+          // This branch is reached if the database was previously opened
+          // and closed.
+          this._module.retryOps.push(
+            (async () => {
+              const file = new File(path, flags);
+              file.persistentFile = this.persistentFiles.get(path);
+              await this.#requestAccessHandle(file);
+            })()
+          );
+          return VFS.SQLITE_BUSY;
+        }
+      }
+
+      if (!this.accessiblePaths.has(path) && !(flags & VFS.SQLITE_OPEN_CREATE)) {
+        throw new Error(`File ${path} not found`);
+      }
+
+      const file = new File(path, flags);
+      this.mapIdToFile.set(fileId, file);
+
+      if (this.persistentFiles.has(path)) {
+        file.persistentFile = this.persistentFiles.get(path);
+      } else if (this.boundAccessHandles.has(path)) {
+        // This temporary file was previously created and closed. Reopen
+        // the same access handle.
+        file.accessHandle = this.boundAccessHandles.get(path);
+      } else if (this.unboundAccessHandles.size) {
+        // Associate an unbound access handle to this file.
+        file.accessHandle = this.unboundAccessHandles.values().next().value;
+        file.accessHandle.truncate(0);
+        this.unboundAccessHandles.delete(file.accessHandle);
+        this.boundAccessHandles.set(path, file.accessHandle);
+      }
+      this.accessiblePaths.add(path);
+
+      pOutFlags.setInt32(0, flags, true);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_CANTOPEN;
+    }
+  }
+
+  /**
+   * @param {string} zName
+   * @param {number} syncDir
+   * @returns {number}
+   */
+  jDelete(zName, syncDir) {
+    try {
+      const url = new URL(zName, 'file://');
+      const path = url.pathname;
+      if (this.persistentFiles.has(path)) {
+        const persistentFile = this.persistentFiles.get(path);
+        persistentFile.accessHandle.truncate(0);
+      } else {
+        this.boundAccessHandles.get(path)?.truncate(0);
+      }
+      this.accessiblePaths.delete(path);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_DELETE;
+    }
+  }
+
+  /**
+   * @param {string} zName
+   * @param {number} flags
+   * @param {DataView} pResOut
+   * @returns {number}
+   */
+  jAccess(zName, flags, pResOut) {
+    try {
+      const url = new URL(zName, 'file://');
+      const path = url.pathname;
+      pResOut.setInt32(0, this.accessiblePaths.has(path) ? 1 : 0, true);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_ACCESS;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @returns {number}
+   */
+  jClose(fileId) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      this.mapIdToFile.delete(fileId);
+
+      if (file?.flags & VFS.SQLITE_OPEN_MAIN_DB) {
+        if (file.persistentFile?.accessHandle) {
+          this.#releaseAccessHandle(file);
+        }
+      } else if (file?.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
+        file.accessHandle.truncate(0);
+        this.accessiblePaths.delete(file.path);
+        if (!this.persistentFiles.has(file.path)) {
+          this.boundAccessHandles.delete(file.path);
+          this.unboundAccessHandles.add(file.accessHandle);
+        }
+      }
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_CLOSE;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {Uint8Array} pData
+   * @param {number} iOffset
+   * @returns {number}
+   */
+  jRead(fileId, pData, iOffset) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+
+      // On Chrome (at least), passing pData to accessHandle.read() is
+      // an error because pData is a Proxy of a Uint8Array. Calling
+      // subarray() produces a real Uint8Array and that works.
+      const accessHandle = file.accessHandle || file.persistentFile.accessHandle;
+      const bytesRead = accessHandle.read(pData.subarray(), { at: iOffset });
+
+      // Opening a database file performs one read without a xLock call.
+      if (file.flags & VFS.SQLITE_OPEN_MAIN_DB && !file.persistentFile.isFileLocked) {
+        this.#releaseAccessHandle(file);
+      }
+
+      if (bytesRead < pData.byteLength) {
+        pData.fill(0, bytesRead);
+        return VFS.SQLITE_IOERR_SHORT_READ;
+      }
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_READ;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {Uint8Array} pData
+   * @param {number} iOffset
+   * @returns {number}
+   */
+  jWrite(fileId, pData, iOffset) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+
+      // On Chrome (at least), passing pData to accessHandle.write() is
+      // an error because pData is a Proxy of a Uint8Array. Calling
+      // subarray() produces a real Uint8Array and that works.
+      const accessHandle = file.accessHandle || file.persistentFile.accessHandle;
+      const nBytes = accessHandle.write(pData.subarray(), { at: iOffset });
+      if (nBytes !== pData.byteLength) throw new Error('short write');
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_WRITE;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {number} iSize
+   * @returns {number}
+   */
+  jTruncate(fileId, iSize) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      const accessHandle = file.accessHandle || file.persistentFile.accessHandle;
+      accessHandle.truncate(iSize);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_TRUNCATE;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {number} flags
+   * @returns {number}
+   */
+  jSync(fileId, flags) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      const accessHandle = file.accessHandle || file.persistentFile.accessHandle;
+      accessHandle.flush();
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_FSYNC;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {DataView} pSize64
+   * @returns {number}
+   */
+  jFileSize(fileId, pSize64) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      const accessHandle = file.accessHandle || file.persistentFile.accessHandle;
+      const size = accessHandle.getSize();
+      pSize64.setBigInt64(0, BigInt(size), true);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_FSTAT;
+    }
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {number} lockType
+   * @returns {number}
+   */
+  jLock(fileId, lockType) {
+    const file = this.mapIdToFile.get(fileId);
+    if (file.persistentFile.isRequestInProgress) {
+      file.persistentFile.isLockBusy = true;
+      return VFS.SQLITE_BUSY;
+    }
+
+    file.persistentFile.isFileLocked = true;
+    if (!file.persistentFile.handleLockReleaser) {
+      // Start listening for notifications from other connections.
+      // This is before we actually get access handles, but waiting to
+      // listen until then allows a race condition where notifications
+      // are missed.
+      file.persistentFile.handleRequestChannel.onmessage = () => {
+        this.log?.(`received notification for ${file.path}`);
+        if (file.persistentFile.isFileLocked) {
+          // We're still using the access handle, so mark it to be
+          // released when we're done.
+          file.persistentFile.isHandleRequested = true;
+        } else {
+          // Release the access handles immediately.
+          this.#releaseAccessHandle(file);
+        }
+        file.persistentFile.handleRequestChannel.onmessage = null;
+      };
+
+      this.#requestAccessHandle(file);
+      this.log?.('returning SQLITE_BUSY');
+      file.persistentFile.isLockBusy = true;
+      return VFS.SQLITE_BUSY;
+    }
+    file.persistentFile.isLockBusy = false;
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {number} lockType
+   * @returns {number}
+   */
+  jUnlock(fileId, lockType) {
+    const file = this.mapIdToFile.get(fileId);
+    if (lockType === VFS.SQLITE_LOCK_NONE) {
+      // Don't change any state if this unlock is because xLock returned
+      // SQLITE_BUSY.
+      if (!file.persistentFile.isLockBusy) {
+        if (file.persistentFile.isHandleRequested) {
+          // Another connection wants the access handle.
+          this.#releaseAccessHandle(file);
+          file.persistentFile.isHandleRequested = false;
+        }
+        file.persistentFile.isFileLocked = false;
+      }
+    }
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {number} op
+   * @param {DataView} pArg
+   * @returns {number|Promise<number>}
+   */
+  jFileControl(fileId, op, pArg) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      switch (op) {
+        case VFS.SQLITE_FCNTL_PRAGMA:
+          const key = extractString(pArg, 4);
+          const value = extractString(pArg, 8);
+          this.log?.('xFileControl', file.path, 'PRAGMA', key, value);
+          switch (key.toLowerCase()) {
+            case 'journal_mode':
+              if (value && !['off', 'memory', 'delete', 'wal'].includes(value.toLowerCase())) {
+                throw new Error('journal_mode must be "off", "memory", "delete", or "wal"');
+              }
+              break;
+          }
+          break;
+      }
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR;
+    }
+    return VFS.SQLITE_NOTFOUND;
+  }
+
+  /**
+   * @param {Uint8Array} zBuf
+   * @returns
+   */
+  jGetLastError(zBuf) {
+    if (this.lastError) {
+      console.error(this.lastError);
+      const outputArray = zBuf.subarray(0, zBuf.byteLength - 1);
+      const { written } = new TextEncoder().encodeInto(this.lastError.message, outputArray);
+      zBuf[written] = 0;
+    }
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {FileSystemFileHandle} fileHandle
+   * @returns {Promise<PersistentFile>}
+   */
+  async #createPersistentFile(fileHandle) {
+    const persistentFile = new PersistentFile(fileHandle);
+    const root = await navigator.storage.getDirectory();
+    const relativePath = await root.resolve(fileHandle);
+    const path = `/${relativePath.join('/')}`;
+    persistentFile.handleRequestChannel = new BroadcastChannel(`ahp:${path}`);
+    this.persistentFiles.set(path, persistentFile);
+
+    const f = await fileHandle.getFile();
+    if (f.size) {
+      this.accessiblePaths.add(path);
+    }
+    return persistentFile;
+  }
+
+  /**
+   * @param {File} file
+   */
+  #requestAccessHandle(file) {
+    console.assert(!file.persistentFile.handleLockReleaser);
+    if (!file.persistentFile.isRequestInProgress) {
+      file.persistentFile.isRequestInProgress = true;
+      this._module.retryOps.push(
+        (async () => {
+          // Acquire the Web Lock.
+          file.persistentFile.handleLockReleaser = await this.#acquireLock(file.persistentFile);
+          try {
+            // Get access handles for the database and releated files in parallel.
+            this.log?.(`creating access handles for ${file.path}`);
+            await Promise.all(
+              DB_RELATED_FILE_SUFFIXES.map(async (suffix) => {
+                const persistentFile = this.persistentFiles.get(file.path + suffix);
+                if (persistentFile) {
+                  persistentFile.accessHandle = await persistentFile.fileHandle.createSyncAccessHandle();
+                }
+              })
+            );
+          } catch (e) {
+            this.log?.(`failed to create access handles for ${file.path}`, e);
+            // Close any of the potentially opened access handles
+            DB_RELATED_FILE_SUFFIXES.forEach(async (suffix) => {
+              const persistentFile = this.persistentFiles.get(file.path + suffix);
+              if (persistentFile) {
+                persistentFile.accessHandle?.close();
+              }
+            });
+            // Release the lock, if we failed here, we'd need to obtain the lock later in order to retry
+            file.persistentFile.handleLockReleaser();
+            throw e;
+          } finally {
+            file.persistentFile.isRequestInProgress = false;
+          }
+        })()
+      );
+      return this._module.retryOps.at(-1);
+    }
+    return Promise.resolve();
+  }
+
+  /**
+   * @param {File} file
+   */
+  async #releaseAccessHandle(file) {
+    DB_RELATED_FILE_SUFFIXES.forEach(async (suffix) => {
+      const persistentFile = this.persistentFiles.get(file.path + suffix);
+      if (persistentFile) {
+        persistentFile.accessHandle?.close();
+        persistentFile.accessHandle = null;
+      }
+    });
+    this.log?.(`access handles closed for ${file.path}`);
+
+    file.persistentFile.handleLockReleaser?.();
+    file.persistentFile.handleLockReleaser = null;
+    this.log?.(`lock released for ${file.path}`);
+  }
+
+  /**
+   * @param {PersistentFile} persistentFile
+   * @returns  {Promise<function>} lock releaser
+   */
+  #acquireLock(persistentFile) {
+    return new Promise((resolve) => {
+      // Tell other connections we want the access handle.
+      const lockName = persistentFile.handleRequestChannel.name;
+      const notify = () => {
+        this.log?.(`notifying for ${lockName}`);
+        persistentFile.handleRequestChannel.postMessage(null);
+      };
+      const notifyId = setInterval(notify, LOCK_NOTIFY_INTERVAL);
+      setTimeout(notify);
+
+      this.log?.(`lock requested: ${lockName}`);
+      navigator.locks.request(lockName, (lock) => {
+        // We have the lock. Stop asking other connections for it.
+        this.log?.(`lock acquired: ${lockName}`, lock);
+        clearInterval(notifyId);
+        return new Promise(resolve);
+      });
+    });
+  }
+}
+
+function extractString(dataView, offset) {
+  const p = dataView.getUint32(offset, true);
+  if (p) {
+    const chars = new Uint8Array(dataView.buffer, p);
+    return new TextDecoder().decode(chars.subarray(0, chars.indexOf(0)));
+  }
+  return null;
+}

package/src/worker/sync/SharedSyncImplementation.ts

@@ -525,12 +525,16 @@ export class SharedSyncImplementation extends BaseObserver<SharedSyncImplementat
 
         client.closeListeners.push(closeListener);
 
-        const workerPort = await withAbort(() => client.clientProvider.getDBWorkerPort(), abortController.signal).catch(
-          (ex) => {
-            removeCloseListener();
-            throw ex;
+        const workerPort = await withAbort({
+          action: () => client.clientProvider.getDBWorkerPort(),
+          signal: abortController.signal,
+          cleanupOnAbort: (port) => {
+            port.close();
           }
-        );
+        }).catch((ex) => {
+          removeCloseListener();
+          throw ex;
+        });
 
         const remote = Comlink.wrap<OpenAsyncDatabaseConnection>(workerPort);
         const identifier = this.syncParams!.dbParams.dbFilename;
@@ -541,7 +545,13 @@ export class SharedSyncImplementation extends BaseObserver<SharedSyncImplementat
          * We typically execute the closeListeners using the portMutex in a different context.
          * We can't rely on the closeListeners to abort the operation if the tab is closed.
          */
-        const db = await withAbort(() => remote(this.syncParams!.dbParams), abortController.signal).finally(() => {
+        const db = await withAbort({
+          action: () => remote(this.syncParams!.dbParams),
+          signal: abortController.signal,
+          cleanupOnAbort: (db) => {
+            db.close();
+          }
+        }).finally(() => {
           // We can remove the close listener here since we no longer need it past this point.
           removeCloseListener();
         });
@@ -588,7 +598,12 @@ export class SharedSyncImplementation extends BaseObserver<SharedSyncImplementat
 /**
  * Runs the action with an abort controller.
  */
-function withAbort<T>(action: () => Promise<T>, signal: AbortSignal): Promise<T> {
+function withAbort<T>(options: {
+  action: () => Promise<T>;
+  signal: AbortSignal;
+  cleanupOnAbort?: (result: T) => void;
+}): Promise<T> {
+  const { action, signal, cleanupOnAbort } = options;
   return new Promise((resolve, reject) => {
     if (signal.aborted) {
       reject(new AbortOperation('Operation aborted by abort controller'));
@@ -608,7 +623,13 @@ function withAbort<T>(action: () => Promise<T>, signal: AbortSignal): Promise<T>
     }
 
     action()
-      .then((data) => completePromise(() => resolve(data)))
+      .then((data) => {
+        // We already rejected due to the abort, allow for cleanup
+        if (signal.aborted) {
+          return completePromise(() => cleanupOnAbort?.(data));
+        }
+        completePromise(() => resolve(data));
+      })
       .catch((e) => completePromise(() => reject(e)));
   });
 }