@livestore/wa-sqlite 1.0.1-dev.0

package/src/examples/MemoryAsyncVFS.js

@@ -0,0 +1,100 @@
+// Copyright 2024 Roy T. Hashimoto. All Rights Reserved.
+import { MemoryVFS } from './MemoryVFS.js';
+
+// Sample asynchronous in-memory filesystem. This filesystem requires an
+// asynchronous WebAssembly build (Asyncify or JSPI).
+export class MemoryAsyncVFS extends MemoryVFS {
+
+  static async create(name, module) {
+    const vfs = new MemoryVFS(name, module);
+    await vfs.isReady();
+    return vfs;
+  }
+
+  constructor(name, module) {
+    super(name, module);
+  }
+
+  async close() {
+    for (const fileId of this.mapIdToFile.keys()) {
+      await this.xClose(fileId);
+    }
+  }
+
+  /**
+   * @param {string?} name 
+   * @param {number} fileId 
+   * @param {number} flags 
+   * @param {DataView} pOutFlags 
+   * @returns {Promise<number>}
+   */
+  async jOpen(name, fileId, flags, pOutFlags) {
+    return super.jOpen(name, fileId, flags, pOutFlags);
+  }
+
+  /**
+   * @param {number} fileId 
+   * @returns {Promise<number>}
+   */
+  async jClose(fileId) {
+    return super.jClose(fileId);
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {Uint8Array} pData 
+   * @param {number} iOffset
+   * @returns {Promise<number>}
+   */
+  async jRead(fileId, pData, iOffset) {
+    return super.jRead(fileId, pData, iOffset);
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {Uint8Array} pData 
+   * @param {number} iOffset
+   * @returns {Promise<number>}
+   */
+  async jWrite(fileId, pData, iOffset) {
+    return super.jWrite(fileId, pData, iOffset);
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {number} iSize 
+   * @returns {Promise<number>}
+   */
+  async xTruncate(fileId, iSize) {
+    return super.jTruncate(fileId, iSize);
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {DataView} pSize64 
+   * @returns {Promise<number>}
+   */
+  async jFileSize(fileId, pSize64) {
+    return super.jFileSize(fileId, pSize64);
+  }
+
+  /**
+   * 
+   * @param {string} name 
+   * @param {number} syncDir 
+   * @returns {Promise<number>}
+   */
+  async jDelete(name, syncDir) {
+    return super.jDelete(name, syncDir);
+  }
+
+  /**
+   * @param {string} name 
+   * @param {number} flags 
+   * @param {DataView} pResOut 
+   * @returns {Promise<number>}
+   */
+  async jAccess(name, flags, pResOut) {
+    return super.jAccess(name, flags, pResOut);
+  }
+}

package/src/examples/MemoryVFS.js

@@ -0,0 +1,176 @@
+// Copyright 2024 Roy T. Hashimoto. All Rights Reserved.
+import { FacadeVFS } from '../FacadeVFS.js';
+import * as VFS from '../VFS.js';
+
+// Sample in-memory filesystem.
+export class MemoryVFS extends FacadeVFS {
+  // Map of existing files, keyed by filename.
+  mapNameToFile = new Map();
+
+  // Map of open files, keyed by id (sqlite3_file pointer).
+  mapIdToFile = new Map();
+
+  static async create(name, module) {
+    const vfs = new MemoryVFS(name, module);
+    await vfs.isReady();
+    return vfs;
+  }
+
+  constructor(name, module) {
+    super(name, module);
+  }
+
+  close() {
+    for (const fileId of this.mapIdToFile.keys()) {
+      this.jClose(fileId);
+    }
+  }
+
+  /**
+   * @param {string?} filename 
+   * @param {number} fileId 
+   * @param {number} flags 
+   * @param {DataView} pOutFlags 
+   * @returns {number|Promise<number>}
+   */
+  jOpen(filename, fileId, flags, pOutFlags) {
+    const url = new URL(filename || Math.random().toString(36).slice(2), 'file://');
+    const pathname = url.pathname;
+
+    let file = this.mapNameToFile.get(pathname);
+    if (!file) {
+      if (flags & VFS.SQLITE_OPEN_CREATE) {
+        // Create a new file object.
+        file = {
+          pathname,
+          flags,
+          size: 0,
+          data: new ArrayBuffer(0)
+        };
+        this.mapNameToFile.set(pathname, file);
+      } else {
+        return VFS.SQLITE_CANTOPEN;
+      }
+    }
+
+    // Put the file in the opened files map.
+    this.mapIdToFile.set(fileId, file);
+    pOutFlags.setInt32(0, flags, true);
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId 
+   * @returns {number|Promise<number>}
+   */
+  jClose(fileId) {
+    const file = this.mapIdToFile.get(fileId);
+    this.mapIdToFile.delete(fileId);
+
+    if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
+      this.mapNameToFile.delete(file.pathname);
+    }
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {Uint8Array} pData 
+   * @param {number} iOffset
+   * @returns {number|Promise<number>}
+   */
+  jRead(fileId, pData, iOffset) {
+    const file = this.mapIdToFile.get(fileId);
+
+    // Clip the requested read to the file boundary.
+    const bgn = Math.min(iOffset, file.size);
+    const end = Math.min(iOffset + pData.byteLength, file.size);
+    const nBytes = end - bgn;
+
+    if (nBytes) {
+      pData.set(new Uint8Array(file.data, bgn, nBytes));
+    }
+
+    if (nBytes < pData.byteLength) {
+      // Zero unused area of read buffer.
+      pData.fill(0, nBytes);
+      return VFS.SQLITE_IOERR_SHORT_READ;
+    }
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {Uint8Array} pData 
+   * @param {number} iOffset
+   * @returns {number|Promise<number>}
+   */
+  jWrite(fileId, pData, iOffset) {
+    const file = this.mapIdToFile.get(fileId);
+    if (iOffset + pData.byteLength > file.data.byteLength) {
+      // Resize the ArrayBuffer to hold more data.
+      const newSize = Math.max(iOffset + pData.byteLength, 2 * file.data.byteLength);
+      const data = new ArrayBuffer(newSize);
+      new Uint8Array(data).set(new Uint8Array(file.data, 0, file.size));
+      file.data = data;
+    }
+
+    // Copy data.
+    new Uint8Array(file.data, iOffset, pData.byteLength).set(pData);
+    file.size = Math.max(file.size, iOffset + pData.byteLength);
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {number} iSize 
+   * @returns {number|Promise<number>}
+   */
+  jTruncate(fileId, iSize) {
+    const file = this.mapIdToFile.get(fileId);
+
+    // For simplicity we don't make the ArrayBuffer smaller.
+    file.size = Math.min(file.size, iSize);
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {DataView} pSize64 
+   * @returns {number|Promise<number>}
+   */
+  jFileSize(fileId, pSize64) {
+    const file = this.mapIdToFile.get(fileId);
+
+    pSize64.setBigInt64(0, BigInt(file.size), true);
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {string} name 
+   * @param {number} syncDir 
+   * @returns {number|Promise<number>}
+   */
+  jDelete(name, syncDir) {
+    const url = new URL(name, 'file://');
+    const pathname = url.pathname;
+
+    this.mapNameToFile.delete(pathname);
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {string} name 
+   * @param {number} flags 
+   * @param {DataView} pResOut 
+   * @returns {number|Promise<number>}
+   */
+  jAccess(name, flags, pResOut) {
+    const url = new URL(name, 'file://');
+    const pathname = url.pathname;
+
+    const file = this.mapNameToFile.get(pathname);
+    pResOut.setInt32(0, file ? 1 : 0, true);
+    return VFS.SQLITE_OK;
+  }
+}

package/src/examples/OPFSAdaptiveVFS.js

@@ -0,0 +1,437 @@
+// Copyright 2024 Roy T. Hashimoto. All Rights Reserved.
+import { FacadeVFS } from '../FacadeVFS.js';
+import * as VFS from '../VFS.js';
+import { WebLocksMixin } from '../WebLocksMixin.js';
+
+const LOCK_NOTIFY_INTERVAL = 1000;
+
+const hasUnsafeAccessHandle =
+  globalThis.FileSystemSyncAccessHandle.prototype.hasOwnProperty('mode');
+
+/**
+ * @param {string} pathname 
+ * @param {boolean} create 
+ * @returns {Promise<[FileSystemDirectoryHandle, string]>}
+ */
+async function getPathComponents(pathname, create) {
+  const [_, directories, filename] = pathname.match(/[/]?(.*)[/](.*)$/);
+
+  let directoryHandle = await navigator.storage.getDirectory();
+  for (const directory of directories.split('/')) {
+    if (directory) {
+      directoryHandle = await directoryHandle.getDirectoryHandle(directory, { create });
+    }
+  }
+  return [directoryHandle, filename];
+};
+
+class File {
+  /** @type {string} */ pathname;
+  /** @type {number} */ flags;
+  /** @type {FileSystemFileHandle} */ fileHandle;
+  /** @type {FileSystemSyncAccessHandle} */ accessHandle;
+
+  // The rest of the properties are for platforms without readwrite-unsafe
+  // access handles. Only one connection can have an open access handle
+  // so coordination is needed in addition to the SQLite locking model.
+  //
+  // Opening and closing the access handle is expensive so we leave the
+  // handle open unless another connection signals on BroadcastChannel.
+  /** @type {BroadcastChannel} */ handleRequestChannel;
+  /** @type {function} */ handleLockReleaser = null;
+  /** @type {boolean} */ isHandleRequested = false;
+  /** @type {boolean} */ isFileLocked = false;
+
+  // SQLite makes one read on file open that is not protected by a lock.
+  // This needs to be handled as a special case.
+  /** @type {function} */ openLockReleaser = null;
+
+  constructor(pathname, flags) {
+    this.pathname = pathname;
+    this.flags = flags;
+  }
+}
+
+export class OPFSAdaptiveVFS extends WebLocksMixin(FacadeVFS) {
+  /** @type {Map<number, File>} */ mapIdToFile = new Map();
+  lastError = null;
+
+  log = null;
+
+  static async create(name, module, options) {
+    const vfs = new OPFSAdaptiveVFS(name, module, options);
+    await vfs.isReady();
+    return vfs;
+  }
+
+  constructor(name, module, options = {}) {
+    super(name, module, options);
+  }
+  
+  getFilename(fileId) {
+    const pathname = this.mapIdToFile.get(fileId).pathname;
+    return `OPFS:${pathname}`
+  }
+
+  /**
+   * @param {string?} zName 
+   * @param {number} fileId 
+   * @param {number} flags 
+   * @param {DataView} pOutFlags 
+   * @returns {Promise<number>}
+   */
+  async jOpen(zName, fileId, flags, pOutFlags) {
+    try {
+      const url = new URL(zName || Math.random().toString(36).slice(2), 'file://');
+      const pathname = url.pathname;
+
+      const file = new File(pathname, flags);
+      this.mapIdToFile.set(fileId, file);
+
+      const create = !!(flags & VFS.SQLITE_OPEN_CREATE);
+      const [directoryHandle, filename] = await getPathComponents(pathname, create);
+      file.fileHandle = await directoryHandle.getFileHandle(filename, { create });
+
+      if ((flags & VFS.SQLITE_OPEN_MAIN_DB) && !hasUnsafeAccessHandle) {
+        file.handleRequestChannel = new BroadcastChannel(this.getFilename(fileId));
+
+        // Acquire the access handle lock. The first read of a database
+        // file is done outside xLock/xUnlock so we get that lock here.
+        function notify() {
+          file.handleRequestChannel.postMessage(null);
+        }
+        const notifyId = setInterval(notify, LOCK_NOTIFY_INTERVAL);
+        setTimeout(notify);
+
+        file.openLockReleaser = await new Promise((resolve, reject) => {
+          navigator.locks.request(this.getFilename(fileId), lock => {
+            clearInterval(notifyId);
+            if (!lock) return reject();
+            return new Promise(release => {
+              resolve(release);
+            });
+          });
+        });
+        this.log?.('access handle acquired for open');
+      }
+
+      // @ts-ignore
+      file.accessHandle = await file.fileHandle.createSyncAccessHandle({
+        mode: 'readwrite-unsafe'
+      });
+  
+      pOutFlags.setInt32(0, flags, true);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_CANTOPEN;
+    }
+  }
+
+  /**
+   * @param {string} zName 
+   * @param {number} syncDir 
+   * @returns {Promise<number>}
+   */
+  async jDelete(zName, syncDir) {
+    try {
+      const url = new URL(zName, 'file://');
+      const pathname = url.pathname;
+   
+      const [directoryHandle, name] = await getPathComponents(pathname, false);
+      const result = directoryHandle.removeEntry(name, { recursive: false });
+      if (syncDir) {
+        await result;
+      }
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      return VFS.SQLITE_IOERR_DELETE;
+    }
+  }
+
+  /**
+   * @param {string} zName 
+   * @param {number} flags 
+   * @param {DataView} pResOut 
+   * @returns {Promise<number>}
+   */
+  async jAccess(zName, flags, pResOut) {
+    try {
+      const url = new URL(zName, 'file://');
+      const pathname = url.pathname;
+
+      const [directoryHandle, dbName] = await getPathComponents(pathname, false);
+      const fileHandle = await directoryHandle.getFileHandle(dbName, { create: false });
+      pResOut.setInt32(0, 1, true);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      if (e.name === 'NotFoundError') {
+        pResOut.setInt32(0, 0, true);
+        return VFS.SQLITE_OK;
+      }
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_ACCESS;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @returns {Promise<number>}
+   */
+  async jClose(fileId) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      this.mapIdToFile.delete(fileId);
+      await file?.accessHandle?.close();
+
+      if (file?.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
+        const [directoryHandle, name] = await getPathComponents(file.pathname, false);
+        await directoryHandle.removeEntry(name, { recursive: false });
+      }
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      return VFS.SQLITE_IOERR_DELETE;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {Uint8Array} pData 
+   * @param {number} iOffset
+   * @returns {number}
+   */
+  jRead(fileId, pData, iOffset) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+
+      // On Chrome (at least), passing pData to accessHandle.read() is
+      // an error because pData is a Proxy of a Uint8Array. Calling
+      // subarray() produces a real Uint8Array and that works.
+      const bytesRead = file.accessHandle.read(pData.subarray(), { at: iOffset });
+      if (file.openLockReleaser) {
+        // We obtained the access handle on file open.
+        file.accessHandle.close();
+        file.accessHandle = null;
+        file.openLockReleaser();
+        file.openLockReleaser = null;
+        this.log?.('access handle released for open');
+      }
+
+      if (bytesRead < pData.byteLength) {
+        pData.fill(0, bytesRead);
+        return VFS.SQLITE_IOERR_SHORT_READ;
+      }
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_READ;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {Uint8Array} pData 
+   * @param {number} iOffset
+   * @returns {number}
+   */
+  jWrite(fileId, pData, iOffset) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+
+      // On Chrome (at least), passing pData to accessHandle.write() is
+      // an error because pData is a Proxy of a Uint8Array. Calling
+      // subarray() produces a real Uint8Array and that works.
+      file.accessHandle.write(pData.subarray(), { at: iOffset });
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_WRITE;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {number} iSize 
+   * @returns {number}
+   */
+  jTruncate(fileId, iSize) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      file.accessHandle.truncate(iSize);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_TRUNCATE;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {number} flags 
+   * @returns {number}
+   */
+  jSync(fileId, flags) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      file.accessHandle.flush();
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_FSYNC;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {DataView} pSize64 
+   * @returns {number}
+   */
+  jFileSize(fileId, pSize64) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      const size = file.accessHandle.getSize();
+      pSize64.setBigInt64(0, BigInt(size), true);
+      return VFS.SQLITE_OK;
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR_FSTAT;
+    }
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {number} lockType 
+   * @returns {Promise<number>}
+   */
+  async jLock(fileId, lockType) {
+    if (hasUnsafeAccessHandle) return super.jLock(fileId, lockType);
+
+    const file = this.mapIdToFile.get(fileId);
+    if (!file.isFileLocked) {
+      file.isFileLocked = true;
+      if (!file.handleLockReleaser) {
+        // Listen for other connections wanting the access handle.
+        file.handleRequestChannel.onmessage = event => {
+          if(!file.isFileLocked) {
+            // We have the access handle but the file is not locked.
+            // Release the access handle for the requester.
+            file.accessHandle.close();
+            file.accessHandle = null;
+            file.handleLockReleaser();
+            file.handleLockReleaser = null;
+            this.log?.('access handle requested and released');
+          } else {
+            // We're still using the access handle, so mark it to be
+            // released when we're done.
+            file.isHandleRequested = true;
+            this.log?.('access handle requested');
+          }
+          file.handleRequestChannel.onmessage = null;
+        };
+
+        // We don't have the access handle. First acquire the lock.
+        file.handleLockReleaser = await new Promise((resolve, reject) => {
+          // Tell everyone we want the access handle.
+          function notify() {
+            file.handleRequestChannel.postMessage(null);
+          }
+          const notifyId = setInterval(notify, LOCK_NOTIFY_INTERVAL);
+          setTimeout(notify);
+
+          navigator.locks.request(this.getFilename(fileId), lock => {
+            clearInterval(notifyId);
+            if (!lock) return reject();
+            return new Promise(release => {
+              resolve(release);
+            });
+          });
+        });
+
+        // The access handle should now be available.
+        file.accessHandle = await file.fileHandle.createSyncAccessHandle();
+        this.log?.('access handle acquired');
+      }
+
+    }
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId 
+   * @param {number} lockType 
+   * @returns {Promise<number>}
+   */
+  async jUnlock(fileId, lockType) {
+    if (hasUnsafeAccessHandle) return super.jUnlock(fileId, lockType);
+
+    if (lockType === VFS.SQLITE_LOCK_NONE) {
+      const file = this.mapIdToFile.get(fileId);
+      if (file.isHandleRequested) {
+        if (file.handleLockReleaser) {
+          // Another connection wants the access handle.
+          file.accessHandle.close();
+          file.accessHandle = null;
+          file.handleLockReleaser();
+          file.handleLockReleaser = null;
+          this.log?.('access handle released');
+        }
+        file.isHandleRequested = false;
+      }
+      file.isFileLocked = false;
+    }
+    return VFS.SQLITE_OK;
+  }
+
+  /**
+   * @param {number} fileId
+   * @param {number} op
+   * @param {DataView} pArg
+   * @returns {number|Promise<number>}
+   */
+  jFileControl(fileId, op, pArg) {
+    try {
+      const file = this.mapIdToFile.get(fileId);
+      switch (op) {
+        case VFS.SQLITE_FCNTL_PRAGMA:
+          const key = extractString(pArg, 4);
+          const value = extractString(pArg, 8);
+          this.log?.('xFileControl', file.pathname, 'PRAGMA', key, value);
+          switch (key.toLowerCase()) {
+            case 'journal_mode':
+              if (value &&
+                  !hasUnsafeAccessHandle && 
+                  !['off', 'memory', 'delete', 'wal'].includes(value.toLowerCase())) {
+                throw new Error('journal_mode must be "off", "memory", "delete", or "wal"');
+              }
+              break;
+            case 'write_hint':
+              return super.jFileControl(fileId, WebLocksMixin.WRITE_HINT_OP_CODE, null);
+          }
+          break;
+      }
+    } catch (e) {
+      this.lastError = e;
+      return VFS.SQLITE_IOERR;
+    }
+    return VFS.SQLITE_NOTFOUND;
+  }
+
+  jGetLastError(zBuf) {
+    if (this.lastError) {
+      console.error(this.lastError);
+      const outputArray = zBuf.subarray(0, zBuf.byteLength - 1);
+      const { written } = new TextEncoder().encodeInto(this.lastError.message, outputArray);
+      zBuf[written] = 0;
+    }
+    return VFS.SQLITE_OK
+  }
+}
+
+function extractString(dataView, offset) {
+  const p = dataView.getUint32(offset, true);
+  if (p) {
+    const chars = new Uint8Array(dataView.buffer, p);
+    return new TextDecoder().decode(chars.subarray(0, chars.indexOf(0)));
+  }
+  return null;
+}