@livestore/sqlite-wasm 0.0.0-snapshot-1d99fea7d2ce2c7a5d9ed0a3752f8a7bda6bc3db

package/src/FacadeVFS.ts

@@ -0,0 +1,510 @@
+// Based on https://github.com/rhashimoto/wa-sqlite/blob/master/src/FacadeVFS.js
+
+/* eslint-disable unicorn/prefer-code-point */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
+
+const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor
+
+export class FacadeVFS extends VFS.Base {
+  /**
+   * @param {string} name
+   * @param {object} module
+   */
+  constructor(name, module) {
+    super(name, module)
+  }
+
+  /**
+   * Override to indicate which methods are asynchronous.
+   * @param {string} methodName
+   * @returns {boolean}
+   */
+  hasAsyncMethod(methodName) {
+    // The input argument is a string like "xOpen", so convert to "jOpen".
+    // Then check if the method exists and is async.
+    const jMethodName = `j${methodName.slice(1)}`
+    return this[jMethodName] instanceof AsyncFunction
+  }
+
+  /**
+   * Return the filename for a file id for use by mixins.
+   * @param {number} pFile
+   * @returns {string}
+   */
+  getFilename(pFile) {
+    throw new Error('unimplemented')
+  }
+
+  /**
+   * @param {string?} filename
+   * @param {number} pFile
+   * @param {number} flags
+   * @param {DataView} pOutFlags
+   * @returns {number|Promise<number>}
+   */
+  jOpen(filename, pFile, flags, pOutFlags): number {
+    return VFS.SQLITE_CANTOPEN
+  }
+
+  /**
+   * @param {string} filename
+   * @param {number} syncDir
+   * @returns {number|Promise<number>}
+   */
+  jDelete(filename, syncDir): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {string} filename
+   * @param {number} flags
+   * @param {DataView} pResOut
+   * @returns {number|Promise<number>}
+   */
+  jAccess(filename, flags, pResOut): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {string} filename
+   * @param {Uint8Array} zOut
+   * @returns {number|Promise<number>}
+   */
+  jFullPathname(filename, zOut): number {
+    // Copy the filename to the output buffer.
+    const { read, written } = new TextEncoder().encodeInto(filename, zOut)
+    if (read < filename.length) return VFS.SQLITE_IOERR
+    if (written >= zOut.length) return VFS.SQLITE_IOERR
+    zOut[written] = 0
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {Uint8Array} zBuf
+   * @returns {number|Promise<number>}
+   */
+  jGetLastError(zBuf) {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @returns {number|Promise<number>}
+   */
+  jClose(pFile): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {Uint8Array} pData
+   * @param {number} iOffset
+   * @returns {number|Promise<number>}
+   */
+  jRead(pFile, pData, iOffset): number {
+    pData.fill(0)
+    return VFS.SQLITE_IOERR_SHORT_READ
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {Uint8Array} pData
+   * @param {number} iOffset
+   * @returns {number|Promise<number>}
+   */
+  jWrite(pFile, pData, iOffset): number {
+    return VFS.SQLITE_IOERR_WRITE
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} size
+   * @returns {number|Promise<number>}
+   */
+  jTruncate(pFile, size): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} flags
+   * @returns {number|Promise<number>}
+   */
+  jSync(pFile, flags): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {DataView} pSize
+   * @returns {number|Promise<number>}
+   */
+  jFileSize(pFile, pSize): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} lockType
+   * @returns {number|Promise<number>}
+   */
+  jLock(pFile, lockType): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} lockType
+   * @returns {number|Promise<number>}
+   */
+  jUnlock(pFile, lockType): number {
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {DataView} pResOut
+   * @returns {number|Promise<number>}
+   */
+  jCheckReservedLock(pFile, pResOut): number {
+    pResOut.setInt32(0, 0, true)
+    return VFS.SQLITE_OK
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} op
+   * @param {DataView} pArg
+   * @returns {number|Promise<number>}
+   */
+  jFileControl(pFile, op, pArg): number {
+    return VFS.SQLITE_NOTFOUND
+  }
+
+  /**
+   * @param {number} pFile
+   * @returns {number|Promise<number>}
+   */
+  jSectorSize(pFile): number {
+    return super.xSectorSize(pFile)
+  }
+
+  /**
+   * @param {number} pFile
+   * @returns {number|Promise<number>}
+   */
+  jDeviceCharacteristics(pFile): number {
+    return 0
+  }
+
+  /**
+   * @param {number} pVfs
+   * @param {number} zName
+   * @param {number} pFile
+   * @param {number} flags
+   * @param {number} pOutFlags
+   * @returns {number|Promise<number>}
+   */
+  xOpen(pVfs, zName, pFile, flags, pOutFlags): number {
+    const filename = this.#decodeFilename(zName, flags)
+    const pOutFlagsView = this.#makeTypedDataView('Int32', pOutFlags)
+    this['log']?.('jOpen', filename, pFile, '0x' + flags.toString(16))
+    return this.jOpen(filename, pFile, flags, pOutFlagsView)
+  }
+
+  /**
+   * @param {number} pVfs
+   * @param {number} zName
+   * @param {number} syncDir
+   * @returns {number|Promise<number>}
+   */
+  xDelete(pVfs, zName, syncDir) {
+    const filename = this._module.UTF8ToString(zName)
+    this['log']?.('jDelete', filename, syncDir)
+    return this.jDelete(filename, syncDir)
+  }
+
+  /**
+   * @param {number} pVfs
+   * @param {number} zName
+   * @param {number} flags
+   * @param {number} pResOut
+   * @returns {number|Promise<number>}
+   */
+  xAccess(pVfs, zName, flags, pResOut) {
+    const filename = this._module.UTF8ToString(zName)
+    const pResOutView = this.#makeTypedDataView('Int32', pResOut)
+    this['log']?.('jAccess', filename, flags)
+    return this.jAccess(filename, flags, pResOutView)
+  }
+
+  /**
+   * @param {number} pVfs
+   * @param {number} zName
+   * @param {number} nOut
+   * @param {number} zOut
+   * @returns {number|Promise<number>}
+   */
+  xFullPathname(pVfs, zName, nOut, zOut) {
+    const filename = this._module.UTF8ToString(zName)
+    const zOutArray = this._module.HEAPU8.subarray(zOut, zOut + nOut)
+    this['log']?.('jFullPathname', filename, nOut)
+    return this.jFullPathname(filename, zOutArray)
+  }
+
+  /**
+   * @param {number} pVfs
+   * @param {number} nBuf
+   * @param {number} zBuf
+   * @returns {number|Promise<number>}
+   */
+  xGetLastError(pVfs, nBuf, zBuf) {
+    const zBufArray = this._module.HEAPU8.subarray(zBuf, zBuf + nBuf)
+    this['log']?.('jGetLastError', nBuf)
+    return this.jGetLastError(zBufArray)
+  }
+
+  /**
+   * @param {number} pFile
+   * @returns {number|Promise<number>}
+   */
+  xClose(pFile) {
+    this['log']?.('jClose', pFile)
+    return this.jClose(pFile)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} pData
+   * @param {number} iAmt
+   * @param {number} iOffsetLo
+   * @param {number} iOffsetHi
+   * @returns {number|Promise<number>}
+   */
+  xRead(pFile, pData, iAmt, iOffsetLo, iOffsetHi) {
+    const pDataArray = this.#makeDataArray(pData, iAmt)
+    const iOffset = delegalize(iOffsetLo, iOffsetHi)
+    this['log']?.('jRead', pFile, iAmt, iOffset)
+    return this.jRead(pFile, pDataArray, iOffset)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} pData
+   * @param {number} iAmt
+   * @param {number} iOffsetLo
+   * @param {number} iOffsetHi
+   * @returns {number|Promise<number>}
+   */
+  xWrite(pFile, pData, iAmt, iOffsetLo, iOffsetHi) {
+    const pDataArray = this.#makeDataArray(pData, iAmt)
+    const iOffset = delegalize(iOffsetLo, iOffsetHi)
+    this['log']?.('jWrite', pFile, pDataArray, iOffset)
+    return this.jWrite(pFile, pDataArray, iOffset)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} sizeLo
+   * @param {number} sizeHi
+   * @returns {number|Promise<number>}
+   */
+  xTruncate(pFile, sizeLo, sizeHi) {
+    const size = delegalize(sizeLo, sizeHi)
+    this['log']?.('jTruncate', pFile, size)
+    return this.jTruncate(pFile, size)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} flags
+   * @returns {number|Promise<number>}
+   */
+  xSync(pFile, flags) {
+    this['log']?.('jSync', pFile, flags)
+    return this.jSync(pFile, flags)
+  }
+
+  /**
+   *
+   * @param {number} pFile
+   * @param {number} pSize
+   * @returns {number|Promise<number>}
+   */
+  xFileSize(pFile, pSize) {
+    const pSizeView = this.#makeTypedDataView('BigInt64', pSize)
+    this['log']?.('jFileSize', pFile)
+    return this.jFileSize(pFile, pSizeView)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} lockType
+   * @returns {number|Promise<number>}
+   */
+  xLock(pFile, lockType) {
+    this['log']?.('jLock', pFile, lockType)
+    return this.jLock(pFile, lockType)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} lockType
+   * @returns {number|Promise<number>}
+   */
+  xUnlock(pFile, lockType) {
+    this['log']?.('jUnlock', pFile, lockType)
+    return this.jUnlock(pFile, lockType)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} pResOut
+   * @returns {number|Promise<number>}
+   */
+  xCheckReservedLock(pFile, pResOut) {
+    const pResOutView = this.#makeTypedDataView('Int32', pResOut)
+    this['log']?.('jCheckReservedLock', pFile)
+    return this.jCheckReservedLock(pFile, pResOutView)
+  }
+
+  /**
+   * @param {number} pFile
+   * @param {number} op
+   * @param {number} pArg
+   * @returns {number|Promise<number>}
+   */
+  xFileControl(pFile, op, pArg) {
+    const pArgView = new DataView(this._module.HEAPU8.buffer, this._module.HEAPU8.byteOffset + pArg)
+    this['log']?.('jFileControl', pFile, op, pArgView)
+    return this.jFileControl(pFile, op, pArgView)
+  }
+
+  /**
+   * @param {number} pFile
+   * @returns {number|Promise<number>}
+   */
+  xSectorSize(pFile) {
+    this['log']?.('jSectorSize', pFile)
+    return this.jSectorSize(pFile)
+  }
+
+  /**
+   * @param {number} pFile
+   * @returns {number|Promise<number>}
+   */
+  xDeviceCharacteristics(pFile) {
+    this['log']?.('jDeviceCharacteristics', pFile)
+    return this.jDeviceCharacteristics(pFile)
+  }
+
+  /**
+   * Wrapped DataView for pointer arguments.
+   * Pointers to a single value are passed using DataView. A Proxy
+   * wrapper prevents use of incorrect type or endianness.
+   * @param {'Int32'|'BigInt64'} type
+   * @param {number} byteOffset
+   * @returns {DataView}
+   */
+  #makeTypedDataView(type, byteOffset) {
+    const byteLength = type === 'Int32' ? 4 : 8
+    const getter = `get${type}`
+    const setter = `set${type}`
+    const makeDataView = () =>
+      new DataView(this._module.HEAPU8.buffer, this._module.HEAPU8.byteOffset + byteOffset, byteLength)
+    let dataView = makeDataView()
+    return new Proxy(dataView, {
+      get(_, prop) {
+        if (dataView.buffer.byteLength === 0) {
+          // WebAssembly memory resize detached the buffer.
+          dataView = makeDataView()
+        }
+        if (prop === getter) {
+          return function (byteOffset, littleEndian) {
+            if (!littleEndian) throw new Error('must be little endian')
+            return dataView[prop](byteOffset, littleEndian)
+          }
+        }
+        if (prop === setter) {
+          return function (byteOffset, value, littleEndian) {
+            if (!littleEndian) throw new Error('must be little endian')
+            return dataView[prop](byteOffset, value, littleEndian)
+          }
+        }
+        if (typeof prop === 'string' && /^(get)|(set)/.test(prop)) {
+          throw new Error('invalid type')
+        }
+        const result = dataView[prop]
+        return typeof result === 'function' ? result.bind(dataView) : result
+      },
+    })
+  }
+
+  /**
+   * @param {number} byteOffset
+   * @param {number} byteLength
+   */
+  #makeDataArray(byteOffset, byteLength) {
+    let target = this._module.HEAPU8.subarray(byteOffset, byteOffset + byteLength)
+    return new Proxy(target, {
+      get: (_, prop, receiver) => {
+        if (target.buffer.byteLength === 0) {
+          // WebAssembly memory resize detached the buffer.
+          target = this._module.HEAPU8.subarray(byteOffset, byteOffset + byteLength)
+        }
+        const result = target[prop]
+        return typeof result === 'function' ? result.bind(target) : result
+      },
+    })
+  }
+
+  #decodeFilename(zName, flags) {
+    if (flags & VFS.SQLITE_OPEN_URI) {
+      // The first null-terminated string is the URI path. Subsequent
+      // strings are query parameter keys and values.
+      // https://www.sqlite.org/c3ref/open.html#urifilenamesinsqlite3open
+      let pName = zName
+      let state = 1
+      const charCodes = []
+      while (state) {
+        const charCode = this._module.HEAPU8[pName++]
+        if (charCode) {
+          charCodes.push(charCode)
+        } else {
+          if (!this._module.HEAPU8[pName]) state = null
+          switch (state) {
+            case 1: {
+              // path
+              charCodes.push('?'.charCodeAt(0))
+              state = 2
+              break
+            }
+            case 2: {
+              // key
+              charCodes.push('='.charCodeAt(0))
+              state = 3
+              break
+            }
+            case 3: {
+              // value
+              charCodes.push('&'.charCodeAt(0))
+              state = 2
+              break
+            }
+          }
+        }
+      }
+      return new TextDecoder().decode(new Uint8Array(charCodes))
+    }
+    return zName ? this._module.UTF8ToString(zName) : null
+  }
+}
+// Emscripten "legalizes" 64-bit integer arguments by passing them as
+// two 32-bit signed integers.
+function delegalize(lo32, hi32) {
+  return hi32 * 0x1_00_00_00_00 + lo32 + (lo32 < 0 ? 2 ** 32 : 0)
+}

package/src/ambient.d.ts

@@ -0,0 +1,18 @@
+// https://developer.mozilla.org/en-US/docs/Web/API/FileSystemSyncAccessHandle
+interface FileSystemSyncAccessHandle {
+  close: () => void
+  flush: () => Promise<void>
+  getSize: () => number
+  read: (buffer: Uint8Array | Uint32Array, options?: FileSystemReadWriteOptions) => number
+  truncate: (newSize: number) => void
+  write: (buffer: Uint8Array | Uint32Array, options?: FileSystemReadWriteOptions) => number
+  seek: (offset: number) => void
+}
+
+interface FileSystemReadWriteOptions {
+  at?: number
+}
+
+interface FileSystemFileHandle {
+  createSyncAccessHandle: () => Promise<FileSystemSyncAccessHandle>
+}

package/src/browser/mod.ts

@@ -0,0 +1,109 @@
+import type { MakeSqliteDb, PersistenceInfo, SqliteDb } from '@livestore/common'
+import { Effect, Hash } from '@livestore/utils/effect'
+import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
+
+import { makeInMemoryDb } from '../in-memory-vfs.js'
+import { makeSqliteDb } from '../make-sqlite-db.js'
+import type { AccessHandlePoolVFS } from './opfs/AccessHandlePoolVFS.js'
+import { makeOpfsDb } from './opfs/index.js'
+
+export * from './opfs/opfs-sah-pool.js'
+
+export type WebDatabaseMetadataInMemory = {
+  _tag: 'in-memory'
+  vfs: MemoryVFS
+  dbPointer: number
+  persistenceInfo: PersistenceInfo
+  deleteDb: () => void
+  configureDb: (db: SqliteDb) => void
+}
+
+export type WebDatabaseMetadataOpfs = {
+  _tag: 'opfs'
+  vfs: AccessHandlePoolVFS
+  dbPointer: number
+  persistenceInfo: PersistenceInfo<{
+    opfsDirectory: string
+    /** Actual filename used by OPFS */
+    opfsFileName: string
+  }>
+  deleteDb: () => void
+  configureDb: (db: SqliteDb) => void
+}
+
+export type WebDatabaseMetadata = WebDatabaseMetadataInMemory | WebDatabaseMetadataOpfs
+
+export type WebDatabaseInputInMemory = {
+  _tag: 'in-memory'
+  configureDb?: (db: SqliteDb) => void
+}
+
+export type WebDatabaseInputOpfs = {
+  _tag: 'opfs'
+  /** Filename of the database file (only used when exporting/downloading the database) */
+  fileName: string
+  opfsDirectory: string
+  configureDb?: (db: SqliteDb) => void
+}
+
+export type WebDatabaseInput = WebDatabaseInputInMemory | WebDatabaseInputOpfs
+
+export const sqliteDbFactory =
+  ({
+    sqlite3,
+  }: {
+    sqlite3: SQLiteAPI
+  }): MakeSqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }, WebDatabaseInput, WebDatabaseMetadata> =>
+  (input: WebDatabaseInput) =>
+    Effect.gen(function* () {
+      if (input._tag === 'in-memory') {
+        const { dbPointer, vfs } = makeInMemoryDb(sqlite3)
+        return makeSqliteDb<WebDatabaseMetadataInMemory>({
+          sqlite3,
+          metadata: {
+            _tag: 'in-memory',
+            vfs,
+            dbPointer,
+            deleteDb: () => {},
+            configureDb: input.configureDb ?? (() => {}),
+            persistenceInfo: {
+              fileName: ':memory:',
+            },
+          },
+        }) as any
+      }
+
+      // TODO figure out the actual max length
+      const MAX_DB_FILENAME_LENGTH = 60
+
+      let dbFilename = input.fileName
+
+      if (input.fileName.length > MAX_DB_FILENAME_LENGTH) {
+        yield* Effect.logWarning(
+          `dbFilename too long: '${input.fileName}'. Max ${MAX_DB_FILENAME_LENGTH} chars, got ${input.fileName.length}. Hashing...`,
+        )
+        dbFilename = `hash-${Hash.string(input.fileName)}.db`
+      }
+
+      const { dbPointer, vfs } = yield* makeOpfsDb({
+        sqlite3,
+        directory: input.opfsDirectory,
+        fileName: dbFilename,
+      })
+
+      return makeSqliteDb<WebDatabaseMetadataOpfs>({
+        sqlite3,
+        metadata: {
+          _tag: 'opfs',
+          vfs,
+          dbPointer,
+          deleteDb: () => vfs.resetAccessHandle(input.fileName),
+          configureDb: input.configureDb ?? (() => {}),
+          persistenceInfo: {
+            fileName: dbFilename,
+            opfsDirectory: input.opfsDirectory,
+            opfsFileName: vfs.getOpfsFileName(dbFilename),
+          },
+        },
+      })
+    })